int logw_append(struct logw *lgw, struct msg *k, struct msg *v, msgtype_t t, int tbn) { int r = NESS_OK; char *base; uint32_t pos = 0; uint32_t size = + 4 /* length of beginning */ + 4 /* table number */ + CRC_SIZE; size += sizeof(k->size); size += k->size; if (v) { size += sizeof(v->size);; size += v->size; } _check_space(lgw, size); base = lgw->base; putuint32(base + pos, size); pos += 4; putuint32(base + pos, tbn); pos += 4; uint32_t fixsize = k->size; fixsize = ((fixsize << 8) | (char)t); putuint32(base + pos, fixsize); pos += 4; putnstr(base + pos, k->data, k->size); pos += k->size; if (v) { putuint32(base + pos, v->size); pos += 4; putnstr(base + pos, v->data, v->size); pos += v->size; } uint32_t xsum; /* not include the length of the header */ r = buf_xsum(base + 4, pos - 4, &xsum); if (r == 0) r = NESS_DO_XSUM_ERR; putuint32(base + pos, xsum); pos += 4; nassert(pos == size); lgw->size += size; ness_os_write(lgw->fd, base, size); return r; }
CTEST2(buf_test, checksum) { buf_clear(data->b); uint32_t cs1, cs2, cs3; uint64_t v1 = 123456789101112; buf_putnstr(data->b, "nessdata", 8); buf_putuint32(data->b, 0); buf_putuint32(data->b, 1); buf_putuint64(data->b, v1); buf_xsum(data->b->buf, data->b->NUL, &cs1); buf_putuint32(data->b, cs1); buf_skip(data->b, 8 + 4 + 4 + 8); buf_getuint32(data->b, &cs2); buf_xsum(data->b->buf, 8 + 4 + 4 + 8, &cs3); ASSERT_EQUAL(cs2, cs3); }
/* * header laytout stored in disk as: * +----------------------+ * | 'nesshdr*' | * +----------------------+ * | version | * +----------------------+ * | last-nid | * +----------------------+ * | roo-nid | * +----------------------+ * | blocksize | * +----------------------+ * | blockoff | * +----------------------+ */ int write_hdr_to_disk(int fd, struct hdr *hdr, DISKOFF off) { int r; uint32_t real_size; uint32_t align_size; struct buffer *wbuf = NULL; nassert(hdr->root_nid >= NID_START); wbuf = buf_new(1 << 20); buf_putnstr(wbuf, "nesshdr*", 8); buf_putuint32(wbuf, LAYOUT_VERSION); buf_putuint64(wbuf, hdr->last_nid); buf_putuint64(wbuf, hdr->root_nid); buf_putuint32(wbuf, hdr->blocksize); buf_putuint64(wbuf, hdr->blockoff); uint32_t xsum; if (!buf_xsum(wbuf->buf, wbuf->NUL, &xsum)) { r = NESS_DO_XSUM_ERR; goto ERR; } buf_putuint32(wbuf, xsum); real_size = wbuf->NUL; align_size = ALIGN(real_size); buf_putnull(wbuf, align_size - real_size); if (ness_os_pwrite(fd, wbuf->buf, align_size, off) != 0) { r = NESS_WRITE_ERR; goto ERR; } buf_free(wbuf); return NESS_OK; ERR: buf_free(wbuf); return r; }
int _deserialize_blockpairs_from_disk(int fd, struct block *b, struct hdr *hdr) { int r = NESS_ERR; uint32_t read_size; uint32_t align_size; struct buffer *rbuf; struct block_pair *pairs; read_size = hdr->blocksize; align_size = ALIGN(read_size); rbuf = buf_new(align_size); if (ness_os_pread(fd, rbuf->buf, align_size, hdr->blockoff) != (ssize_t)align_size) { r = NESS_READ_ERR; goto ERR; } if (!buf_seek(rbuf, read_size - CRC_SIZE)) goto ERR; uint32_t exp_xsum, act_xsum; if (!buf_getuint32(rbuf, &exp_xsum)) goto ERR; if (!buf_xsum(rbuf->buf, hdr->blocksize - CRC_SIZE, &act_xsum)) goto ERR; if (exp_xsum != act_xsum) { __ERROR("blockpairs xsum check error," "exp_xsum: [%" PRIu32 "]," "act_xsum: [%" PRIu32 "]", exp_xsum, act_xsum); goto ERR; } buf_seekfirst(rbuf); /* * skip magic with 8bytes */ if (!buf_skip(rbuf, 8)) goto ERR; uint32_t i; uint32_t block_count = 0U; if (!buf_getuint32(rbuf, &block_count)) goto ERR; pairs = xcalloc(block_count, sizeof(*pairs)); for (i = 0; i < block_count; i++) { if (!buf_getuint64(rbuf, &pairs[i].nid)) goto ERR1; if (!buf_getuint64(rbuf, &pairs[i].offset)) goto ERR1; if (!buf_getuint32(rbuf, &pairs[i].real_size)) goto ERR1; if (!buf_getuint32(rbuf, &pairs[i].skeleton_size)) goto ERR1; if (!buf_getuint32(rbuf, &pairs[i].height)) goto ERR1; pairs[i].used = 1; } if (block_count > 0) block_init(b, pairs, block_count); xfree(pairs); buf_free(rbuf); return NESS_OK; ERR: buf_free(rbuf); return r; ERR1: buf_free(rbuf); xfree(pairs); return r; }
/* * * blockpairs store in disk as: * +------------------------------------------------------+ * | 'blkpairs' | * +----------+------+-----------+---------------+--------+ * | nid | off | real-size | skeleton-size | height | * +----------+------+-----------+---------------+--------+ * ... */ int _serialize_blockpairs_to_disk(int fd, struct block *b, struct hdr *hdr) { uint32_t i; int r = NESS_OK; uint32_t used_count = 0; DISKOFF address; struct buffer *wbuf; wbuf = buf_new(1 << 20); for (i = 0; i < b->pairs_used; i++) { if (!b->pairs[i].used) continue; used_count++; } __DEBUG(" block pairs count [%d], used count [%d]", b->pairs_used, used_count); buf_putnstr(wbuf, "blkpairs", 8); buf_putuint32(wbuf, used_count); for (i = 0; i < b->pairs_used; i++) { if (!b->pairs[i].used) continue; buf_putuint64(wbuf, b->pairs[i].nid); buf_putuint64(wbuf, b->pairs[i].offset); buf_putuint32(wbuf, b->pairs[i].real_size); buf_putuint32(wbuf, b->pairs[i].skeleton_size); buf_putuint32(wbuf, b->pairs[i].height); } uint32_t xsum; if (!buf_xsum(wbuf->buf, wbuf->NUL, &xsum)) { r = NESS_DO_XSUM_ERR; goto ERR; } buf_putuint32(wbuf, xsum); uint32_t real_size; uint32_t align_size; real_size = wbuf->NUL; hdr->blocksize = real_size; align_size = ALIGN(real_size); buf_putnull(wbuf, align_size - real_size); address = block_alloc_off(b, 0, real_size, 0, 0); if (ness_os_pwrite(fd, wbuf->buf, align_size, address) != 0) { r = NESS_WRITE_ERR; goto ERR; } hdr->blockoff = address; buf_free(wbuf); return NESS_OK; ERR: buf_free(wbuf); return r; }
int read_hdr_from_disk(int fd, struct block *b, struct hdr **h, DISKOFF off) { int r = NESS_ERR; struct hdr *hdr = NULL; struct buffer *rbuf = NULL; uint32_t exp_xsum, act_xsum; uint32_t read_size, align_size; hdr = xcalloc(1, sizeof(*hdr)); read_size = ( + 8 /* magic */ + 8 /* last nid */ + 8 /* root nid */ + 4 /* version */ + 4 /* block size */ + 8 /* block offset */ + CRC_SIZE); /* checksum */ align_size = ALIGN(read_size); rbuf = buf_new(align_size); if (ness_os_pread(fd, rbuf->buf, align_size, off) != (ssize_t)align_size) { __ERROR("ness pread error, read size [%" PRIu32 "], " "offset [%" PRIu64 "]", align_size, 0UL); r = NESS_READ_ERR; goto ERR; } if (!buf_seek(rbuf, read_size - CRC_SIZE)) goto ERR; if (!buf_getuint32(rbuf, &exp_xsum)) goto ERR; if (!buf_xsum(rbuf->buf, read_size - CRC_SIZE, &act_xsum)) goto ERR; if (exp_xsum != act_xsum) { __ERROR("header xsum check error, " "exp_xsum: [%" PRIu32 "], " "act_xsum: [%" PRIu32 "], ", exp_xsum, act_xsum); r = NESS_HDR_XSUM_ERR; goto ERR; } buf_seekfirst(rbuf); if (!buf_skip(rbuf, 8)) goto ERR; if (!buf_getuint32(rbuf, &hdr->version)) goto ERR; if (!buf_getuint64(rbuf, &hdr->last_nid)) goto ERR; if (!buf_getuint64(rbuf, &hdr->root_nid)) goto ERR; if (!buf_getuint32(rbuf, &hdr->blocksize)) goto ERR; if (!buf_getuint64(rbuf, &hdr->blockoff)) goto ERR; nassert(hdr->root_nid >= NID_START); if (hdr->version < LAYOUT_MIN_SUPPORTED_VERSION) { r = NESS_LAYOUT_VERSION_OLDER_ERR; __ERROR("tree layout too older [%d], " "min_support_version [%d]", hdr->version, LAYOUT_MIN_SUPPORTED_VERSION); goto ERR; } /* block pairs */ r = _deserialize_blockpairs_from_disk(fd, b, hdr); if (r != NESS_OK) { r = NESS_DESERIAL_BLOCKPAIR_ERR; goto ERR; } *h = hdr; buf_free(rbuf); return NESS_OK; ERR: buf_free(rbuf); xfree(hdr); return r; }
int logr_read(struct logr *lgr, struct msg *k, struct msg *v, msgtype_t *t, uint32_t *tbn) { int r; uint32_t size = 0U; if (lgr->read < lgr->fsize) { char *base; uint32_t read; if (ness_os_read(lgr->fd, lgr->base, 4) != 4) { r = NESS_LOG_READ_SIZE_ERR; __ERROR(" log read size error, errno [%d]", r); goto ERR; } getuint32(lgr->base, &size); if (size > lgr->base_size) { lgr->base_size = size; lgr->base = xrealloc(lgr->base, lgr->base_size); } /* skip the length */ read = (size - 4); if (ness_os_read(lgr->fd, lgr->base, read) != read) { r = NESS_LOG_READ_DATA_ERR; __ERROR(" log read entry error, errno [%d]", r); goto ERR; } base = lgr->base; uint32_t exp_xsum; uint32_t act_xsum; getuint32(base + read - 4, &exp_xsum); buf_xsum(base, read - 4, &act_xsum); if (exp_xsum != act_xsum) { r = NESS_LOG_READ_XSUM_ERR; __ERROR("log read xsum error, exp_xsum [%" PRIu32 "],act_xsum [%" PRIu32 "]", exp_xsum, act_xsum); goto ERR; } int pos = 0; /* tbn */ getuint32(base + pos, tbn); pos += 4; /* key */ uint32_t fixsize; getuint32(base + pos, &fixsize); pos += 4; k->size = (fixsize >> 8); *t = (msgtype_t)(fixsize & 0xff); getnstr(base + pos, k->size, (char**)&k->data); pos += k->size; if (*t != MSG_DELETE) { /* value */ getuint32(base + pos, &v->size); pos += 4; getnstr(base + pos, v->size, (char**)&v->data); pos += v->size; } } else {