int logw_append(struct logw *lgw,
struct msg *k,
struct msg *v,
msgtype_t t,
int tbn)
{
int r = NESS_OK;
char *base;
uint32_t pos = 0;
uint32_t size = + 4 /* length of beginning */
+ 4 /* table number */
+ CRC_SIZE;
size += sizeof(k->size);
size += k->size;
if (v) {
size += sizeof(v->size);;
size += v->size;
}
_check_space(lgw, size);
base = lgw->base;
putuint32(base + pos, size);
pos += 4;
putuint32(base + pos, tbn);
pos += 4;
uint32_t fixsize = k->size;
fixsize = ((fixsize << 8) | (char)t);
putuint32(base + pos, fixsize);
pos += 4;
putnstr(base + pos, k->data, k->size);
pos += k->size;
if (v) {
putuint32(base + pos, v->size);
pos += 4;
putnstr(base + pos, v->data, v->size);
pos += v->size;
}
uint32_t xsum;
/* not include the length of the header */
r = buf_xsum(base + 4, pos - 4, &xsum);
if (r == 0)
r = NESS_DO_XSUM_ERR;
putuint32(base + pos, xsum);
pos += 4;
nassert(pos == size);
lgw->size += size;
ness_os_write(lgw->fd, base, size);
return r;
}
CTEST2(buf_test, checksum)
{
buf_clear(data->b);
uint32_t cs1, cs2, cs3;
uint64_t v1 = 123456789101112;
buf_putnstr(data->b, "nessdata", 8);
buf_putuint32(data->b, 0);
buf_putuint32(data->b, 1);
buf_putuint64(data->b, v1);
buf_xsum(data->b->buf, data->b->NUL, &cs1);
buf_putuint32(data->b, cs1);
buf_skip(data->b, 8 + 4 + 4 + 8);
buf_getuint32(data->b, &cs2);
buf_xsum(data->b->buf, 8 + 4 + 4 + 8, &cs3);
ASSERT_EQUAL(cs2, cs3);
}
/*
* header laytout stored in disk as:
* +----------------------+
* | 'nesshdr*' |
* +----------------------+
* | version |
* +----------------------+
* | last-nid |
* +----------------------+
* | roo-nid |
* +----------------------+
* | blocksize |
* +----------------------+
* | blockoff |
* +----------------------+
*/
int write_hdr_to_disk(int fd,
struct hdr *hdr,
DISKOFF off)
{
int r;
uint32_t real_size;
uint32_t align_size;
struct buffer *wbuf = NULL;
nassert(hdr->root_nid >= NID_START);
wbuf = buf_new(1 << 20);
buf_putnstr(wbuf, "nesshdr*", 8);
buf_putuint32(wbuf, LAYOUT_VERSION);
buf_putuint64(wbuf, hdr->last_nid);
buf_putuint64(wbuf, hdr->root_nid);
buf_putuint32(wbuf, hdr->blocksize);
buf_putuint64(wbuf, hdr->blockoff);
uint32_t xsum;
if (!buf_xsum(wbuf->buf, wbuf->NUL, &xsum)) {
r = NESS_DO_XSUM_ERR;
goto ERR;
}
buf_putuint32(wbuf, xsum);
real_size = wbuf->NUL;
align_size = ALIGN(real_size);
buf_putnull(wbuf, align_size - real_size);
if (ness_os_pwrite(fd,
wbuf->buf,
align_size,
off) != 0) {
r = NESS_WRITE_ERR;
goto ERR;
}
buf_free(wbuf);
return NESS_OK;
ERR:
buf_free(wbuf);
return r;
}
int _deserialize_blockpairs_from_disk(int fd,
struct block *b,
struct hdr *hdr)
{
int r = NESS_ERR;
uint32_t read_size;
uint32_t align_size;
struct buffer *rbuf;
struct block_pair *pairs;
read_size = hdr->blocksize;
align_size = ALIGN(read_size);
rbuf = buf_new(align_size);
if (ness_os_pread(fd,
rbuf->buf,
align_size,
hdr->blockoff) != (ssize_t)align_size) {
r = NESS_READ_ERR;
goto ERR;
}
if (!buf_seek(rbuf, read_size - CRC_SIZE))
goto ERR;
uint32_t exp_xsum, act_xsum;
if (!buf_getuint32(rbuf, &exp_xsum)) goto ERR;
if (!buf_xsum(rbuf->buf, hdr->blocksize - CRC_SIZE, &act_xsum)) goto ERR;
if (exp_xsum != act_xsum) {
__ERROR("blockpairs xsum check error,"
"exp_xsum: [%" PRIu32 "],"
"act_xsum: [%" PRIu32 "]",
exp_xsum,
act_xsum);
goto ERR;
}
buf_seekfirst(rbuf);
/*
* skip magic with 8bytes
*/
if (!buf_skip(rbuf, 8)) goto ERR;
uint32_t i;
uint32_t block_count = 0U;
if (!buf_getuint32(rbuf, &block_count)) goto ERR;
pairs = xcalloc(block_count, sizeof(*pairs));
for (i = 0; i < block_count; i++) {
if (!buf_getuint64(rbuf, &pairs[i].nid)) goto ERR1;
if (!buf_getuint64(rbuf, &pairs[i].offset)) goto ERR1;
if (!buf_getuint32(rbuf, &pairs[i].real_size)) goto ERR1;
if (!buf_getuint32(rbuf, &pairs[i].skeleton_size)) goto ERR1;
if (!buf_getuint32(rbuf, &pairs[i].height)) goto ERR1;
pairs[i].used = 1;
}
if (block_count > 0)
block_init(b, pairs, block_count);
xfree(pairs);
buf_free(rbuf);
return NESS_OK;
ERR:
buf_free(rbuf);
return r;
ERR1:
buf_free(rbuf);
xfree(pairs);
return r;
}
/*
*
* blockpairs store in disk as:
* +------------------------------------------------------+
* | 'blkpairs' |
* +----------+------+-----------+---------------+--------+
* | nid | off | real-size | skeleton-size | height |
* +----------+------+-----------+---------------+--------+
* ...
*/
int _serialize_blockpairs_to_disk(int fd,
struct block *b,
struct hdr *hdr)
{
uint32_t i;
int r = NESS_OK;
uint32_t used_count = 0;
DISKOFF address;
struct buffer *wbuf;
wbuf = buf_new(1 << 20);
for (i = 0; i < b->pairs_used; i++) {
if (!b->pairs[i].used) continue;
used_count++;
}
__DEBUG(" block pairs count [%d], used count [%d]",
b->pairs_used,
used_count);
buf_putnstr(wbuf, "blkpairs", 8);
buf_putuint32(wbuf, used_count);
for (i = 0; i < b->pairs_used; i++) {
if (!b->pairs[i].used) continue;
buf_putuint64(wbuf, b->pairs[i].nid);
buf_putuint64(wbuf, b->pairs[i].offset);
buf_putuint32(wbuf, b->pairs[i].real_size);
buf_putuint32(wbuf, b->pairs[i].skeleton_size);
buf_putuint32(wbuf, b->pairs[i].height);
}
uint32_t xsum;
if (!buf_xsum(wbuf->buf, wbuf->NUL, &xsum)) {
r = NESS_DO_XSUM_ERR;
goto ERR;
}
buf_putuint32(wbuf, xsum);
uint32_t real_size;
uint32_t align_size;
real_size = wbuf->NUL;
hdr->blocksize = real_size;
align_size = ALIGN(real_size);
buf_putnull(wbuf, align_size - real_size);
address = block_alloc_off(b, 0, real_size, 0, 0);
if (ness_os_pwrite(fd,
wbuf->buf,
align_size,
address) != 0) {
r = NESS_WRITE_ERR;
goto ERR;
}
hdr->blockoff = address;
buf_free(wbuf);
return NESS_OK;
ERR:
buf_free(wbuf);
return r;
}
int read_hdr_from_disk(int fd,
struct block *b,
struct hdr **h,
DISKOFF off)
{
int r = NESS_ERR;
struct hdr *hdr = NULL;
struct buffer *rbuf = NULL;
uint32_t exp_xsum, act_xsum;
uint32_t read_size, align_size;
hdr = xcalloc(1, sizeof(*hdr));
read_size = (
+ 8 /* magic */
+ 8 /* last nid */
+ 8 /* root nid */
+ 4 /* version */
+ 4 /* block size */
+ 8 /* block offset */
+ CRC_SIZE); /* checksum */
align_size = ALIGN(read_size);
rbuf = buf_new(align_size);
if (ness_os_pread(fd, rbuf->buf, align_size, off) !=
(ssize_t)align_size) {
__ERROR("ness pread error, read size [%" PRIu32 "], "
"offset [%" PRIu64 "]",
align_size,
0UL);
r = NESS_READ_ERR;
goto ERR;
}
if (!buf_seek(rbuf, read_size - CRC_SIZE)) goto ERR;
if (!buf_getuint32(rbuf, &exp_xsum)) goto ERR;
if (!buf_xsum(rbuf->buf, read_size - CRC_SIZE, &act_xsum)) goto ERR;
if (exp_xsum != act_xsum) {
__ERROR("header xsum check error, "
"exp_xsum: [%" PRIu32 "], "
"act_xsum: [%" PRIu32 "], ",
exp_xsum,
act_xsum);
r = NESS_HDR_XSUM_ERR;
goto ERR;
}
buf_seekfirst(rbuf);
if (!buf_skip(rbuf, 8)) goto ERR;
if (!buf_getuint32(rbuf, &hdr->version)) goto ERR;
if (!buf_getuint64(rbuf, &hdr->last_nid)) goto ERR;
if (!buf_getuint64(rbuf, &hdr->root_nid)) goto ERR;
if (!buf_getuint32(rbuf, &hdr->blocksize)) goto ERR;
if (!buf_getuint64(rbuf, &hdr->blockoff)) goto ERR;
nassert(hdr->root_nid >= NID_START);
if (hdr->version < LAYOUT_MIN_SUPPORTED_VERSION) {
r = NESS_LAYOUT_VERSION_OLDER_ERR;
__ERROR("tree layout too older [%d], "
"min_support_version [%d]",
hdr->version,
LAYOUT_MIN_SUPPORTED_VERSION);
goto ERR;
}
/* block pairs */
r = _deserialize_blockpairs_from_disk(fd, b, hdr);
if (r != NESS_OK) {
r = NESS_DESERIAL_BLOCKPAIR_ERR;
goto ERR;
}
*h = hdr;
buf_free(rbuf);
return NESS_OK;
ERR:
buf_free(rbuf);
xfree(hdr);
return r;
}
int logr_read(struct logr *lgr,
struct msg *k,
struct msg *v,
msgtype_t *t,
uint32_t *tbn)
{
int r;
uint32_t size = 0U;
if (lgr->read < lgr->fsize) {
char *base;
uint32_t read;
if (ness_os_read(lgr->fd, lgr->base, 4) != 4) {
r = NESS_LOG_READ_SIZE_ERR;
__ERROR(" log read size error, errno [%d]", r);
goto ERR;
}
getuint32(lgr->base, &size);
if (size > lgr->base_size) {
lgr->base_size = size;
lgr->base = xrealloc(lgr->base, lgr->base_size);
}
/* skip the length */
read = (size - 4);
if (ness_os_read(lgr->fd, lgr->base, read) != read) {
r = NESS_LOG_READ_DATA_ERR;
__ERROR(" log read entry error, errno [%d]", r);
goto ERR;
}
base = lgr->base;
uint32_t exp_xsum;
uint32_t act_xsum;
getuint32(base + read - 4, &exp_xsum);
buf_xsum(base, read - 4, &act_xsum);
if (exp_xsum != act_xsum) {
r = NESS_LOG_READ_XSUM_ERR;
__ERROR("log read xsum error, exp_xsum [%" PRIu32 "],act_xsum [%" PRIu32 "]",
exp_xsum,
act_xsum);
goto ERR;
}
int pos = 0;
/* tbn */
getuint32(base + pos, tbn);
pos += 4;
/* key */
uint32_t fixsize;
getuint32(base + pos, &fixsize);
pos += 4;
k->size = (fixsize >> 8);
*t = (msgtype_t)(fixsize & 0xff);
getnstr(base + pos, k->size, (char**)&k->data);
pos += k->size;
if (*t != MSG_DELETE) {
/* value */
getuint32(base + pos, &v->size);
pos += 4;
getnstr(base + pos, v->size, (char**)&v->data);
pos += v->size;
}
} else {
