DataRecord* decode_record(char* buf, uint32_t size, bool decomp)
{
DataRecord *r = (DataRecord *) (buf - sizeof(char*));
int ksz = r->ksz, vsz = r->vsz;
if (ksz < 0 || ksz > 200 || vsz < 0 || vsz > 100 * 1024 * 1024){
//fprintf(stderr, "invalid ksz=: %d, vsz=%d\n", ksz, vsz);
return NULL;
}
int need = sizeof(DataRecord) - sizeof(char*) + ksz + vsz;
if (size < need) {
fprintf(stderr, "not enough data in buffer: %d < %d\n", size, need);
return NULL;
}
uint32_t crc = crc32(0, buf + sizeof(uint32_t), need - sizeof(uint32_t));
if (r->crc != crc) {
fprintf(stderr, "CRC checksum failed\n");
return NULL;
}
DataRecord *r2 = (DataRecord *) malloc(need + 1 + sizeof(char*));
memcpy(&r2->crc, &r->crc, sizeof(DataRecord) - sizeof(char*) + ksz);
r2->key[ksz] = 0; // c str
r2->free_value = false;
r2->value = r2->key + ksz + 1;
memcpy(r2->value, r->key + ksz, vsz);
if (decomp) {
r2 = decompress_record(r2);
}
return r2;
}
/*
* Read all the records from one persistent store backend. Create
* files in our filesystem. Don't warn about -EEXIST errors
* when we are re-scanning the backing store looking to add new
* error records.
*/
void pstore_get_backend_records(struct pstore_info *psi,
struct dentry *root, int quiet)
{
int failed = 0;
unsigned int stop_loop = 65536;
if (!psi || !root)
return;
mutex_lock(&psi->read_mutex);
if (psi->open && psi->open(psi))
goto out;
/*
* Backend callback read() allocates record.buf. decompress_record()
* may reallocate record.buf. On success, pstore_mkfile() will keep
* the record.buf, so free it only on failure.
*/
for (; stop_loop; stop_loop--) {
struct pstore_record *record;
int rc;
record = kzalloc(sizeof(*record), GFP_KERNEL);
if (!record) {
pr_err("out of memory creating record\n");
break;
}
pstore_record_init(record, psi);
record->size = psi->read(record);
/* No more records left in backend? */
if (record->size <= 0) {
kfree(record);
break;
}
decompress_record(record);
rc = pstore_mkfile(root, record);
if (rc) {
/* pstore_mkfile() did not take record, so free it. */
kfree(record->buf);
kfree(record);
if (rc != -EEXIST || !quiet)
failed++;
}
}
if (psi->close)
psi->close(psi);
out:
mutex_unlock(&psi->read_mutex);
if (failed)
pr_warn("failed to create %d record(s) from '%s'\n",
failed, psi->name);
if (!stop_loop)
pr_err("looping? Too many records seen from '%s'\n",
psi->name);
}
DataRecord* fast_read_record(int fd, off_t offset, bool decomp)
{
DataRecord *r = (DataRecord*) malloc(PADDING + sizeof(char*));
r->value = NULL;
if (pread(fd, &r->crc, PADDING, offset) != PADDING) {
fprintf(stderr, "read record faied\n");
goto READ_END;
}
int ksz = r->ksz, vsz = r->vsz;
if (ksz < 0 || ksz > 200 || vsz < 0 || vsz > 100 * 1024 * 1024){
fprintf(stderr, "invalid ksz=: %d, vsz=%d\n", ksz, vsz);
goto READ_END;
}
uint32_t crc_old = r->crc;
int read_size = PADDING - (sizeof(DataRecord) - sizeof(char*)) - ksz;
if (vsz < read_size) {
r->value = r->key + ksz + 1;
r->free_value = false;
memmove(r->value, r->key + ksz, vsz);
}else{
r->value = malloc(vsz);
r->free_value = true;
memcpy(r->value, r->key + ksz, read_size);
int need = vsz - read_size;
int ret = 0;
if (need > 0 && need != (ret=pread(fd, r->value + read_size, need, offset+PADDING))) {
r->key[ksz] = 0; // c str
fprintf(stderr, "read record %s faied: %d < %d @%ld\n", r->key, ret, need, offset);
goto READ_END;
}
}
r->key[ksz] = 0; // c str
uint32_t crc = crc32(0, (char*)(&r->tstamp),
sizeof(DataRecord) - sizeof(char*) - sizeof(uint32_t) + ksz);
crc = crc32(crc, r->value, vsz);
if (crc != crc_old){
fprintf(stderr, "%s @%ld crc32 check failed %d != %d\n", r->key, offset, crc, r->crc);
goto READ_END;
}
if (decomp) {
r = decompress_record(r);
}
return r;
READ_END:
free_record(r);
return NULL;
}
void scanDataFileBefore(HTree* tree, int bucket, const char* path, time_t before)
{
MFile *f = open_mfile(path);
if (f == NULL) return;
fprintf(stderr, "scan datafile %s before %ld\n", path, before);
char *p = f->addr, *end = f->addr + f->size;
int broken = 0;
while (p < end) {
DataRecord *r = decode_record(p, end-p, false);
if (r != NULL) {
if (r->tstamp >= before ){
break;
}
uint32_t pos = p - f->addr;
p += record_length(r);
r = decompress_record(r);
uint16_t hash = gen_hash(r->value, r->vsz);
if (r->version > 0){
uint16_t hash = gen_hash(r->value, r->vsz);
ht_add2(tree, r->key, r->ksz, pos | bucket, hash, r->version);
}else{
ht_remove2(tree, r->key, r->ksz);
}
free_record(r);
} else {
broken ++;
if (broken > 40960) { // 10M
fprintf(stderr, "unexpected broken data in %s at %ld\n", path, p - f->addr - broken * PADDING);
break;
}
p += PADDING;
}
}
close_mfile(f);
}
