DataRecord* decode_record(char* buf, uint32_t size, bool decomp) { DataRecord *r = (DataRecord *) (buf - sizeof(char*)); int ksz = r->ksz, vsz = r->vsz; if (ksz < 0 || ksz > 200 || vsz < 0 || vsz > 100 * 1024 * 1024){ //fprintf(stderr, "invalid ksz=: %d, vsz=%d\n", ksz, vsz); return NULL; } int need = sizeof(DataRecord) - sizeof(char*) + ksz + vsz; if (size < need) { fprintf(stderr, "not enough data in buffer: %d < %d\n", size, need); return NULL; } uint32_t crc = crc32(0, buf + sizeof(uint32_t), need - sizeof(uint32_t)); if (r->crc != crc) { fprintf(stderr, "CRC checksum failed\n"); return NULL; } DataRecord *r2 = (DataRecord *) malloc(need + 1 + sizeof(char*)); memcpy(&r2->crc, &r->crc, sizeof(DataRecord) - sizeof(char*) + ksz); r2->key[ksz] = 0; // c str r2->free_value = false; r2->value = r2->key + ksz + 1; memcpy(r2->value, r->key + ksz, vsz); if (decomp) { r2 = decompress_record(r2); } return r2; }
/* * Read all the records from one persistent store backend. Create * files in our filesystem. Don't warn about -EEXIST errors * when we are re-scanning the backing store looking to add new * error records. */ void pstore_get_backend_records(struct pstore_info *psi, struct dentry *root, int quiet) { int failed = 0; unsigned int stop_loop = 65536; if (!psi || !root) return; mutex_lock(&psi->read_mutex); if (psi->open && psi->open(psi)) goto out; /* * Backend callback read() allocates record.buf. decompress_record() * may reallocate record.buf. On success, pstore_mkfile() will keep * the record.buf, so free it only on failure. */ for (; stop_loop; stop_loop--) { struct pstore_record *record; int rc; record = kzalloc(sizeof(*record), GFP_KERNEL); if (!record) { pr_err("out of memory creating record\n"); break; } pstore_record_init(record, psi); record->size = psi->read(record); /* No more records left in backend? */ if (record->size <= 0) { kfree(record); break; } decompress_record(record); rc = pstore_mkfile(root, record); if (rc) { /* pstore_mkfile() did not take record, so free it. */ kfree(record->buf); kfree(record); if (rc != -EEXIST || !quiet) failed++; } } if (psi->close) psi->close(psi); out: mutex_unlock(&psi->read_mutex); if (failed) pr_warn("failed to create %d record(s) from '%s'\n", failed, psi->name); if (!stop_loop) pr_err("looping? Too many records seen from '%s'\n", psi->name); }
DataRecord* fast_read_record(int fd, off_t offset, bool decomp) { DataRecord *r = (DataRecord*) malloc(PADDING + sizeof(char*)); r->value = NULL; if (pread(fd, &r->crc, PADDING, offset) != PADDING) { fprintf(stderr, "read record faied\n"); goto READ_END; } int ksz = r->ksz, vsz = r->vsz; if (ksz < 0 || ksz > 200 || vsz < 0 || vsz > 100 * 1024 * 1024){ fprintf(stderr, "invalid ksz=: %d, vsz=%d\n", ksz, vsz); goto READ_END; } uint32_t crc_old = r->crc; int read_size = PADDING - (sizeof(DataRecord) - sizeof(char*)) - ksz; if (vsz < read_size) { r->value = r->key + ksz + 1; r->free_value = false; memmove(r->value, r->key + ksz, vsz); }else{ r->value = malloc(vsz); r->free_value = true; memcpy(r->value, r->key + ksz, read_size); int need = vsz - read_size; int ret = 0; if (need > 0 && need != (ret=pread(fd, r->value + read_size, need, offset+PADDING))) { r->key[ksz] = 0; // c str fprintf(stderr, "read record %s faied: %d < %d @%ld\n", r->key, ret, need, offset); goto READ_END; } } r->key[ksz] = 0; // c str uint32_t crc = crc32(0, (char*)(&r->tstamp), sizeof(DataRecord) - sizeof(char*) - sizeof(uint32_t) + ksz); crc = crc32(crc, r->value, vsz); if (crc != crc_old){ fprintf(stderr, "%s @%ld crc32 check failed %d != %d\n", r->key, offset, crc, r->crc); goto READ_END; } if (decomp) { r = decompress_record(r); } return r; READ_END: free_record(r); return NULL; }
void scanDataFileBefore(HTree* tree, int bucket, const char* path, time_t before) { MFile *f = open_mfile(path); if (f == NULL) return; fprintf(stderr, "scan datafile %s before %ld\n", path, before); char *p = f->addr, *end = f->addr + f->size; int broken = 0; while (p < end) { DataRecord *r = decode_record(p, end-p, false); if (r != NULL) { if (r->tstamp >= before ){ break; } uint32_t pos = p - f->addr; p += record_length(r); r = decompress_record(r); uint16_t hash = gen_hash(r->value, r->vsz); if (r->version > 0){ uint16_t hash = gen_hash(r->value, r->vsz); ht_add2(tree, r->key, r->ksz, pos | bucket, hash, r->version); }else{ ht_remove2(tree, r->key, r->ksz); } free_record(r); } else { broken ++; if (broken > 40960) { // 10M fprintf(stderr, "unexpected broken data in %s at %ld\n", path, p - f->addr - broken * PADDING); break; } p += PADDING; } } close_mfile(f); }