struct containerMeta* retrieve_container_meta_by_id(containerid id) {
struct containerMeta* cm = NULL;
/* First, we find it in the buffer */
cm = sync_queue_find(container_buffer, container_check_id, &id,
container_meta_duplicate);
if (cm)
return cm;
cm = (struct containerMeta*) malloc(sizeof(struct containerMeta));
init_container_meta(cm);
unsigned char buf[CONTAINER_META_SIZE];
pthread_mutex_lock(&mutex);
if (destor.simulation_level >= SIMULATION_APPEND)
fseek(fp, id * CONTAINER_META_SIZE + 8, SEEK_SET);
else
fseek(fp, (id + 1) * CONTAINER_SIZE - CONTAINER_META_SIZE + 8,
SEEK_SET);
fread(buf, CONTAINER_META_SIZE, 1, fp);
pthread_mutex_unlock(&mutex);
unser_declare;
unser_begin(buf, CONTAINER_META_SIZE);
unser_int64(cm->id);
unser_int32(cm->chunk_num);
unser_int32(cm->data_size);
if(cm->id != id){
WARNING("expect %lld, but read %lld", id, cm->id);
assert(cm->id == id);
}
int i;
for (i = 0; i < cm->chunk_num; i++) {
struct metaEntry* me = (struct metaEntry*) malloc(
sizeof(struct metaEntry));
unser_bytes(&me->fp, sizeof(fingerprint));
unser_bytes(&me->len, sizeof(int32_t));
unser_bytes(&me->off, sizeof(int32_t));
g_hash_table_insert(cm->map, &me->fp, me);
}
return cm;
}
/*
* Note, we receive the whole attribute record, but we select out only the stat
* packet, VolSessionId, VolSessionTime, FileIndex, file type, and file name to
* store in the catalog.
*/
static void update_attribute(JCR *jcr, char *msg, int32_t msglen)
{
unser_declare;
uint32_t VolSessionId, VolSessionTime;
int32_t Stream;
uint32_t FileIndex;
char *p;
int len;
char *fname, *attr;
ATTR_DBR *ar = NULL;
uint32_t reclen;
/*
* Start transaction allocates jcr->attr and jcr->ar if needed
*/
db_start_transaction(jcr, jcr->db); /* start transaction if not already open */
ar = jcr->ar;
/*
* Start by scanning directly in the message buffer to get Stream
* there may be a cached attr so we cannot yet write into
* jcr->attr or jcr->ar
*/
p = msg;
skip_nonspaces(&p); /* UpdCat */
skip_spaces(&p);
skip_nonspaces(&p); /* Job=nnn */
skip_spaces(&p);
skip_nonspaces(&p); /* "FileAttributes" */
p += 1;
/*
* The following "SD header" fields are serialized
*/
unser_begin(p, 0);
unser_uint32(VolSessionId); /* VolSessionId */
unser_uint32(VolSessionTime); /* VolSessionTime */
unser_int32(FileIndex); /* FileIndex */
unser_int32(Stream); /* Stream */
unser_uint32(reclen); /* Record length */
p += unser_length(p); /* Raw record follows */
/**
* At this point p points to the raw record, which varies according
* to what kind of a record (Stream) was sent. Note, the integer
* fields at the beginning of these "raw" records are in ASCII with
* spaces between them so one can use scanf or manual scanning to
* extract the fields.
*
* File Attributes
* File_index
* File type
* Filename (full path)
* Encoded attributes
* Link name (if type==FT_LNK or FT_LNKSAVED)
* Encoded extended-attributes (for Win32)
* Delta sequence number (32 bit int)
*
* Restore Object
* File_index
* File_type
* Object_index
* Object_len (possibly compressed)
* Object_full_len (not compressed)
* Object_compression
* Plugin_name
* Object_name
* Binary Object data
*/
Dmsg1(400, "UpdCat msg=%s\n", msg);
Dmsg5(400, "UpdCat VolSessId=%d VolSessT=%d FI=%d Strm=%d reclen=%d\n",
VolSessionId, VolSessionTime, FileIndex, Stream, reclen);
jcr->SDJobBytes += reclen; /* update number of bytes transferred for quotas */
/*
* Depending on the stream we are handling dispatch.
*/
switch (Stream) {
case STREAM_UNIX_ATTRIBUTES:
case STREAM_UNIX_ATTRIBUTES_EX:
if (jcr->cached_attribute) {
Dmsg2(400, "Cached attr. Stream=%d fname=%s\n", ar->Stream, ar->fname);
if (!db_create_attributes_record(jcr, jcr->db, ar)) {
Jmsg1(jcr, M_FATAL, 0, _("Attribute create error: ERR=%s"), db_strerror(jcr->db));
}
jcr->cached_attribute = false;
}
/*
* Any cached attr is flushed so we can reuse jcr->attr and jcr->ar
*/
jcr->attr = check_pool_memory_size(jcr->attr, msglen);
memcpy(jcr->attr, msg, msglen);
p = jcr->attr - msg + p; /* point p into jcr->attr */
skip_nonspaces(&p); /* skip FileIndex */
skip_spaces(&p);
ar->FileType = str_to_int32(p);
skip_nonspaces(&p); /* skip FileType */
skip_spaces(&p);
fname = p;
len = strlen(fname); /* length before attributes */
attr = &fname[len+1];
ar->DeltaSeq = 0;
if (ar->FileType == FT_REG) {
p = attr + strlen(attr) + 1; /* point to link */
p = p + strlen(p) + 1; /* point to extended attributes */
p = p + strlen(p) + 1; /* point to delta sequence */
/*
* Older FDs don't have a delta sequence, so check if it is there
*/
if (p - jcr->attr < msglen) {
ar->DeltaSeq = str_to_int32(p); /* delta_seq */
}
}
Dmsg2(400, "dird<stored: stream=%d %s\n", Stream, fname);
Dmsg1(400, "dird<stored: attr=%s\n", attr);
ar->attr = attr;
ar->fname = fname;
if (ar->FileType == FT_DELETED) {
ar->FileIndex = 0; /* special value */
} else {
ar->FileIndex = FileIndex;
}
ar->Stream = Stream;
ar->link = NULL;
if (jcr->mig_jcr) {
ar->JobId = jcr->mig_jcr->JobId;
} else {
ar->JobId = jcr->JobId;
}
ar->Digest = NULL;
ar->DigestType = CRYPTO_DIGEST_NONE;
jcr->cached_attribute = true;
Dmsg2(400, "dird<filed: stream=%d %s\n", Stream, fname);
Dmsg1(400, "dird<filed: attr=%s\n", attr);
break;
case STREAM_RESTORE_OBJECT: {
ROBJECT_DBR ro;
memset(&ro, 0, sizeof(ro));
ro.Stream = Stream;
ro.FileIndex = FileIndex;
if (jcr->mig_jcr) {
ro.JobId = jcr->mig_jcr->JobId;
} else {
ro.JobId = jcr->JobId;
}
Dmsg1(100, "Robj=%s\n", p);
skip_nonspaces(&p); /* skip FileIndex */
skip_spaces(&p);
ro.FileType = str_to_int32(p); /* FileType */
skip_nonspaces(&p);
skip_spaces(&p);
ro.object_index = str_to_int32(p); /* Object Index */
skip_nonspaces(&p);
skip_spaces(&p);
ro.object_len = str_to_int32(p); /* object length possibly compressed */
skip_nonspaces(&p);
skip_spaces(&p);
ro.object_full_len = str_to_int32(p); /* uncompressed object length */
skip_nonspaces(&p);
skip_spaces(&p);
ro.object_compression = str_to_int32(p); /* compression */
skip_nonspaces(&p);
skip_spaces(&p);
ro.plugin_name = p; /* point to plugin name */
len = strlen(ro.plugin_name);
ro.object_name = &ro.plugin_name[len+1]; /* point to object name */
len = strlen(ro.object_name);
ro.object = &ro.object_name[len+1]; /* point to object */
ro.object[ro.object_len] = 0; /* add zero for those who attempt printing */
Dmsg7(100, "oname=%s stream=%d FT=%d FI=%d JobId=%d, obj_len=%d\nobj=\"%s\"\n",
ro.object_name, ro.Stream, ro.FileType, ro.FileIndex, ro.JobId,
ro.object_len, ro.object);
/*
* Store it.
*/
if (!db_create_restore_object_record(jcr, jcr->db, &ro)) {
Jmsg1(jcr, M_FATAL, 0, _("Restore object create error. %s"), db_strerror(jcr->db));
}
break;
}
default:
if (crypto_digest_stream_type(Stream) != CRYPTO_DIGEST_NONE) {
fname = p;
if (ar->FileIndex != FileIndex) {
Jmsg3(jcr, M_WARNING, 0, _("%s not same File=%d as attributes=%d\n"),
stream_to_ascii(Stream), FileIndex, ar->FileIndex);
} else {
/*
* Update digest in catalog
*/
char digestbuf[BASE64_SIZE(CRYPTO_DIGEST_MAX_SIZE)];
int len = 0;
int type = CRYPTO_DIGEST_NONE;
switch(Stream) {
case STREAM_MD5_DIGEST:
len = CRYPTO_DIGEST_MD5_SIZE;
type = CRYPTO_DIGEST_MD5;
break;
case STREAM_SHA1_DIGEST:
len = CRYPTO_DIGEST_SHA1_SIZE;
type = CRYPTO_DIGEST_SHA1;
break;
case STREAM_SHA256_DIGEST:
len = CRYPTO_DIGEST_SHA256_SIZE;
type = CRYPTO_DIGEST_SHA256;
break;
case STREAM_SHA512_DIGEST:
len = CRYPTO_DIGEST_SHA512_SIZE;
type = CRYPTO_DIGEST_SHA512;
break;
default:
/*
* Never reached ...
*/
Jmsg(jcr, M_ERROR, 0, _("Catalog error updating file digest. Unsupported digest stream type: %d"), Stream);
}
bin_to_base64(digestbuf, sizeof(digestbuf), fname, len, true);
Dmsg3(400, "DigestLen=%d Digest=%s type=%d\n", strlen(digestbuf),
digestbuf, Stream);
if (jcr->cached_attribute) {
ar->Digest = digestbuf;
ar->DigestType = type;
Dmsg2(400, "Cached attr with digest. Stream=%d fname=%s\n", ar->Stream, ar->fname);
/*
* Update BaseFile table
*/
if (!db_create_attributes_record(jcr, jcr->db, ar)) {
Jmsg1(jcr, M_FATAL, 0, _("attribute create error. %s"), db_strerror(jcr->db));
}
jcr->cached_attribute = false;
} else {
if (!db_add_digest_to_file_record(jcr, jcr->db, ar->FileId, digestbuf, type)) {
Jmsg(jcr, M_ERROR, 0, _("Catalog error updating file digest. %s"), db_strerror(jcr->db));
}
}
}
}
break;
}
}
/*
* Read the header record
*/
static bool read_header(DCR *dcr, DEV_BLOCK *block, DEV_RECORD *rec)
{
ser_declare;
uint32_t VolSessionId;
uint32_t VolSessionTime;
int32_t FileIndex;
int32_t Stream;
uint32_t rhl;
char buf1[100], buf2[100];
Dmsg0(dbgep, "=== rpath 1 read_header\n");
/* Clear state flags */
rec->state_bits = 0;
if (block->dev->is_tape()) {
rec->state_bits |= REC_ISTAPE;
}
rec->Block = ((DEVICE *)block->dev)->EndBlock;
rec->File = ((DEVICE *)block->dev)->EndFile;
/*
* Get the header. There is always a full header,
* otherwise we find it in the next block.
*/
Dmsg3(read_dbglvl, "Block=%d Ver=%d block_len=%u\n",
block->BlockNumber, block->BlockVer, block->block_len);
if (block->BlockVer == 1) {
rhl = RECHDR1_LENGTH;
} else {
rhl = RECHDR2_LENGTH;
}
if (rec->remlen >= rhl) {
Dmsg0(dbgep, "=== rpath 2 begin unserial header\n");
Dmsg4(read_dbglvl, "read_header: remlen=%d data_len=%d rem=%d blkver=%d\n",
rec->remlen, rec->data_len, rec->remainder, block->BlockVer);
unser_begin(block->bufp, WRITE_RECHDR_LENGTH);
if (block->BlockVer == 1) {
unser_uint32(VolSessionId);
unser_uint32(VolSessionTime);
} else {
VolSessionId = block->VolSessionId;
VolSessionTime = block->VolSessionTime;
}
unser_int32(FileIndex);
unser_int32(Stream);
unser_uint32(rec->data_bytes);
block->bufp += rhl;
block->binbuf -= rhl;
rec->remlen -= rhl;
/* If we are looking for more (remainder!=0), we reject anything
* where the VolSessionId and VolSessionTime don't agree
*/
if (rec->remainder && (rec->VolSessionId != VolSessionId ||
rec->VolSessionTime != VolSessionTime)) {
rec->state_bits |= REC_NO_MATCH;
Dmsg0(read_dbglvl, "remainder and VolSession doesn't match\n");
Dmsg0(dbgep, "=== rpath 4 VolSession no match\n");
return false; /* This is from some other Session */
}
/* if Stream is negative, it means that this is a continuation
* of a previous partially written record.
*/
if (Stream < 0) { /* continuation record? */
Dmsg0(dbgep, "=== rpath 5 negative stream\n");
Dmsg1(read_dbglvl, "Got negative Stream => continuation. remainder=%d\n",
rec->remainder);
rec->state_bits |= REC_CONTINUATION;
if (!rec->remainder) { /* if we didn't read previously */
Dmsg0(dbgep, "=== rpath 6 no remainder\n");
rec->data_len = 0; /* return data as if no continuation */
} else if (rec->Stream != -Stream) {
Dmsg0(dbgep, "=== rpath 7 wrong cont stream\n");
rec->state_bits |= REC_NO_MATCH;
return false; /* This is from some other Session */
}
rec->Stream = -Stream; /* set correct Stream */
rec->maskedStream = rec->Stream & STREAMMASK_TYPE;
} else { /* Regular record */
Dmsg0(dbgep, "=== rpath 8 normal stream\n");
rec->Stream = Stream;
rec->maskedStream = rec->Stream & STREAMMASK_TYPE;
rec->data_len = 0; /* transfer to beginning of data */
}
rec->VolSessionId = VolSessionId;
rec->VolSessionTime = VolSessionTime;
rec->FileIndex = FileIndex;
if (FileIndex > 0) {
Dmsg0(dbgep, "=== rpath 9 FileIndex>0\n");
if (block->FirstIndex == 0) {
Dmsg0(dbgep, "=== rpath 10 FirstIndex\n");
block->FirstIndex = FileIndex;
}
block->LastIndex = rec->FileIndex;
}
Dmsg6(read_dbglvl, "read_header: FI=%s SessId=%d Strm=%s len=%u rec->remlen=%d data_len=%d\n",
FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId,
stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_bytes, rec->remlen,
rec->data_len);
} else {
Dmsg0(dbgep, "=== rpath 11a block out of records\n");
/*
* No more records in this block because the number
* of remaining bytes are less than a record header
* length, so return empty handed, but indicate that
* he must read again. By returning, we allow the
* higher level routine to fetch the next block and
* then reread.
*/
Dmsg0(read_dbglvl, "read_header: End of block\n");
rec->state_bits |= (REC_NO_HEADER | REC_BLOCK_EMPTY);
empty_block(block); /* mark block empty */
return false;
}
/* Sanity check */
if (rec->data_bytes >= MAX_BLOCK_LENGTH) {
Dmsg0(dbgep, "=== rpath 11b maxlen too big\n");
/*
* Something is wrong, force read of next block, abort
* continuing with this block.
*/
rec->state_bits |= (REC_NO_HEADER | REC_BLOCK_EMPTY);
empty_block(block);
Jmsg2(dcr->jcr, M_WARNING, 0, _("Sanity check failed. maxlen=%d datalen=%d. Block discarded.\n"),
MAX_BLOCK_LENGTH, rec->data_bytes);
return false;
}
rec->data = check_pool_memory_size(rec->data, rec->data_len+rec->data_bytes);
rec->rstate = st_data;
return true;
}
/*
* Read a Record from the block
*
* Returns: false if nothing read or if the continuation record does not match.
* In both of these cases, a block read must be done.
* true if at least the record header was read, this
* routine may have to be called again with a new
* block if the entire record was not read.
*/
bool read_record_from_block(DCR *dcr, DEV_RECORD *rec)
{
ser_declare;
uint32_t remlen;
uint32_t VolSessionId;
uint32_t VolSessionTime;
int32_t FileIndex;
int32_t Stream;
uint32_t data_bytes;
uint32_t rhl;
char buf1[100], buf2[100];
remlen = dcr->block->binbuf;
/*
* Clear state flags
*/
clear_all_bits(REC_STATE_MAX, rec->state_bits);
if (dcr->block->dev->is_tape()) {
set_bit(REC_ISTAPE, rec->state_bits);
}
rec->Block = ((DEVICE *)(dcr->block->dev))->EndBlock;
rec->File = ((DEVICE *)(dcr->block->dev))->EndFile;
/*
* Get the header. There is always a full header, otherwise we find it in the next block.
*/
Dmsg3(450, "Block=%d Ver=%d size=%u\n", dcr->block->BlockNumber, dcr->block->BlockVer,
dcr->block->block_len);
if (dcr->block->BlockVer == 1) {
rhl = RECHDR1_LENGTH;
} else {
rhl = RECHDR2_LENGTH;
}
if (remlen >= rhl) {
Dmsg4(450, "Enter read_record_block: remlen=%d data_len=%d rem=%d blkver=%d\n",
remlen, rec->data_len, rec->remainder, dcr->block->BlockVer);
unser_begin(dcr->block->bufp, WRITE_RECHDR_LENGTH);
if (dcr->block->BlockVer == 1) {
unser_uint32(VolSessionId);
unser_uint32(VolSessionTime);
} else {
VolSessionId = dcr->block->VolSessionId;
VolSessionTime = dcr->block->VolSessionTime;
}
unser_int32(FileIndex);
unser_int32(Stream);
unser_uint32(data_bytes);
dcr->block->bufp += rhl;
dcr->block->binbuf -= rhl;
remlen -= rhl;
/*
* If we are looking for more (remainder!=0), we reject anything
* where the VolSessionId and VolSessionTime don't agree
*/
if (rec->remainder && (rec->VolSessionId != VolSessionId ||
rec->VolSessionTime != VolSessionTime)) {
set_bit(REC_NO_MATCH, rec->state_bits);
Dmsg0(450, "remainder and VolSession doesn't match\n");
return false; /* This is from some other Session */
}
/*
* If Stream is negative, it means that this is a continuation
* of a previous partially written record.
*/
if (Stream < 0) { /* continuation record? */
Dmsg1(500, "Got negative Stream => continuation. remainder=%d\n", rec->remainder);
set_bit(REC_CONTINUATION, rec->state_bits);
if (!rec->remainder) { /* if we didn't read previously */
rec->data_len = 0; /* return data as if no continuation */
} else if (rec->Stream != -Stream) {
set_bit(REC_NO_MATCH, rec->state_bits);
return false; /* This is from some other Session */
}
rec->Stream = -Stream; /* set correct Stream */
rec->maskedStream = rec->Stream & STREAMMASK_TYPE;
} else { /* Regular record */
rec->Stream = Stream;
rec->maskedStream = rec->Stream & STREAMMASK_TYPE;
rec->data_len = 0; /* transfer to beginning of data */
}
rec->VolSessionId = VolSessionId;
rec->VolSessionTime = VolSessionTime;
rec->FileIndex = FileIndex;
if (FileIndex > 0) {
if (dcr->block->FirstIndex == 0) {
dcr->block->FirstIndex = FileIndex;
}
dcr->block->LastIndex = FileIndex;
}
Dmsg6(450, "rd_rec_blk() got FI=%s SessId=%d Strm=%s len=%u\n"
"remlen=%d data_len=%d\n",
FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId,
stream_to_ascii(buf2, rec->Stream, rec->FileIndex), data_bytes, remlen,
rec->data_len);
} else {
/*
* No more records in this block because the number
* of remaining bytes are less than a record header
* length, so return empty handed, but indicate that
* he must read again. By returning, we allow the
* higher level routine to fetch the next block and
* then reread.
*/
Dmsg0(450, "read_record_block: nothing\n");
set_bit(REC_NO_HEADER, rec->state_bits);
set_bit(REC_BLOCK_EMPTY, rec->state_bits);
empty_block(dcr->block); /* mark block empty */
return false;
}
/* Sanity check */
if (data_bytes >= MAX_BLOCK_LENGTH) {
/*
* Something is wrong, force read of next block, abort
* continuing with this block.
*/
set_bit(REC_NO_HEADER, rec->state_bits);
set_bit(REC_BLOCK_EMPTY, rec->state_bits);
empty_block(dcr->block);
Jmsg2(dcr->jcr, M_WARNING, 0, _("Sanity check failed. maxlen=%d datalen=%d. Block discarded.\n"),
MAX_BLOCK_LENGTH, data_bytes);
return false;
}
rec->data = check_pool_memory_size(rec->data, rec->data_len + data_bytes);
/*
* At this point, we have read the header, now we
* must transfer as much of the data record as
* possible taking into account: 1. A partial
* data record may have previously been transferred,
* 2. The current block may not contain the whole data
* record.
*/
if (remlen >= data_bytes) {
/*
* Got whole record
*/
memcpy(rec->data+rec->data_len, dcr->block->bufp, data_bytes);
dcr->block->bufp += data_bytes;
dcr->block->binbuf -= data_bytes;
rec->data_len += data_bytes;
} else {
/*
* Partial record
*/
memcpy(rec->data+rec->data_len, dcr->block->bufp, remlen);
dcr->block->bufp += remlen;
dcr->block->binbuf -= remlen;
rec->data_len += remlen;
rec->remainder = 1; /* partial record transferred */
Dmsg1(450, "read_record_block: partial xfered=%d\n", rec->data_len);
set_bit(REC_PARTIAL_RECORD, rec->state_bits);
set_bit(REC_BLOCK_EMPTY, rec->state_bits);
return true;
}
rec->remainder = 0;
Dmsg4(450, "Rtn full rd_rec_blk FI=%s SessId=%d Strm=%s len=%d\n",
FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId,
stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len);
return true; /* transferred full record */
}
struct container* retrieve_container_by_id(containerid id) {
struct container *c = (struct container*) malloc(sizeof(struct container));
init_container_meta(&c->meta);
unsigned char *cur = 0;
if (destor.simulation_level >= SIMULATION_RESTORE) {
c->data = malloc(CONTAINER_META_SIZE);
pthread_mutex_lock(&mutex);
if (destor.simulation_level >= SIMULATION_APPEND)
fseek(fp, id * CONTAINER_META_SIZE + 8, SEEK_SET);
else
fseek(fp, (id + 1) * CONTAINER_SIZE - CONTAINER_META_SIZE + 8,
SEEK_SET);
fread(c->data, CONTAINER_META_SIZE, 1, fp);
pthread_mutex_unlock(&mutex);
cur = c->data;
} else {
c->data = malloc(CONTAINER_SIZE);
pthread_mutex_lock(&mutex);
fseek(fp, id * CONTAINER_SIZE + 8, SEEK_SET);
fread(c->data, CONTAINER_SIZE, 1, fp);
pthread_mutex_unlock(&mutex);
cur = &c->data[CONTAINER_SIZE - CONTAINER_META_SIZE];
}
unser_declare;
unser_begin(cur, CONTAINER_META_SIZE);
unser_int64(c->meta.id);
unser_int32(c->meta.chunk_num);
unser_int32(c->meta.data_size);
if(c->meta.id != id){
WARNING("expect %lld, but read %lld", id, c->meta.id);
assert(c->meta.id == id);
}
int i;
for (i = 0; i < c->meta.chunk_num; i++) {
struct metaEntry* me = (struct metaEntry*) malloc(
sizeof(struct metaEntry));
unser_bytes(&me->fp, sizeof(fingerprint));
unser_bytes(&me->len, sizeof(int32_t));
unser_bytes(&me->off, sizeof(int32_t));
g_hash_table_insert(c->meta.map, &me->fp, me);
}
unser_end(cur, CONTAINER_META_SIZE);
if (destor.simulation_level >= SIMULATION_RESTORE) {
free(c->data);
c->data = 0;
}
return c;
}
