struct containerMeta* retrieve_container_meta_by_id(containerid id) { struct containerMeta* cm = NULL; /* First, we find it in the buffer */ cm = sync_queue_find(container_buffer, container_check_id, &id, container_meta_duplicate); if (cm) return cm; cm = (struct containerMeta*) malloc(sizeof(struct containerMeta)); init_container_meta(cm); unsigned char buf[CONTAINER_META_SIZE]; pthread_mutex_lock(&mutex); if (destor.simulation_level >= SIMULATION_APPEND) fseek(fp, id * CONTAINER_META_SIZE + 8, SEEK_SET); else fseek(fp, (id + 1) * CONTAINER_SIZE - CONTAINER_META_SIZE + 8, SEEK_SET); fread(buf, CONTAINER_META_SIZE, 1, fp); pthread_mutex_unlock(&mutex); unser_declare; unser_begin(buf, CONTAINER_META_SIZE); unser_int64(cm->id); unser_int32(cm->chunk_num); unser_int32(cm->data_size); if(cm->id != id){ WARNING("expect %lld, but read %lld", id, cm->id); assert(cm->id == id); } int i; for (i = 0; i < cm->chunk_num; i++) { struct metaEntry* me = (struct metaEntry*) malloc( sizeof(struct metaEntry)); unser_bytes(&me->fp, sizeof(fingerprint)); unser_bytes(&me->len, sizeof(int32_t)); unser_bytes(&me->off, sizeof(int32_t)); g_hash_table_insert(cm->map, &me->fp, me); } return cm; }
static bRC setAcl(bpContext *ctx, acl_pkt *ap) { int status; unser_declare; uint32_t acl_name_length; uint32_t xattr_value_length; POOL_MEM xattr_value(PM_MESSAGE), acl_name(PM_MESSAGE); plugin_ctx *p_ctx = (plugin_ctx *)ctx->pContext; if (!p_ctx) { return bRC_Error; } unser_begin(ap->content, ap->content_length); while (unser_length(ap->content) < ap->content_length) { unser_uint32(acl_name_length); /* * Decode the ACL name including the \0 */ acl_name.check_size(acl_name_length); unser_bytes(acl_name.c_str(), acl_name_length); unser_uint32(xattr_value_length); /* * Decode the actual ACL data as stored as XATTR. */ xattr_value.check_size(xattr_value_length); unser_bytes(xattr_value.c_str(), xattr_value_length); status = ceph_lsetxattr(p_ctx->cmount, ap->fname, acl_name.c_str(), xattr_value.c_str(), xattr_value_length, 0); if (status < 0) { berrno be; Jmsg(ctx, M_ERROR, "ceph_lsetxattr(%s) failed: %s\n", ap->fname, be.bstrerror(-status)); return bRC_Error; } } unser_end(ap->content, ap->content_length); return bRC_OK; }
bool unser_session_label(SESSION_LABEL *label, DEV_RECORD *rec) { ser_declare; rec->data = check_pool_memory_size(rec->data, SER_LENGTH_Session_Label); unser_begin(rec->data, SER_LENGTH_Session_Label); unser_string(label->Id); unser_uint32(label->VerNum); unser_uint32(label->JobId); if (label->VerNum >= 11) { unser_btime(label->write_btime); } else { unser_float64(label->write_date); } unser_float64(label->write_time); unser_string(label->PoolName); unser_string(label->PoolType); unser_string(label->JobName); unser_string(label->ClientName); if (label->VerNum >= 10) { unser_string(label->Job); /* Unique name of this Job */ unser_string(label->FileSetName); unser_uint32(label->JobType); unser_uint32(label->JobLevel); } if (label->VerNum >= 11) { unser_string(label->FileSetMD5); } else { label->FileSetMD5[0] = 0; } if (rec->FileIndex == EOS_LABEL) { unser_uint32(label->JobFiles); unser_uint64(label->JobBytes); unser_uint32(label->StartBlock); unser_uint32(label->EndBlock); unser_uint32(label->StartFile); unser_uint32(label->EndFile); unser_uint32(label->JobErrors); if (label->VerNum >= 11) { unser_uint32(label->JobStatus); } else { label->JobStatus = JS_Terminated; /* kludge */ } } return true; }
/* * Note, we receive the whole attribute record, but we select out only the stat * packet, VolSessionId, VolSessionTime, FileIndex, file type, and file name to * store in the catalog. */ static void update_attribute(JCR *jcr, char *msg, int32_t msglen) { unser_declare; uint32_t VolSessionId, VolSessionTime; int32_t Stream; uint32_t FileIndex; char *p; int len; char *fname, *attr; ATTR_DBR *ar = NULL; uint32_t reclen; /* * Start transaction allocates jcr->attr and jcr->ar if needed */ db_start_transaction(jcr, jcr->db); /* start transaction if not already open */ ar = jcr->ar; /* * Start by scanning directly in the message buffer to get Stream * there may be a cached attr so we cannot yet write into * jcr->attr or jcr->ar */ p = msg; skip_nonspaces(&p); /* UpdCat */ skip_spaces(&p); skip_nonspaces(&p); /* Job=nnn */ skip_spaces(&p); skip_nonspaces(&p); /* "FileAttributes" */ p += 1; /* * The following "SD header" fields are serialized */ unser_begin(p, 0); unser_uint32(VolSessionId); /* VolSessionId */ unser_uint32(VolSessionTime); /* VolSessionTime */ unser_int32(FileIndex); /* FileIndex */ unser_int32(Stream); /* Stream */ unser_uint32(reclen); /* Record length */ p += unser_length(p); /* Raw record follows */ /** * At this point p points to the raw record, which varies according * to what kind of a record (Stream) was sent. Note, the integer * fields at the beginning of these "raw" records are in ASCII with * spaces between them so one can use scanf or manual scanning to * extract the fields. * * File Attributes * File_index * File type * Filename (full path) * Encoded attributes * Link name (if type==FT_LNK or FT_LNKSAVED) * Encoded extended-attributes (for Win32) * Delta sequence number (32 bit int) * * Restore Object * File_index * File_type * Object_index * Object_len (possibly compressed) * Object_full_len (not compressed) * Object_compression * Plugin_name * Object_name * Binary Object data */ Dmsg1(400, "UpdCat msg=%s\n", msg); Dmsg5(400, "UpdCat VolSessId=%d VolSessT=%d FI=%d Strm=%d reclen=%d\n", VolSessionId, VolSessionTime, FileIndex, Stream, reclen); jcr->SDJobBytes += reclen; /* update number of bytes transferred for quotas */ /* * Depending on the stream we are handling dispatch. */ switch (Stream) { case STREAM_UNIX_ATTRIBUTES: case STREAM_UNIX_ATTRIBUTES_EX: if (jcr->cached_attribute) { Dmsg2(400, "Cached attr. Stream=%d fname=%s\n", ar->Stream, ar->fname); if (!db_create_attributes_record(jcr, jcr->db, ar)) { Jmsg1(jcr, M_FATAL, 0, _("Attribute create error: ERR=%s"), db_strerror(jcr->db)); } jcr->cached_attribute = false; } /* * Any cached attr is flushed so we can reuse jcr->attr and jcr->ar */ jcr->attr = check_pool_memory_size(jcr->attr, msglen); memcpy(jcr->attr, msg, msglen); p = jcr->attr - msg + p; /* point p into jcr->attr */ skip_nonspaces(&p); /* skip FileIndex */ skip_spaces(&p); ar->FileType = str_to_int32(p); skip_nonspaces(&p); /* skip FileType */ skip_spaces(&p); fname = p; len = strlen(fname); /* length before attributes */ attr = &fname[len+1]; ar->DeltaSeq = 0; if (ar->FileType == FT_REG) { p = attr + strlen(attr) + 1; /* point to link */ p = p + strlen(p) + 1; /* point to extended attributes */ p = p + strlen(p) + 1; /* point to delta sequence */ /* * Older FDs don't have a delta sequence, so check if it is there */ if (p - jcr->attr < msglen) { ar->DeltaSeq = str_to_int32(p); /* delta_seq */ } } Dmsg2(400, "dird<stored: stream=%d %s\n", Stream, fname); Dmsg1(400, "dird<stored: attr=%s\n", attr); ar->attr = attr; ar->fname = fname; if (ar->FileType == FT_DELETED) { ar->FileIndex = 0; /* special value */ } else { ar->FileIndex = FileIndex; } ar->Stream = Stream; ar->link = NULL; if (jcr->mig_jcr) { ar->JobId = jcr->mig_jcr->JobId; } else { ar->JobId = jcr->JobId; } ar->Digest = NULL; ar->DigestType = CRYPTO_DIGEST_NONE; jcr->cached_attribute = true; Dmsg2(400, "dird<filed: stream=%d %s\n", Stream, fname); Dmsg1(400, "dird<filed: attr=%s\n", attr); break; case STREAM_RESTORE_OBJECT: { ROBJECT_DBR ro; memset(&ro, 0, sizeof(ro)); ro.Stream = Stream; ro.FileIndex = FileIndex; if (jcr->mig_jcr) { ro.JobId = jcr->mig_jcr->JobId; } else { ro.JobId = jcr->JobId; } Dmsg1(100, "Robj=%s\n", p); skip_nonspaces(&p); /* skip FileIndex */ skip_spaces(&p); ro.FileType = str_to_int32(p); /* FileType */ skip_nonspaces(&p); skip_spaces(&p); ro.object_index = str_to_int32(p); /* Object Index */ skip_nonspaces(&p); skip_spaces(&p); ro.object_len = str_to_int32(p); /* object length possibly compressed */ skip_nonspaces(&p); skip_spaces(&p); ro.object_full_len = str_to_int32(p); /* uncompressed object length */ skip_nonspaces(&p); skip_spaces(&p); ro.object_compression = str_to_int32(p); /* compression */ skip_nonspaces(&p); skip_spaces(&p); ro.plugin_name = p; /* point to plugin name */ len = strlen(ro.plugin_name); ro.object_name = &ro.plugin_name[len+1]; /* point to object name */ len = strlen(ro.object_name); ro.object = &ro.object_name[len+1]; /* point to object */ ro.object[ro.object_len] = 0; /* add zero for those who attempt printing */ Dmsg7(100, "oname=%s stream=%d FT=%d FI=%d JobId=%d, obj_len=%d\nobj=\"%s\"\n", ro.object_name, ro.Stream, ro.FileType, ro.FileIndex, ro.JobId, ro.object_len, ro.object); /* * Store it. */ if (!db_create_restore_object_record(jcr, jcr->db, &ro)) { Jmsg1(jcr, M_FATAL, 0, _("Restore object create error. %s"), db_strerror(jcr->db)); } break; } default: if (crypto_digest_stream_type(Stream) != CRYPTO_DIGEST_NONE) { fname = p; if (ar->FileIndex != FileIndex) { Jmsg3(jcr, M_WARNING, 0, _("%s not same File=%d as attributes=%d\n"), stream_to_ascii(Stream), FileIndex, ar->FileIndex); } else { /* * Update digest in catalog */ char digestbuf[BASE64_SIZE(CRYPTO_DIGEST_MAX_SIZE)]; int len = 0; int type = CRYPTO_DIGEST_NONE; switch(Stream) { case STREAM_MD5_DIGEST: len = CRYPTO_DIGEST_MD5_SIZE; type = CRYPTO_DIGEST_MD5; break; case STREAM_SHA1_DIGEST: len = CRYPTO_DIGEST_SHA1_SIZE; type = CRYPTO_DIGEST_SHA1; break; case STREAM_SHA256_DIGEST: len = CRYPTO_DIGEST_SHA256_SIZE; type = CRYPTO_DIGEST_SHA256; break; case STREAM_SHA512_DIGEST: len = CRYPTO_DIGEST_SHA512_SIZE; type = CRYPTO_DIGEST_SHA512; break; default: /* * Never reached ... */ Jmsg(jcr, M_ERROR, 0, _("Catalog error updating file digest. Unsupported digest stream type: %d"), Stream); } bin_to_base64(digestbuf, sizeof(digestbuf), fname, len, true); Dmsg3(400, "DigestLen=%d Digest=%s type=%d\n", strlen(digestbuf), digestbuf, Stream); if (jcr->cached_attribute) { ar->Digest = digestbuf; ar->DigestType = type; Dmsg2(400, "Cached attr with digest. Stream=%d fname=%s\n", ar->Stream, ar->fname); /* * Update BaseFile table */ if (!db_create_attributes_record(jcr, jcr->db, ar)) { Jmsg1(jcr, M_FATAL, 0, _("attribute create error. %s"), db_strerror(jcr->db)); } jcr->cached_attribute = false; } else { if (!db_add_digest_to_file_record(jcr, jcr->db, ar->FileId, digestbuf, type)) { Jmsg(jcr, M_ERROR, 0, _("Catalog error updating file digest. %s"), db_strerror(jcr->db)); } } } } break; } }
bool decompress_data(JCR *jcr, const char *last_fname, int32_t stream, char **data, uint32_t *length, bool want_data_stream) { Dmsg1(400, "Stream found in decompress_data(): %d\n", stream); switch (stream) { case STREAM_COMPRESSED_DATA: case STREAM_SPARSE_COMPRESSED_DATA: case STREAM_WIN32_COMPRESSED_DATA: case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA: case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA: { uint32_t comp_magic, comp_len; uint16_t comp_level, comp_version; /* * Read compress header */ unser_declare; unser_begin(*data, sizeof(comp_stream_header)); unser_uint32(comp_magic); unser_uint32(comp_len); unser_uint16(comp_level); unser_uint16(comp_version); unser_end(*data, sizeof(comp_stream_header)); Dmsg4(400, "Compressed data stream found: magic=0x%x, len=%d, level=%d, ver=0x%x\n", comp_magic, comp_len, comp_level, comp_version); /* * Version check */ if (comp_version != COMP_HEAD_VERSION) { Qmsg(jcr, M_ERROR, 0, _("Compressed header version error. version=0x%x\n"), comp_version); return false; } /* * Size check */ if (comp_len + sizeof(comp_stream_header) != *length) { Qmsg(jcr, M_ERROR, 0, _("Compressed header size error. comp_len=%d, msglen=%d\n"), comp_len, *length); return false; } /* * Based on the compression used perform the actual decompression of the data. */ switch (comp_magic) { #ifdef HAVE_LIBZ case COMPRESS_GZIP: switch (stream) { case STREAM_SPARSE_COMPRESSED_DATA: return decompress_with_zlib(jcr, last_fname, data, length, true, true, want_data_stream); default: return decompress_with_zlib(jcr, last_fname, data, length, false, true, want_data_stream); } #endif #ifdef HAVE_LZO case COMPRESS_LZO1X: switch (stream) { case STREAM_SPARSE_COMPRESSED_DATA: return decompress_with_lzo(jcr, last_fname, data, length, true, want_data_stream); default: return decompress_with_lzo(jcr, last_fname, data, length, false, want_data_stream); } #endif #ifdef HAVE_FASTLZ case COMPRESS_FZFZ: case COMPRESS_FZ4L: case COMPRESS_FZ4H: switch (stream) { case STREAM_SPARSE_COMPRESSED_DATA: return decompress_with_fastlz(jcr, last_fname, data, length, comp_magic, true, want_data_stream); default: return decompress_with_fastlz(jcr, last_fname, data, length, comp_magic, false, want_data_stream); } #endif default: Qmsg(jcr, M_ERROR, 0, _("Compression algorithm 0x%x found, but not supported!\n"), comp_magic); return false; } break; } default: #ifdef HAVE_LIBZ switch (stream) { case STREAM_SPARSE_GZIP_DATA: return decompress_with_zlib(jcr, last_fname, data, length, true, false, want_data_stream); default: return decompress_with_zlib(jcr, last_fname, data, length, false, false, want_data_stream); } #else Qmsg(jcr, M_ERROR, 0, _("Compression algorithm GZIP found, but not supported!\n")); return false; #endif } }
/* * Read the header record */ static bool read_header(DCR *dcr, DEV_BLOCK *block, DEV_RECORD *rec) { ser_declare; uint32_t VolSessionId; uint32_t VolSessionTime; int32_t FileIndex; int32_t Stream; uint32_t rhl; char buf1[100], buf2[100]; Dmsg0(dbgep, "=== rpath 1 read_header\n"); /* Clear state flags */ rec->state_bits = 0; if (block->dev->is_tape()) { rec->state_bits |= REC_ISTAPE; } rec->Block = ((DEVICE *)block->dev)->EndBlock; rec->File = ((DEVICE *)block->dev)->EndFile; /* * Get the header. There is always a full header, * otherwise we find it in the next block. */ Dmsg3(read_dbglvl, "Block=%d Ver=%d block_len=%u\n", block->BlockNumber, block->BlockVer, block->block_len); if (block->BlockVer == 1) { rhl = RECHDR1_LENGTH; } else { rhl = RECHDR2_LENGTH; } if (rec->remlen >= rhl) { Dmsg0(dbgep, "=== rpath 2 begin unserial header\n"); Dmsg4(read_dbglvl, "read_header: remlen=%d data_len=%d rem=%d blkver=%d\n", rec->remlen, rec->data_len, rec->remainder, block->BlockVer); unser_begin(block->bufp, WRITE_RECHDR_LENGTH); if (block->BlockVer == 1) { unser_uint32(VolSessionId); unser_uint32(VolSessionTime); } else { VolSessionId = block->VolSessionId; VolSessionTime = block->VolSessionTime; } unser_int32(FileIndex); unser_int32(Stream); unser_uint32(rec->data_bytes); block->bufp += rhl; block->binbuf -= rhl; rec->remlen -= rhl; /* If we are looking for more (remainder!=0), we reject anything * where the VolSessionId and VolSessionTime don't agree */ if (rec->remainder && (rec->VolSessionId != VolSessionId || rec->VolSessionTime != VolSessionTime)) { rec->state_bits |= REC_NO_MATCH; Dmsg0(read_dbglvl, "remainder and VolSession doesn't match\n"); Dmsg0(dbgep, "=== rpath 4 VolSession no match\n"); return false; /* This is from some other Session */ } /* if Stream is negative, it means that this is a continuation * of a previous partially written record. */ if (Stream < 0) { /* continuation record? */ Dmsg0(dbgep, "=== rpath 5 negative stream\n"); Dmsg1(read_dbglvl, "Got negative Stream => continuation. remainder=%d\n", rec->remainder); rec->state_bits |= REC_CONTINUATION; if (!rec->remainder) { /* if we didn't read previously */ Dmsg0(dbgep, "=== rpath 6 no remainder\n"); rec->data_len = 0; /* return data as if no continuation */ } else if (rec->Stream != -Stream) { Dmsg0(dbgep, "=== rpath 7 wrong cont stream\n"); rec->state_bits |= REC_NO_MATCH; return false; /* This is from some other Session */ } rec->Stream = -Stream; /* set correct Stream */ rec->maskedStream = rec->Stream & STREAMMASK_TYPE; } else { /* Regular record */ Dmsg0(dbgep, "=== rpath 8 normal stream\n"); rec->Stream = Stream; rec->maskedStream = rec->Stream & STREAMMASK_TYPE; rec->data_len = 0; /* transfer to beginning of data */ } rec->VolSessionId = VolSessionId; rec->VolSessionTime = VolSessionTime; rec->FileIndex = FileIndex; if (FileIndex > 0) { Dmsg0(dbgep, "=== rpath 9 FileIndex>0\n"); if (block->FirstIndex == 0) { Dmsg0(dbgep, "=== rpath 10 FirstIndex\n"); block->FirstIndex = FileIndex; } block->LastIndex = rec->FileIndex; } Dmsg6(read_dbglvl, "read_header: FI=%s SessId=%d Strm=%s len=%u rec->remlen=%d data_len=%d\n", FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId, stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_bytes, rec->remlen, rec->data_len); } else { Dmsg0(dbgep, "=== rpath 11a block out of records\n"); /* * No more records in this block because the number * of remaining bytes are less than a record header * length, so return empty handed, but indicate that * he must read again. By returning, we allow the * higher level routine to fetch the next block and * then reread. */ Dmsg0(read_dbglvl, "read_header: End of block\n"); rec->state_bits |= (REC_NO_HEADER | REC_BLOCK_EMPTY); empty_block(block); /* mark block empty */ return false; } /* Sanity check */ if (rec->data_bytes >= MAX_BLOCK_LENGTH) { Dmsg0(dbgep, "=== rpath 11b maxlen too big\n"); /* * Something is wrong, force read of next block, abort * continuing with this block. */ rec->state_bits |= (REC_NO_HEADER | REC_BLOCK_EMPTY); empty_block(block); Jmsg2(dcr->jcr, M_WARNING, 0, _("Sanity check failed. maxlen=%d datalen=%d. Block discarded.\n"), MAX_BLOCK_LENGTH, rec->data_bytes); return false; } rec->data = check_pool_memory_size(rec->data, rec->data_len+rec->data_bytes); rec->rstate = st_data; return true; }
/* * Read a Record from the block * * Returns: false if nothing read or if the continuation record does not match. * In both of these cases, a block read must be done. * true if at least the record header was read, this * routine may have to be called again with a new * block if the entire record was not read. */ bool read_record_from_block(DCR *dcr, DEV_RECORD *rec) { ser_declare; uint32_t remlen; uint32_t VolSessionId; uint32_t VolSessionTime; int32_t FileIndex; int32_t Stream; uint32_t data_bytes; uint32_t rhl; char buf1[100], buf2[100]; remlen = dcr->block->binbuf; /* * Clear state flags */ clear_all_bits(REC_STATE_MAX, rec->state_bits); if (dcr->block->dev->is_tape()) { set_bit(REC_ISTAPE, rec->state_bits); } rec->Block = ((DEVICE *)(dcr->block->dev))->EndBlock; rec->File = ((DEVICE *)(dcr->block->dev))->EndFile; /* * Get the header. There is always a full header, otherwise we find it in the next block. */ Dmsg3(450, "Block=%d Ver=%d size=%u\n", dcr->block->BlockNumber, dcr->block->BlockVer, dcr->block->block_len); if (dcr->block->BlockVer == 1) { rhl = RECHDR1_LENGTH; } else { rhl = RECHDR2_LENGTH; } if (remlen >= rhl) { Dmsg4(450, "Enter read_record_block: remlen=%d data_len=%d rem=%d blkver=%d\n", remlen, rec->data_len, rec->remainder, dcr->block->BlockVer); unser_begin(dcr->block->bufp, WRITE_RECHDR_LENGTH); if (dcr->block->BlockVer == 1) { unser_uint32(VolSessionId); unser_uint32(VolSessionTime); } else { VolSessionId = dcr->block->VolSessionId; VolSessionTime = dcr->block->VolSessionTime; } unser_int32(FileIndex); unser_int32(Stream); unser_uint32(data_bytes); dcr->block->bufp += rhl; dcr->block->binbuf -= rhl; remlen -= rhl; /* * If we are looking for more (remainder!=0), we reject anything * where the VolSessionId and VolSessionTime don't agree */ if (rec->remainder && (rec->VolSessionId != VolSessionId || rec->VolSessionTime != VolSessionTime)) { set_bit(REC_NO_MATCH, rec->state_bits); Dmsg0(450, "remainder and VolSession doesn't match\n"); return false; /* This is from some other Session */ } /* * If Stream is negative, it means that this is a continuation * of a previous partially written record. */ if (Stream < 0) { /* continuation record? */ Dmsg1(500, "Got negative Stream => continuation. remainder=%d\n", rec->remainder); set_bit(REC_CONTINUATION, rec->state_bits); if (!rec->remainder) { /* if we didn't read previously */ rec->data_len = 0; /* return data as if no continuation */ } else if (rec->Stream != -Stream) { set_bit(REC_NO_MATCH, rec->state_bits); return false; /* This is from some other Session */ } rec->Stream = -Stream; /* set correct Stream */ rec->maskedStream = rec->Stream & STREAMMASK_TYPE; } else { /* Regular record */ rec->Stream = Stream; rec->maskedStream = rec->Stream & STREAMMASK_TYPE; rec->data_len = 0; /* transfer to beginning of data */ } rec->VolSessionId = VolSessionId; rec->VolSessionTime = VolSessionTime; rec->FileIndex = FileIndex; if (FileIndex > 0) { if (dcr->block->FirstIndex == 0) { dcr->block->FirstIndex = FileIndex; } dcr->block->LastIndex = FileIndex; } Dmsg6(450, "rd_rec_blk() got FI=%s SessId=%d Strm=%s len=%u\n" "remlen=%d data_len=%d\n", FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId, stream_to_ascii(buf2, rec->Stream, rec->FileIndex), data_bytes, remlen, rec->data_len); } else { /* * No more records in this block because the number * of remaining bytes are less than a record header * length, so return empty handed, but indicate that * he must read again. By returning, we allow the * higher level routine to fetch the next block and * then reread. */ Dmsg0(450, "read_record_block: nothing\n"); set_bit(REC_NO_HEADER, rec->state_bits); set_bit(REC_BLOCK_EMPTY, rec->state_bits); empty_block(dcr->block); /* mark block empty */ return false; } /* Sanity check */ if (data_bytes >= MAX_BLOCK_LENGTH) { /* * Something is wrong, force read of next block, abort * continuing with this block. */ set_bit(REC_NO_HEADER, rec->state_bits); set_bit(REC_BLOCK_EMPTY, rec->state_bits); empty_block(dcr->block); Jmsg2(dcr->jcr, M_WARNING, 0, _("Sanity check failed. maxlen=%d datalen=%d. Block discarded.\n"), MAX_BLOCK_LENGTH, data_bytes); return false; } rec->data = check_pool_memory_size(rec->data, rec->data_len + data_bytes); /* * At this point, we have read the header, now we * must transfer as much of the data record as * possible taking into account: 1. A partial * data record may have previously been transferred, * 2. The current block may not contain the whole data * record. */ if (remlen >= data_bytes) { /* * Got whole record */ memcpy(rec->data+rec->data_len, dcr->block->bufp, data_bytes); dcr->block->bufp += data_bytes; dcr->block->binbuf -= data_bytes; rec->data_len += data_bytes; } else { /* * Partial record */ memcpy(rec->data+rec->data_len, dcr->block->bufp, remlen); dcr->block->bufp += remlen; dcr->block->binbuf -= remlen; rec->data_len += remlen; rec->remainder = 1; /* partial record transferred */ Dmsg1(450, "read_record_block: partial xfered=%d\n", rec->data_len); set_bit(REC_PARTIAL_RECORD, rec->state_bits); set_bit(REC_BLOCK_EMPTY, rec->state_bits); return true; } rec->remainder = 0; Dmsg4(450, "Rtn full rd_rec_blk FI=%s SessId=%d Strm=%s len=%d\n", FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId, stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len); return true; /* transferred full record */ }
/* * Called here for each record from read_records() */ static bool record_cb(DCR *dcr, DEV_RECORD *rec) { int status; JCR *jcr = dcr->jcr; if (rec->FileIndex < 0) { return true; /* we don't want labels */ } /* File Attributes stream */ switch (rec->maskedStream) { case STREAM_UNIX_ATTRIBUTES: case STREAM_UNIX_ATTRIBUTES_EX: /* If extracting, it was from previous stream, so * close the output file. */ if (extract) { if (!is_bopen(&bfd)) { Emsg0(M_ERROR, 0, _("Logic error output file should be open but is not.\n")); } set_attributes(jcr, attr, &bfd); extract = false; } if (!unpack_attributes_record(jcr, rec->Stream, rec->data, rec->data_len, attr)) { Emsg0(M_ERROR_TERM, 0, _("Cannot continue.\n")); } if (file_is_included(ff, attr->fname) && !file_is_excluded(ff, attr->fname)) { attr->data_stream = decode_stat(attr->attr, &attr->statp, sizeof(attr->statp), &attr->LinkFI); if (!is_restore_stream_supported(attr->data_stream)) { if (!non_support_data++) { Jmsg(jcr, M_ERROR, 0, _("%s stream not supported on this Client.\n"), stream_to_ascii(attr->data_stream)); } extract = false; return true; } build_attr_output_fnames(jcr, attr); if (attr->type == FT_DELETED) { /* TODO: choose the right fname/ofname */ Jmsg(jcr, M_INFO, 0, _("%s was deleted.\n"), attr->fname); extract = false; return true; } extract = false; status = create_file(jcr, attr, &bfd, REPLACE_ALWAYS); switch (status) { case CF_ERROR: case CF_SKIP: break; case CF_EXTRACT: extract = true; print_ls_output(jcr, attr); num_files++; fileAddr = 0; break; case CF_CREATED: set_attributes(jcr, attr, &bfd); print_ls_output(jcr, attr); num_files++; fileAddr = 0; break; } } break; case STREAM_RESTORE_OBJECT: /* nothing to do */ break; /* Data stream and extracting */ case STREAM_FILE_DATA: case STREAM_SPARSE_DATA: case STREAM_WIN32_DATA: if (extract) { if (rec->maskedStream == STREAM_SPARSE_DATA) { ser_declare; uint64_t faddr; wbuf = rec->data + OFFSET_FADDR_SIZE; wsize = rec->data_len - OFFSET_FADDR_SIZE; ser_begin(rec->data, OFFSET_FADDR_SIZE); unser_uint64(faddr); if (fileAddr != faddr) { fileAddr = faddr; if (blseek(&bfd, (boffset_t)fileAddr, SEEK_SET) < 0) { berrno be; Emsg2(M_ERROR_TERM, 0, _("Seek error on %s: %s\n"), attr->ofname, be.bstrerror()); } } } else { wbuf = rec->data; wsize = rec->data_len; } total += wsize; Dmsg2(8, "Write %u bytes, total=%u\n", wsize, total); store_data(&bfd, wbuf, wsize); fileAddr += wsize; } break; /* GZIP data stream */ case STREAM_GZIP_DATA: case STREAM_SPARSE_GZIP_DATA: case STREAM_WIN32_GZIP_DATA: #ifdef HAVE_LIBZ if (extract) { uLong compress_len = compress_buf_size; int status = Z_BUF_ERROR; if (rec->maskedStream == STREAM_SPARSE_GZIP_DATA) { ser_declare; uint64_t faddr; char ec1[50]; wbuf = rec->data + OFFSET_FADDR_SIZE; wsize = rec->data_len - OFFSET_FADDR_SIZE; ser_begin(rec->data, OFFSET_FADDR_SIZE); unser_uint64(faddr); if (fileAddr != faddr) { fileAddr = faddr; if (blseek(&bfd, (boffset_t)fileAddr, SEEK_SET) < 0) { berrno be; Emsg3(M_ERROR, 0, _("Seek to %s error on %s: ERR=%s\n"), edit_uint64(fileAddr, ec1), attr->ofname, be.bstrerror()); extract = false; return true; } } } else { wbuf = rec->data; wsize = rec->data_len; } while (compress_len < 10000000 && (status = uncompress((Byte *)compress_buf, &compress_len, (const Byte *)wbuf, (uLong)wsize)) == Z_BUF_ERROR) { /* The buffer size is too small, try with a bigger one */ compress_len = 2 * compress_len; compress_buf = check_pool_memory_size(compress_buf, compress_len); } if (status != Z_OK) { Emsg1(M_ERROR, 0, _("Uncompression error. ERR=%d\n"), status); extract = false; return true; } Dmsg2(100, "Write uncompressed %d bytes, total before write=%d\n", compress_len, total); store_data(&bfd, compress_buf, compress_len); total += compress_len; fileAddr += compress_len; Dmsg2(100, "Compress len=%d uncompressed=%d\n", rec->data_len, compress_len); } #else if (extract) { Emsg0(M_ERROR, 0, _("GZIP data stream found, but GZIP not configured!\n")); extract = false; return true; } #endif break; /* Compressed data stream */ case STREAM_COMPRESSED_DATA: case STREAM_SPARSE_COMPRESSED_DATA: case STREAM_WIN32_COMPRESSED_DATA: if (extract) { uint32_t comp_magic, comp_len; uint16_t comp_level, comp_version; #ifdef HAVE_LZO lzo_uint compress_len; const unsigned char *cbuf; int r, real_compress_len; #endif if (rec->maskedStream == STREAM_SPARSE_COMPRESSED_DATA) { ser_declare; uint64_t faddr; char ec1[50]; wbuf = rec->data + OFFSET_FADDR_SIZE; wsize = rec->data_len - OFFSET_FADDR_SIZE; ser_begin(rec->data, OFFSET_FADDR_SIZE); unser_uint64(faddr); if (fileAddr != faddr) { fileAddr = faddr; if (blseek(&bfd, (boffset_t)fileAddr, SEEK_SET) < 0) { berrno be; Emsg3(M_ERROR, 0, _("Seek to %s error on %s: ERR=%s\n"), edit_uint64(fileAddr, ec1), attr->ofname, be.bstrerror()); extract = false; return true; } } } else { wbuf = rec->data; wsize = rec->data_len; } /* read compress header */ unser_declare; unser_begin(wbuf, sizeof(comp_stream_header)); unser_uint32(comp_magic); unser_uint32(comp_len); unser_uint16(comp_level); unser_uint16(comp_version); Dmsg4(200, "Compressed data stream found: magic=0x%x, len=%d, level=%d, ver=0x%x\n", comp_magic, comp_len, comp_level, comp_version); /* version check */ if (comp_version != COMP_HEAD_VERSION) { Emsg1(M_ERROR, 0, _("Compressed header version error. version=0x%x\n"), comp_version); return false; } /* size check */ if (comp_len + sizeof(comp_stream_header) != wsize) { Emsg2(M_ERROR, 0, _("Compressed header size error. comp_len=%d, msglen=%d\n"), comp_len, wsize); return false; } switch(comp_magic) { #ifdef HAVE_LZO case COMPRESS_LZO1X: compress_len = compress_buf_size; cbuf = (const unsigned char*) wbuf + sizeof(comp_stream_header); real_compress_len = wsize - sizeof(comp_stream_header); Dmsg2(200, "Comp_len=%d msglen=%d\n", compress_len, wsize); while ((r=lzo1x_decompress_safe(cbuf, real_compress_len, (unsigned char *)compress_buf, &compress_len, NULL)) == LZO_E_OUTPUT_OVERRUN) { /* The buffer size is too small, try with a bigger one */ compress_len = 2 * compress_len; compress_buf = check_pool_memory_size(compress_buf, compress_len); } if (r != LZO_E_OK) { Emsg1(M_ERROR, 0, _("LZO uncompression error. ERR=%d\n"), r); extract = false; return true; } Dmsg2(100, "Write uncompressed %d bytes, total before write=%d\n", compress_len, total); store_data(&bfd, compress_buf, compress_len); total += compress_len; fileAddr += compress_len; Dmsg2(100, "Compress len=%d uncompressed=%d\n", rec->data_len, compress_len); break; #endif default: Emsg1(M_ERROR, 0, _("Compression algorithm 0x%x found, but not supported!\n"), comp_magic); extract = false; return true; } } break; case STREAM_MD5_DIGEST: case STREAM_SHA1_DIGEST: case STREAM_SHA256_DIGEST: case STREAM_SHA512_DIGEST: break; case STREAM_SIGNED_DIGEST: case STREAM_ENCRYPTED_SESSION_DATA: // TODO landonf: Investigate crypto support in the storage daemon break; case STREAM_PROGRAM_NAMES: case STREAM_PROGRAM_DATA: if (!prog_name_msg) { Pmsg0(000, _("Got Program Name or Data Stream. Ignored.\n")); prog_name_msg++; } break; case STREAM_UNIX_ACCESS_ACL: /* Deprecated Standard ACL attributes on UNIX */ case STREAM_UNIX_DEFAULT_ACL: /* Deprecated Default ACL attributes on UNIX */ case STREAM_ACL_AIX_TEXT: case STREAM_ACL_DARWIN_ACCESS_ACL: case STREAM_ACL_FREEBSD_DEFAULT_ACL: case STREAM_ACL_FREEBSD_ACCESS_ACL: case STREAM_ACL_HPUX_ACL_ENTRY: case STREAM_ACL_IRIX_DEFAULT_ACL: case STREAM_ACL_IRIX_ACCESS_ACL: case STREAM_ACL_LINUX_DEFAULT_ACL: case STREAM_ACL_LINUX_ACCESS_ACL: case STREAM_ACL_TRU64_DEFAULT_ACL: case STREAM_ACL_TRU64_DEFAULT_DIR_ACL: case STREAM_ACL_TRU64_ACCESS_ACL: case STREAM_ACL_SOLARIS_ACLENT: case STREAM_ACL_SOLARIS_ACE: case STREAM_ACL_AFS_TEXT: case STREAM_ACL_AIX_AIXC: case STREAM_ACL_AIX_NFS4: case STREAM_ACL_FREEBSD_NFS4_ACL: case STREAM_ACL_HURD_DEFAULT_ACL: case STREAM_ACL_HURD_ACCESS_ACL: if (extract) { wbuf = rec->data; wsize = rec->data_len; pm_strcpy(acl_data.last_fname, attr->fname); parse_acl_streams(jcr, &acl_data, rec->maskedStream, wbuf, wsize); } break; case STREAM_XATTR_HURD: case STREAM_XATTR_IRIX: case STREAM_XATTR_TRU64: case STREAM_XATTR_AIX: case STREAM_XATTR_OPENBSD: case STREAM_XATTR_SOLARIS_SYS: case STREAM_XATTR_SOLARIS: case STREAM_XATTR_DARWIN: case STREAM_XATTR_FREEBSD: case STREAM_XATTR_LINUX: case STREAM_XATTR_NETBSD: if (extract) { wbuf = rec->data; wsize = rec->data_len; pm_strcpy(xattr_data.last_fname, attr->fname); parse_xattr_streams(jcr, &xattr_data, rec->maskedStream, wbuf, wsize); } break; case STREAM_NDMP_SEPERATOR: break; default: /* If extracting, weird stream (not 1 or 2), close output file anyway */ if (extract) { if (!is_bopen(&bfd)) { Emsg0(M_ERROR, 0, _("Logic error output file should be open but is not.\n")); } set_attributes(jcr, attr, &bfd); extract = false; } Jmsg(jcr, M_ERROR, 0, _("Unknown stream=%d ignored. This shouldn't happen!\n"), rec->Stream); break; } /* end switch */ return true; }
struct container* retrieve_container_by_id(containerid id) { struct container *c = (struct container*) malloc(sizeof(struct container)); init_container_meta(&c->meta); unsigned char *cur = 0; if (destor.simulation_level >= SIMULATION_RESTORE) { c->data = malloc(CONTAINER_META_SIZE); pthread_mutex_lock(&mutex); if (destor.simulation_level >= SIMULATION_APPEND) fseek(fp, id * CONTAINER_META_SIZE + 8, SEEK_SET); else fseek(fp, (id + 1) * CONTAINER_SIZE - CONTAINER_META_SIZE + 8, SEEK_SET); fread(c->data, CONTAINER_META_SIZE, 1, fp); pthread_mutex_unlock(&mutex); cur = c->data; } else { c->data = malloc(CONTAINER_SIZE); pthread_mutex_lock(&mutex); fseek(fp, id * CONTAINER_SIZE + 8, SEEK_SET); fread(c->data, CONTAINER_SIZE, 1, fp); pthread_mutex_unlock(&mutex); cur = &c->data[CONTAINER_SIZE - CONTAINER_META_SIZE]; } unser_declare; unser_begin(cur, CONTAINER_META_SIZE); unser_int64(c->meta.id); unser_int32(c->meta.chunk_num); unser_int32(c->meta.data_size); if(c->meta.id != id){ WARNING("expect %lld, but read %lld", id, c->meta.id); assert(c->meta.id == id); } int i; for (i = 0; i < c->meta.chunk_num; i++) { struct metaEntry* me = (struct metaEntry*) malloc( sizeof(struct metaEntry)); unser_bytes(&me->fp, sizeof(fingerprint)); unser_bytes(&me->len, sizeof(int32_t)); unser_bytes(&me->off, sizeof(int32_t)); g_hash_table_insert(c->meta.map, &me->fp, me); } unser_end(cur, CONTAINER_META_SIZE); if (destor.simulation_level >= SIMULATION_RESTORE) { free(c->data); c->data = 0; } return c; }