/*
* Append Data sent from File daemon
*
*/
bool do_append_data(JCR *jcr)
{
int32_t n;
int32_t file_index, stream, last_file_index;
BSOCK *fd = jcr->file_bsock;
bool ok = true;
DEV_RECORD rec;
char buf1[100], buf2[100];
DCR *dcr = jcr->dcr;
DEVICE *dev;
char ec[50];
if (!dcr) {
Jmsg0(jcr, M_FATAL, 0, _("DCR is NULL!!!\n"));
return false;
}
dev = dcr->dev;
if (!dev) {
Jmsg0(jcr, M_FATAL, 0, _("DEVICE is NULL!!!\n"));
return false;
}
Dmsg1(100, "Start append data. res=%d\n", dev->num_reserved());
memset(&rec, 0, sizeof(rec));
if (!fd->set_buffer_size(dcr->device->max_network_buffer_size, BNET_SETBUF_WRITE)) {
jcr->setJobStatus(JS_ErrorTerminated);
Jmsg0(jcr, M_FATAL, 0, _("Unable to set network buffer size.\n"));
return false;
}
if (!acquire_device_for_append(dcr)) {
jcr->setJobStatus(JS_ErrorTerminated);
return false;
}
jcr->setJobStatus(JS_Running);
dir_send_job_status(jcr);
if (dev->VolCatInfo.VolCatName[0] == 0) {
Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
}
Dmsg1(50, "Begin append device=%s\n", dev->print_name());
begin_data_spool(dcr);
begin_attribute_spool(jcr);
Dmsg0(100, "Just after acquire_device_for_append\n");
if (dev->VolCatInfo.VolCatName[0] == 0) {
Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
}
/*
* Write Begin Session Record
*/
if (!write_session_label(dcr, SOS_LABEL)) {
Jmsg1(jcr, M_FATAL, 0, _("Write session label failed. ERR=%s\n"),
dev->bstrerror());
jcr->setJobStatus(JS_ErrorTerminated);
ok = false;
}
if (dev->VolCatInfo.VolCatName[0] == 0) {
Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
}
/* Tell File daemon to send data */
if (!fd->fsend(OK_data)) {
berrno be;
Jmsg1(jcr, M_FATAL, 0, _("Network send error to FD. ERR=%s\n"),
be.bstrerror(fd->b_errno));
ok = false;
}
/*
* Get Data from File daemon, write to device. To clarify what is
* going on here. We expect:
* - A stream header
* - Multiple records of data
* - EOD record
*
* The Stream header is just used to sychronize things, and
* none of the stream header is written to tape.
* The Multiple records of data, contain first the Attributes,
* then after another stream header, the file data, then
* after another stream header, the MD5 data if any.
*
* So we get the (stream header, data, EOD) three time for each
* file. 1. for the Attributes, 2. for the file data if any,
* and 3. for the MD5 if any.
*/
dcr->VolFirstIndex = dcr->VolLastIndex = 0;
jcr->run_time = time(NULL); /* start counting time for rates */
for (last_file_index = 0; ok && !jcr->is_job_canceled(); ) {
/* Read Stream header from the File daemon.
* The stream header consists of the following:
* file_index (sequential Bacula file index, base 1)
* stream (Bacula number to distinguish parts of data)
* info (Info for Storage daemon -- compressed, encrypted, ...)
* info is not currently used, so is read, but ignored!
*/
if ((n=bget_msg(fd)) <= 0) {
if (n == BNET_SIGNAL && fd->msglen == BNET_EOD) {
break; /* end of data */
}
Jmsg1(jcr, M_FATAL, 0, _("Error reading data header from FD. ERR=%s\n"),
fd->bstrerror());
possible_incomplete_job(jcr, last_file_index);
ok = false;
break;
}
if (sscanf(fd->msg, "%ld %ld", &file_index, &stream) != 2) {
Jmsg1(jcr, M_FATAL, 0, _("Malformed data header from FD: %s\n"), fd->msg);
ok = false;
possible_incomplete_job(jcr, last_file_index);
break;
}
Dmsg2(890, "<filed: Header FilInx=%d stream=%d\n", file_index, stream);
/*
* We make sure the file_index is advancing sequentially.
* An incomplete job can start the file_index at any number.
* otherwise, it must start at 1.
*/
if (jcr->rerunning && file_index > 0 && last_file_index == 0) {
goto fi_checked;
}
if (file_index > 0 && (file_index == last_file_index ||
file_index == last_file_index + 1)) {
goto fi_checked;
}
Jmsg2(jcr, M_FATAL, 0, _("FI=%d from FD not positive or sequential=%d\n"),
file_index, last_file_index);
possible_incomplete_job(jcr, last_file_index);
ok = false;
break;
fi_checked:
if (file_index != last_file_index) {
jcr->JobFiles = file_index;
last_file_index = file_index;
}
/* Read data stream from the File daemon.
* The data stream is just raw bytes
*/
while ((n=bget_msg(fd)) > 0 && !jcr->is_job_canceled()) {
rec.VolSessionId = jcr->VolSessionId;
rec.VolSessionTime = jcr->VolSessionTime;
rec.FileIndex = file_index;
rec.Stream = stream;
rec.maskedStream = stream & STREAMMASK_TYPE; /* strip high bits */
rec.data_len = fd->msglen;
rec.data = fd->msg; /* use message buffer */
Dmsg4(850, "before writ_rec FI=%d SessId=%d Strm=%s len=%d\n",
rec.FileIndex, rec.VolSessionId,
stream_to_ascii(buf1, rec.Stream,rec.FileIndex),
rec.data_len);
while (!write_record_to_block(dcr->block, &rec)) {
Dmsg2(850, "!write_record_to_block data_len=%d rem=%d\n", rec.data_len,
rec.remainder);
if (!write_block_to_device(dcr)) {
Dmsg2(90, "Got write_block_to_dev error on device %s. %s\n",
dev->print_name(), dev->bstrerror());
ok = false;
break;
}
}
if (!ok) {
Dmsg0(400, "Not OK\n");
break;
}
jcr->JobBytes += rec.data_len; /* increment bytes this job */
Dmsg4(850, "write_record FI=%s SessId=%d Strm=%s len=%d\n",
FI_to_ascii(buf1, rec.FileIndex), rec.VolSessionId,
stream_to_ascii(buf2, rec.Stream, rec.FileIndex), rec.data_len);
send_attrs_to_dir(jcr, &rec);
Dmsg0(650, "Enter bnet_get\n");
}
Dmsg1(650, "End read loop with FD. Stat=%d\n", n);
if (fd->is_error()) {
if (!jcr->is_job_canceled()) {
Dmsg1(350, "Network read error from FD. ERR=%s\n", fd->bstrerror());
Jmsg1(jcr, M_FATAL, 0, _("Network error reading from FD. ERR=%s\n"),
fd->bstrerror());
possible_incomplete_job(jcr, last_file_index);
}
ok = false;
break;
}
}
/* Create Job status for end of session label */
jcr->setJobStatus(ok?JS_Terminated:JS_ErrorTerminated);
if (ok) {
/* Terminate connection with FD */
fd->fsend(OK_append);
do_fd_commands(jcr); /* finish dialog with FD */
} else {
fd->fsend("3999 Failed append\n");
}
/*
* Don't use time_t for job_elapsed as time_t can be 32 or 64 bits,
* and the subsequent Jmsg() editing will break
*/
int32_t job_elapsed = time(NULL) - jcr->run_time;
if (job_elapsed <= 0) {
job_elapsed = 1;
}
Jmsg(dcr->jcr, M_INFO, 0, _("Job write elapsed time = %02d:%02d:%02d, Transfer rate = %s Bytes/second\n"),
job_elapsed / 3600, job_elapsed % 3600 / 60, job_elapsed % 60,
edit_uint64_with_suffix(jcr->JobBytes / job_elapsed, ec));
Dmsg1(200, "Write EOS label JobStatus=%c\n", jcr->JobStatus);
/*
* Check if we can still write. This may not be the case
* if we are at the end of the tape or we got a fatal I/O error.
*/
if (ok || dev->can_write()) {
if (!write_session_label(dcr, EOS_LABEL)) {
/* Print only if ok and not cancelled to avoid spurious messages */
if (ok && !jcr->is_job_canceled()) {
Jmsg1(jcr, M_FATAL, 0, _("Error writing end session label. ERR=%s\n"),
dev->bstrerror());
possible_incomplete_job(jcr, last_file_index);
}
jcr->setJobStatus(JS_ErrorTerminated);
ok = false;
}
if (dev->VolCatInfo.VolCatName[0] == 0) {
Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
Dmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
}
Dmsg0(90, "back from write_end_session_label()\n");
/* Flush out final partial block of this session */
if (!write_block_to_device(dcr)) {
/* Print only if ok and not cancelled to avoid spurious messages */
if (ok && !jcr->is_job_canceled()) {
Jmsg2(jcr, M_FATAL, 0, _("Fatal append error on device %s: ERR=%s\n"),
dev->print_name(), dev->bstrerror());
Dmsg0(100, _("Set ok=FALSE after write_block_to_device.\n"));
possible_incomplete_job(jcr, last_file_index);
}
jcr->setJobStatus(JS_ErrorTerminated);
ok = false;
}
}
if (!ok && !jcr->is_JobStatus(JS_Incomplete)) {
discard_data_spool(dcr);
} else {
/* Note: if commit is OK, the device will remain blocked */
commit_data_spool(dcr);
}
if (ok) {
ok = dvd_close_job(dcr); /* do DVD cleanup if any */
}
/*
* Release the device -- and send final Vol info to DIR
* and unlock it.
*/
release_device(dcr);
if ((!ok || jcr->is_job_canceled()) && !jcr->is_JobStatus(JS_Incomplete)) {
discard_attribute_spool(jcr);
} else {
commit_attribute_spool(jcr);
}
dir_send_job_status(jcr); /* update director */
Dmsg1(100, "return from do_append_data() ok=%d\n", ok);
return ok;
}
/*
* Compare a files chksum against a stored chksum.
*
* Returns: true if chksum matches.
* false if chksum is different.
*/
bool calculate_and_compare_file_chksum(JCR *jcr, FF_PKT *ff_pkt,
const char *fname, const char *chksum)
{
DIGEST *digest = NULL;
int digest_stream = STREAM_NONE;
char *digest_buf = NULL;
const char *digest_name;
bool retval = false;
if (calculate_file_chksum(jcr, ff_pkt, &digest, &digest_stream, &digest_buf, &digest_name)) {
if (digest_stream != STREAM_NONE && digest == NULL) {
Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"), stream_to_ascii(digest_stream));
} else if (digest && digest_buf) {
if (!bstrcmp(digest_buf, chksum)) {
Dmsg4(100, "%s %s chksum diff. Cat: %s File: %s\n", fname, digest_name, chksum, digest_buf);
} else {
retval = true;
}
}
if (digest_buf) {
free(digest_buf);
}
if (digest) {
crypto_digest_free(digest);
}
}
return retval;
}
void dump_record(const char *tag, const DEV_RECORD *rec)
{
char stream[128];
char findex[128];
Dmsg2(100, "%s: rec %p\n", tag, rec);
Dmsg3(100, "%-14s next %p prev %p\n", "link", rec->link.next, rec->link.prev);
Dmsg2(100, "%-14s %u\n", "File", rec->File);
Dmsg2(100, "%-14s %u\n", "Block", rec->Block);
Dmsg2(100, "%-14s %u\n", "VolSessionId", rec->VolSessionId);
Dmsg2(100, "%-14s %u\n", "VolSessionTime", rec->VolSessionTime);
Dmsg2(100, "%-14s %s\n", "FileIndex", findex_to_str(rec->FileIndex, findex, sizeof(findex)));
Dmsg2(100, "%-14s %s\n", "Stream", stream_to_ascii(stream, rec->Stream, rec->FileIndex));
Dmsg2(100, "%-14s %d\n", "maskedStream", rec->maskedStream);
Dmsg2(100, "%-14s %u\n", "data_len", rec->data_len);
Dmsg2(100, "%-14s %u\n", "remainder", rec->remainder);
for (unsigned int i = 0; i < (sizeof(rec->state_bits) / sizeof(rec->state_bits[0])); i++) {
Dmsg3(100, "%-11s[%d] %2.2x\n", "state_bits", i, (uint8_t)rec->state_bits[i]);
}
Dmsg3(100, "%-14s %u (%s)\n", "state", rec->state, record_state_to_ascii(rec->state));
Dmsg2(100, "%-14s %p\n", "bsr", rec->bsr);
for (unsigned int i = 0; i < (sizeof(rec->ser_buf) / sizeof(rec->ser_buf[0])); i++) {
Dmsg3(100, "%-11s[%d] %2.2x\n", "ser_buf", i, (uint8_t)rec->ser_buf[i]);
}
Dmsg2(100, "%-14s %p\n", "data", rec->data);
Dmsg2(100, "%-14s %d\n", "match_stat", rec->match_stat);
Dmsg2(100, "%-14s %u\n", "last_VolSessionId", rec->last_VolSessionId);
Dmsg2(100, "%-14s %u\n", "last_VolSessionTime", rec->last_VolSessionTime);
Dmsg2(100, "%-14s %d\n", "last_FileIndex", rec->last_FileIndex);
Dmsg2(100, "%-14s %d\n", "last_Stream", rec->last_Stream);
Dmsg2(100, "%-14s %s\n", "own_mempool", rec->own_mempool ? "true" : "false");
}
static inline ssize_t write_data_to_block(DEV_BLOCK *block, const DEV_RECORD *rec)
{
uint32_t len;
len = MIN(rec->remainder, block_write_navail(block));
memcpy(block->bufp,
((unsigned char *)rec->data) + (rec->data_len - rec->remainder),
len);
block->bufp += len;
block->binbuf += len;
#ifdef xxxxxSMCHECK
if (!sm_check_rtn(__FILE__, __LINE__, False)) {
/*
* We damaged a buffer
*/
Dmsg7(0, "Damaged block FI=%s SessId=%d Strm=%s datalen=%d\n"
"len=%d rem=%d remainder=%d\n",
FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId,
stream_to_ascii(buf2, rec->Stream, rec->FileIndex),
rec->data_len, len, block_write_navail(block),
rec->remainder);
Dmsg5(0, "Damaged block: bufp=%x binbuf=%d buf_len=%d rem=%d moved=%d\n",
block->bufp, block->binbuf, block->buf_len,
block_write_navail(block), len);
Dmsg2(0, "Damaged block: buf=%x binbuffrombuf=%d \n",
block->buf, block->bufp-block->buf);
Emsg0(M_ABORT, 0, _("Damaged buffer\n"));
}
#endif
return len;
}
/*
* Write a Record to the block
*
* Returns: false means the block could not be written to tape/disk.
* true on success (all bytes written to the block).
*/
bool DCR::write_record()
{
bool retval = false;
bool translated_record = false;
char buf1[100], buf2[100];
/*
* Perform record translations.
*/
before_rec = rec;
after_rec = NULL;
if (generate_plugin_event(jcr, bsdEventWriteRecordTranslation, this) != bRC_OK) {
goto bail_out;
}
/*
* The record got translated when we got an after_rec pointer after calling the
* bsdEventWriteRecordTranslation plugin event. If no translation has taken place
* we just point the after_rec pointer to same DEV_RECORD as in the before_rec pointer.
*/
if (!after_rec) {
after_rec = before_rec;
} else {
translated_record = true;
}
while (!write_record_to_block(this, after_rec)) {
Dmsg2(850, "!write_record_to_block data_len=%d rem=%d\n",
after_rec->data_len, after_rec->remainder);
if (!write_block_to_device()) {
Dmsg2(90, "Got write_block_to_dev error on device %s. %s\n",
dev->print_name(), dev->bstrerror());
goto bail_out;
}
}
jcr->JobBytes += after_rec->data_len; /* increment bytes this job */
if (jcr->RemainingQuota && jcr->JobBytes > jcr->RemainingQuota) {
Jmsg0(jcr, M_FATAL, 0, _("Quota Exceeded. Job Terminated.\n"));
goto bail_out;
}
Dmsg4(850, "write_record FI=%s SessId=%d Strm=%s len=%d\n",
FI_to_ascii(buf1, after_rec->FileIndex), after_rec->VolSessionId,
stream_to_ascii(buf2, after_rec->Stream, after_rec->FileIndex), after_rec->data_len);
retval = true;
bail_out:
if (translated_record) {
copy_record_state(before_rec, after_rec);
free_record(after_rec);
after_rec = NULL;
}
return retval;
}
/* unser_volume_label
*
* Unserialize the Bareos Volume label into the device Volume_Label
* structure.
*
* Assumes that the record is already read.
*
* Returns: false on error
* true on success
*/
bool unser_volume_label(DEVICE *dev, DEV_RECORD *rec)
{
ser_declare;
char buf1[100], buf2[100];
if (rec->FileIndex != VOL_LABEL && rec->FileIndex != PRE_LABEL) {
Mmsg3(dev->errmsg, _("Expecting Volume Label, got FI=%s Stream=%s len=%d\n"),
FI_to_ascii(buf1, rec->FileIndex),
stream_to_ascii(buf2, rec->Stream, rec->FileIndex),
rec->data_len);
if (!forge_on) {
return false;
}
}
dev->VolHdr.LabelType = rec->FileIndex;
dev->VolHdr.LabelSize = rec->data_len;
/* Unserialize the record into the Volume Header */
rec->data = check_pool_memory_size(rec->data, SER_LENGTH_Volume_Label);
ser_begin(rec->data, SER_LENGTH_Volume_Label);
unser_string(dev->VolHdr.Id);
unser_uint32(dev->VolHdr.VerNum);
if (dev->VolHdr.VerNum >= 11) {
unser_btime(dev->VolHdr.label_btime);
unser_btime(dev->VolHdr.write_btime);
} else { /* old way */
unser_float64(dev->VolHdr.label_date);
unser_float64(dev->VolHdr.label_time);
}
unser_float64(dev->VolHdr.write_date); /* Unused with VerNum >= 11 */
unser_float64(dev->VolHdr.write_time); /* Unused with VerNum >= 11 */
unser_string(dev->VolHdr.VolumeName);
unser_string(dev->VolHdr.PrevVolumeName);
unser_string(dev->VolHdr.PoolName);
unser_string(dev->VolHdr.PoolType);
unser_string(dev->VolHdr.MediaType);
unser_string(dev->VolHdr.HostName);
unser_string(dev->VolHdr.LabelProg);
unser_string(dev->VolHdr.ProgVersion);
unser_string(dev->VolHdr.ProgDate);
ser_end(rec->data, SER_LENGTH_Volume_Label);
Dmsg0(190, "unser_vol_label\n");
if (debug_level >= 190) {
dump_volume_label(dev);
}
return true;
}
/*
* Called here for each record from read_records()
*
* Returns: true if OK
* false if error
*/
static bool record_cb(DCR *dcr, DEV_RECORD *rec)
{
JCR *jcr = dcr->jcr;
BSOCK *fd = jcr->file_bsock;
bool ok = true;
POOLMEM *save_msg;
char ec1[50], ec2[50];
if (rec->FileIndex < 0) {
return true;
}
Dmsg5(400, "Send to FD: SessId=%u SessTim=%u FI=%s Strm=%s, len=%d\n",
rec->VolSessionId, rec->VolSessionTime,
FI_to_ascii(ec1, rec->FileIndex),
stream_to_ascii(ec2, rec->Stream, rec->FileIndex),
rec->data_len);
/*
* Send record header to File daemon
*/
if (!fd->fsend(rec_header, rec->VolSessionId, rec->VolSessionTime,
rec->FileIndex, rec->Stream, rec->data_len)) {
Pmsg1(000, _(">filed: Error Hdr=%s"), fd->msg);
Jmsg1(jcr, M_FATAL, 0, _("Error sending to File daemon. ERR=%s\n"),
fd->bstrerror());
return false;
} else {
Dmsg1(400, ">filed: Hdr=%s", fd->msg);
}
/*
* Send data record to File daemon
*/
save_msg = fd->msg; /* save fd message pointer */
fd->msg = rec->data; /* pass data directly to the FD */
fd->msglen = rec->data_len;
Dmsg1(400, ">filed: send %d bytes data.\n", fd->msglen);
if (!fd->send()) {
Pmsg1(000, _("Error sending to FD. ERR=%s\n"), fd->bstrerror());
Jmsg1(jcr, M_FATAL, 0, _("Error sending to File daemon. ERR=%s\n"),
fd->bstrerror());
ok = false;
}
fd->msg = save_msg; /* restore fd message pointer */
return ok;
}
/*
* Called here for each record from read_records()
*/
static bool record_cb(DCR *dcr, DEV_RECORD *rec)
{
if (verbose && rec->FileIndex < 0) {
dump_label_record(dcr->dev, rec, verbose);
return true;
}
if (verbose) {
char buf1[100], buf2[100];
Pmsg6(000, "Record: FI=%s SessId=%d Strm=%s len=%u remlen=%d data_len=%d\n",
FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId,
stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_bytes, rec->remlen,
rec->data_len);
}
/* File Attributes stream */
if (rec->maskedStream == STREAM_UNIX_ATTRIBUTES ||
rec->maskedStream == STREAM_UNIX_ATTRIBUTES_EX) {
if (!unpack_attributes_record(jcr, rec->Stream, rec->data, rec->data_len, attr)) {
if (!forge_on) {
Emsg0(M_ERROR_TERM, 0, _("Cannot continue.\n"));
} else {
Emsg0(M_ERROR, 0, _("Attrib unpack error!\n"));
}
num_files++;
return true;
}
attr->data_stream = decode_stat(attr->attr, &attr->statp, sizeof(attr->statp), &attr->LinkFI);
build_attr_output_fnames(jcr, attr);
if (file_is_included(ff, attr->fname) && !file_is_excluded(ff, attr->fname)) {
if (verbose) {
Pmsg5(000, _("FileIndex=%d VolSessionId=%d VolSessionTime=%d Stream=%d DataLen=%d\n"),
rec->FileIndex, rec->VolSessionId, rec->VolSessionTime, rec->Stream, rec->data_len);
}
print_ls_output(jcr, attr);
num_files++;
}
} else if (rec->Stream == STREAM_PLUGIN_NAME) {
char data[100];
int len = MIN(rec->data_len+1, sizeof(data));
bstrncpy(data, rec->data, len);
Dmsg1(100, "Plugin data: %s\n", data);
} else if (rec->Stream == STREAM_RESTORE_OBJECT) {
Dmsg0(100, "Restore Object record\n");
}
return true;
}
/*
* Read a Record from the block
* Returns: false if nothing read or if the continuation record does not match.
* In both of these cases, a block read must be done.
* true if at least the record header was read, this
* routine may have to be called again with a new
* block if the entire record was not read.
*/
bool read_record_from_block(DCR *dcr, DEV_RECORD *rec)
{
bool rtn;
Dmsg0(dbgep, "=== rpath 1 Enter read_record_from block\n");
for ( ;; ) {
switch (rec->rstate) {
case st_none:
dump_block(dcr->block, "st_none");
case st_header:
Dmsg0(dbgep, "=== rpath 33 st_header\n");
rec->remlen = dcr->block->binbuf;
/* Note read_header sets rec->rstate on return true */
if (!read_header(dcr, dcr->block, rec)) { /* sets state */
Dmsg0(dbgep, "=== rpath 34 failed read header\n");
Dmsg0(read_dbglvl, "read_header returned EOF.\n");
goto fail_out;
}
continue;
case st_data:
Dmsg0(dbgep, "=== rpath 37 st_data\n");
read_data(dcr->block, rec);
rec->rstate = st_header; /* next pass look for a header */
goto get_out;
default:
Dmsg0(dbgep, "=== rpath 50 default\n");
Dmsg0(0, "======= In default !!!!!\n");
Pmsg1(190, "Read: unknown state=%d\n", rec->rstate);
goto fail_out;
}
}
get_out:
char buf1[100], buf2[100];
Dmsg5(read_dbglvl, "read_rec return: FI=%s Strm=%s len=%d rem=%d remainder=%d\n",
FI_to_ascii(buf1, rec->FileIndex),
stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len,
rec->remlen, rec->remainder);
rtn = true;
goto out;
fail_out:
rec->rstate = st_none;
rtn = false;
out:
return rtn;
}
static inline bool write_data_to_block(DEV_BLOCK *block, DEV_RECORD *rec)
{
rec->remlen = block->buf_len - block->binbuf;
/*
* Write as much of data as possible
*/
if (rec->remlen >= rec->remainder) {
memcpy(block->bufp, rec->data + (rec->data_len - rec->remainder), rec->remainder);
block->bufp += rec->remainder;
block->binbuf += rec->remainder;
} else {
memcpy(block->bufp, rec->data + (rec->data_len - rec->remainder), rec->remlen);
#ifdef xxxxxSMCHECK
if (!sm_check_rtn(__FILE__, __LINE__, False)) {
/*
* We damaged a buffer
*/
Dmsg6(0, "Damaged block FI=%s SessId=%d Strm=%s len=%d\n"
"rem=%d remainder=%d\n",
FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId,
stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len,
rec->remlen, rec->remainder);
Dmsg5(0, "Damaged block: bufp=%x binbuf=%d buf_len=%d rem=%d moved=%d\n",
block->bufp, block->binbuf, block->buf_len, block->buf_len-block->binbuf,
rec->remlen);
Dmsg2(0, "Damaged block: buf=%x binbuffrombuf=%d \n",
block->buf, block->bufp-block->buf);
Emsg0(M_ABORT, 0, _("Damaged buffer\n"));
}
#endif
block->bufp += rec->remlen;
block->binbuf += rec->remlen;
rec->remainder -= rec->remlen;
return false; /* did partial transfer */
}
return true;
}
/*
* We have just read a header, now read the data into the record.
* Note, if we do not read the full record, we will return to
* read the next header, which will then come back here later
* to finish reading the full record.
*/
static void read_data(DEV_BLOCK *block, DEV_RECORD *rec)
{
char buf1[100], buf2[100];
Dmsg0(dbgep, "=== rpath 22 read_data\n");
/*
* At this point, we have read the header, now we
* must transfer as much of the data record as
* possible taking into account: 1. A partial
* data record may have previously been transferred,
* 2. The current block may not contain the whole data
* record.
*/
if (rec->remlen >= rec->data_bytes) {
Dmsg0(dbgep, "=== rpath 23 full record\n");
/* Got whole record */
memcpy(rec->data+rec->data_len, block->bufp, rec->data_bytes);
block->bufp += rec->data_bytes;
block->binbuf -= rec->data_bytes;
rec->data_len += rec->data_bytes;
rec->remainder = 0;
Dmsg5(190, "Rdata full FI=%s SessId=%d Strm=%s len=%d block=%p\n",
FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId,
stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len,
block);
} else {
Dmsg0(dbgep, "=== rpath 24 partial record\n");
/* Partial record */
memcpy(rec->data+rec->data_len, block->bufp, rec->remlen);
block->bufp += rec->remlen;
block->binbuf -= rec->remlen;
rec->data_len += rec->remlen;
rec->remainder = 1; /* partial record transferred */
Dmsg1(read_dbglvl, "read_data: partial xfered=%d\n", rec->data_len);
rec->state_bits |= (REC_PARTIAL_RECORD | REC_BLOCK_EMPTY);
}
}
/*
* Called here by find() for each file included.
* This is a callback. The original is find_files() above.
*
* Send the file and its data to the Storage daemon.
*
* Returns: 1 if OK
* 0 if error
* -1 to ignore file/directory (not used here)
*/
int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
{
bool do_read = false;
int stat, data_stream;
int rtnstat = 0;
DIGEST *digest = NULL;
DIGEST *signing_digest = NULL;
int digest_stream = STREAM_NONE;
SIGNATURE *sig = NULL;
bool has_file_data = false;
// TODO landonf: Allow the user to specify the digest algorithm
#ifdef HAVE_SHA2
crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
#else
crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
#endif
BSOCK *sd = jcr->store_bsock;
if (job_canceled(jcr)) {
return 0;
}
jcr->num_files_examined++; /* bump total file count */
switch (ff_pkt->type) {
case FT_LNKSAVED: /* Hard linked, file already saved */
Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
break;
case FT_REGE:
Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
has_file_data = true;
break;
case FT_REG:
Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
has_file_data = true;
break;
case FT_LNK:
Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
break;
case FT_DIRBEGIN:
jcr->num_files_examined--; /* correct file count */
return 1; /* not used */
case FT_NORECURSE:
Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
ff_pkt->top_fname, ff_pkt->fname);
ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
break;
case FT_NOFSCHG:
/* Suppress message for /dev filesystems */
if (!is_in_fileset(ff_pkt)) {
Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
}
ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
break;
case FT_INVALIDFS:
Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
ff_pkt->top_fname, ff_pkt->fname);
ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
break;
case FT_INVALIDDT:
Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
ff_pkt->fname);
break;
case FT_REPARSE:
case FT_DIREND:
Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
break;
case FT_SPEC:
Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
if (S_ISSOCK(ff_pkt->statp.st_mode)) {
Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname);
return 1;
}
break;
case FT_RAW:
Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
has_file_data = true;
break;
case FT_FIFO:
Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
break;
case FT_NOACCESS: {
berrno be;
Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access \"%s\": ERR=%s\n"), ff_pkt->fname,
be.bstrerror(ff_pkt->ff_errno));
jcr->JobErrors++;
return 1;
}
case FT_NOFOLLOW: {
berrno be;
Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link \"%s\": ERR=%s\n"),
ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
jcr->JobErrors++;
return 1;
}
case FT_NOSTAT: {
berrno be;
Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat \"%s\": ERR=%s\n"), ff_pkt->fname,
be.bstrerror(ff_pkt->ff_errno));
jcr->JobErrors++;
return 1;
}
case FT_DIRNOCHG:
case FT_NOCHG:
Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
return 1;
case FT_ISARCH:
Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
return 1;
case FT_NOOPEN: {
berrno be;
Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory \"%s\": ERR=%s\n"),
ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
jcr->JobErrors++;
return 1;
}
default:
Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
ff_pkt->type, ff_pkt->fname);
jcr->JobErrors++;
return 1;
}
Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
/* Digests and encryption are only useful if there's file data */
if (has_file_data) {
/*
* Setup for digest handling. If this fails, the digest will be set to NULL
* and not used. Note, the digest (file hash) can be any one of the four
* algorithms below.
*
* The signing digest is a single algorithm depending on
* whether or not we have SHA2.
* ****FIXME**** the signing algoritm should really be
* determined a different way!!!!!! What happens if
* sha2 was available during backup but not restore?
*/
if (ff_pkt->flags & FO_MD5) {
digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
digest_stream = STREAM_MD5_DIGEST;
} else if (ff_pkt->flags & FO_SHA1) {
digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
digest_stream = STREAM_SHA1_DIGEST;
} else if (ff_pkt->flags & FO_SHA256) {
digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
digest_stream = STREAM_SHA256_DIGEST;
} else if (ff_pkt->flags & FO_SHA512) {
digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
digest_stream = STREAM_SHA512_DIGEST;
}
/* Did digest initialization fail? */
if (digest_stream != STREAM_NONE && digest == NULL) {
Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
stream_to_ascii(digest_stream));
}
/*
* Set up signature digest handling. If this fails, the signature digest will be set to
* NULL and not used.
*/
// TODO landonf: We should really only calculate the digest once, for both verification and signing.
if (jcr->crypto.pki_sign) {
signing_digest = crypto_digest_new(jcr, signing_algorithm);
/* Full-stop if a failure occurred initializing the signature digest */
if (signing_digest == NULL) {
Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
stream_to_ascii(signing_algorithm));
jcr->JobErrors++;
goto good_rtn;
}
}
/* Enable encryption */
if (jcr->crypto.pki_encrypt) {
ff_pkt->flags |= FO_ENCRYPT;
}
}
/* Initialize the file descriptor we use for data and other streams. */
binit(&ff_pkt->bfd);
if (ff_pkt->flags & FO_PORTABLE) {
set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
}
if (ff_pkt->cmd_plugin) {
if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) {
goto bail_out;
}
send_plugin_name(jcr, sd, true); /* signal start of plugin data */
}
/* Send attributes -- must be done after binit() */
if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
goto bail_out;
}
/* Set up the encryption context and send the session data to the SD */
if (has_file_data && jcr->crypto.pki_encrypt) {
if (!crypto_session_send(jcr, sd)) {
goto bail_out;
}
}
/*
* Open any file with data that we intend to save, then save it.
*
* Note, if is_win32_backup, we must open the Directory so that
* the BackupRead will save its permissions and ownership streams.
*/
if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
#ifdef HAVE_WIN32
do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
#else
do_read = ff_pkt->statp.st_size > 0;
#endif
} else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
ff_pkt->type == FT_REPARSE ||
(!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
do_read = true;
}
if (ff_pkt->cmd_plugin) {
do_read = true;
}
Dmsg1(400, "do_read=%d\n", do_read);
if (do_read) {
btimer_t *tid;
if (ff_pkt->type == FT_FIFO) {
tid = start_thread_timer(jcr, pthread_self(), 60);
} else {
tid = NULL;
}
int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
ff_pkt->bfd.reparse_point = ff_pkt->type == FT_REPARSE;
if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
ff_pkt->ff_errno = errno;
berrno be;
Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open \"%s\": ERR=%s.\n"), ff_pkt->fname,
be.bstrerror());
jcr->JobErrors++;
if (tid) {
stop_thread_timer(tid);
tid = NULL;
}
goto good_rtn;
}
if (tid) {
stop_thread_timer(tid);
tid = NULL;
}
stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
if (ff_pkt->flags & FO_CHKCHANGES) {
has_file_changed(jcr, ff_pkt);
}
bclose(&ff_pkt->bfd);
if (!stat) {
goto bail_out;
}
}
#ifdef HAVE_DARWIN_OS
/* Regular files can have resource forks and Finder Info */
if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
ff_pkt->flags & FO_HFSPLUS)) {
if (ff_pkt->hfsinfo.rsrclength > 0) {
int flags;
int rsrc_stream;
if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
ff_pkt->ff_errno = errno;
berrno be;
Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for \"%s\": ERR=%s.\n"),
ff_pkt->fname, be.bstrerror());
jcr->JobErrors++;
if (is_bopen(&ff_pkt->bfd)) {
bclose(&ff_pkt->bfd);
}
goto good_rtn;
}
flags = ff_pkt->flags;
ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
if (flags & FO_ENCRYPT) {
rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
} else {
rsrc_stream = STREAM_MACOS_FORK_DATA;
}
stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
ff_pkt->flags = flags;
bclose(&ff_pkt->bfd);
if (!stat) {
goto bail_out;
}
}
Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
pm_memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
sd->msglen = 32;
if (digest) {
crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
}
if (signing_digest) {
crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
}
sd->send();
sd->signal(BNET_EOD);
}
#endif
/*
* Save ACLs for anything not being a symlink and not being a plugin.
*/
if (!ff_pkt->cmd_plugin) {
if (ff_pkt->flags & FO_ACL && ff_pkt->type != FT_LNK) {
if (!build_acl_streams(jcr, ff_pkt))
goto bail_out;
}
}
/*
* Save Extended Attributes for all files not being a plugin.
*/
if (!ff_pkt->cmd_plugin) {
if (ff_pkt->flags & FO_XATTR) {
if (!build_xattr_streams(jcr, ff_pkt))
goto bail_out;
}
}
/* Terminate the signing digest and send it to the Storage daemon */
if (signing_digest) {
uint32_t size = 0;
if ((sig = crypto_sign_new(jcr)) == NULL) {
Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
goto bail_out;
}
if (!crypto_sign_add_signer(sig, signing_digest, jcr->crypto.pki_keypair)) {
Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
goto bail_out;
}
/* Get signature size */
if (!crypto_sign_encode(sig, NULL, &size)) {
Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
goto bail_out;
}
/* Grow the bsock buffer to fit our message if necessary */
if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
sd->msg = realloc_pool_memory(sd->msg, size);
}
/* Send our header */
sd->fsend("%ld %ld 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
/* Encode signature data */
if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) {
Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
goto bail_out;
}
sd->msglen = size;
sd->send();
sd->signal(BNET_EOD); /* end of checksum */
}
/* Terminate any digest and send it to Storage daemon */
if (digest) {
uint32_t size;
sd->fsend("%ld %d 0", jcr->JobFiles, digest_stream);
Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
size = CRYPTO_DIGEST_MAX_SIZE;
/* Grow the bsock buffer to fit our message if necessary */
if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
sd->msg = realloc_pool_memory(sd->msg, size);
}
if (!crypto_digest_finalize(digest, (uint8_t *)sd->msg, &size)) {
Jmsg(jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
goto bail_out;
}
sd->msglen = size;
sd->send();
sd->signal(BNET_EOD); /* end of checksum */
}
if (ff_pkt->cmd_plugin) {
send_plugin_name(jcr, sd, false); /* signal end of plugin data */
}
good_rtn:
rtnstat = 1; /* good return */
bail_out:
if (digest) {
crypto_digest_free(digest);
}
if (signing_digest) {
crypto_digest_free(signing_digest);
}
if (sig) {
crypto_sign_free(sig);
}
return rtnstat;
}
/*
* Verify attributes of the requested files on the Volume
*
*/
void do_verify_volume(JCR *jcr)
{
BSOCK *sd, *dir;
POOLMEM *fname; /* original file name */
POOLMEM *lname; /* link name */
int32_t stream;
uint32_t size;
uint32_t VolSessionId, VolSessionTime, file_index;
uint32_t record_file_index;
char digest[BASE64_SIZE(CRYPTO_DIGEST_MAX_SIZE)];
int type, stat;
sd = jcr->store_bsock;
if (!sd) {
Jmsg(jcr, M_FATAL, 0, _("Storage command not issued before Verify.\n"));
jcr->setJobStatus(JS_FatalError);
return;
}
dir = jcr->dir_bsock;
jcr->setJobStatus(JS_Running);
LockRes();
CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
UnlockRes();
uint32_t buf_size;
if (client) {
buf_size = client->max_network_buffer_size;
} else {
buf_size = 0; /* use default */
}
if (!sd->set_buffer_size(buf_size, BNET_SETBUF_WRITE)) {
jcr->setJobStatus(JS_FatalError);
return;
}
jcr->buf_size = sd->msglen;
fname = get_pool_memory(PM_FNAME);
lname = get_pool_memory(PM_FNAME);
/*
* Get a record from the Storage daemon
*/
while (bget_msg(sd) >= 0 && !job_canceled(jcr)) {
/*
* First we expect a Stream Record Header
*/
if (sscanf(sd->msg, rec_header, &VolSessionId, &VolSessionTime, &file_index,
&stream, &size) != 5) {
Jmsg1(jcr, M_FATAL, 0, _("Record header scan error: %s\n"), sd->msg);
goto bail_out;
}
Dmsg3(30, "Got hdr: FilInx=%d Stream=%d size=%d.\n", file_index, stream, size);
/*
* Now we expect the Stream Data
*/
if (bget_msg(sd) < 0) {
Jmsg1(jcr, M_FATAL, 0, _("Data record error. ERR=%s\n"), sd->bstrerror());
goto bail_out;
}
if (size != ((uint32_t)sd->msglen)) {
Jmsg2(jcr, M_FATAL, 0, _("Actual data size %d not same as header %d\n"), sd->msglen, size);
goto bail_out;
}
Dmsg2(30, "Got stream data %s, len=%d\n", stream_to_ascii(stream), sd->msglen);
/* File Attributes stream */
switch (stream) {
case STREAM_UNIX_ATTRIBUTES:
case STREAM_UNIX_ATTRIBUTES_EX:
char *ap, *lp, *fp;
Dmsg0(400, "Stream=Unix Attributes.\n");
if ((int)sizeof_pool_memory(fname) < sd->msglen) {
fname = realloc_pool_memory(fname, sd->msglen + 1);
}
if ((int)sizeof_pool_memory(lname) < sd->msglen) {
lname = realloc_pool_memory(lname, sd->msglen + 1);
}
*fname = 0;
*lname = 0;
/*
* An Attributes record consists of:
* File_index
* Type (FT_types)
* Filename
* Attributes
* Link name (if file linked i.e. FT_LNK)
* Extended Attributes (if Win32)
*/
if (sscanf(sd->msg, "%d %d", &record_file_index, &type) != 2) {
Jmsg(jcr, M_FATAL, 0, _("Error scanning record header: %s\n"), sd->msg);
Dmsg0(0, "\nError scanning header\n");
goto bail_out;
}
Dmsg2(30, "Got Attr: FilInx=%d type=%d\n", record_file_index, type);
ap = sd->msg;
while (*ap++ != ' ') /* skip record file index */
;
while (*ap++ != ' ') /* skip type */
;
/* Save filename and position to attributes */
fp = fname;
while (*ap != 0) {
*fp++ = *ap++; /* copy filename to fname */
}
*fp = *ap++; /* terminate filename & point to attribs */
Dmsg2(100, "File=%s Attr=%s\n", fname, ap);
/* Skip to Link name */
if (type == FT_LNK || type == FT_LNKSAVED) {
lp = ap;
while (*lp++ != 0) {
;
}
pm_strcat(lname, lp); /* "save" link name */
} else {
*lname = 0;
}
jcr->lock();
jcr->JobFiles++;
jcr->num_files_examined++;
pm_strcpy(jcr->last_fname, fname); /* last file examined */
jcr->unlock();
/*
* Send file attributes to Director
* File_index
* Stream
* Verify Options
* Filename (full path)
* Encoded attributes
* Link name (if type==FT_LNK)
* For a directory, link is the same as fname, but with trailing
* slash. For a linked file, link is the link.
*/
/* Send file attributes to Director */
Dmsg2(200, "send ATTR inx=%d fname=%s\n", jcr->JobFiles, fname);
if (type == FT_LNK || type == FT_LNKSAVED) {
stat = dir->fsend("%d %d %s %s%c%s%c%s%c", jcr->JobFiles,
STREAM_UNIX_ATTRIBUTES, "pinsug5", fname,
0, ap, 0, lname, 0);
/* for a deleted record, we set fileindex=0 */
} else if (type == FT_DELETED) {
stat = dir->fsend("%d %d %s %s%c%s%c%c", 0,
STREAM_UNIX_ATTRIBUTES, "pinsug5", fname,
0, ap, 0, 0);
} else {
stat = dir->fsend("%d %d %s %s%c%s%c%c", jcr->JobFiles,
STREAM_UNIX_ATTRIBUTES, "pinsug5", fname,
0, ap, 0, 0);
}
Dmsg2(200, "bfiled>bdird: attribs len=%d: msg=%s\n", dir->msglen, dir->msg);
if (!stat) {
Jmsg(jcr, M_FATAL, 0, _("Network error in send to Director: ERR=%s\n"), dir->bstrerror());
goto bail_out;
}
break;
case STREAM_MD5_DIGEST:
bin_to_base64(digest, sizeof(digest), (char *)sd->msg, CRYPTO_DIGEST_MD5_SIZE, true);
Dmsg2(400, "send inx=%d MD5=%s\n", jcr->JobFiles, digest);
dir->fsend("%d %d %s *MD5-%d*", jcr->JobFiles, STREAM_MD5_DIGEST, digest,
jcr->JobFiles);
Dmsg2(20, "bfiled>bdird: MD5 len=%d: msg=%s\n", dir->msglen, dir->msg);
break;
case STREAM_SHA1_DIGEST:
bin_to_base64(digest, sizeof(digest), (char *)sd->msg, CRYPTO_DIGEST_SHA1_SIZE, true);
Dmsg2(400, "send inx=%d SHA1=%s\n", jcr->JobFiles, digest);
dir->fsend("%d %d %s *SHA1-%d*", jcr->JobFiles, STREAM_SHA1_DIGEST,
digest, jcr->JobFiles);
Dmsg2(20, "bfiled>bdird: SHA1 len=%d: msg=%s\n", dir->msglen, dir->msg);
break;
case STREAM_SHA256_DIGEST:
bin_to_base64(digest, sizeof(digest), (char *)sd->msg, CRYPTO_DIGEST_SHA256_SIZE, true);
Dmsg2(400, "send inx=%d SHA256=%s\n", jcr->JobFiles, digest);
dir->fsend("%d %d %s *SHA256-%d*", jcr->JobFiles, STREAM_SHA256_DIGEST,
digest, jcr->JobFiles);
Dmsg2(20, "bfiled>bdird: SHA256 len=%d: msg=%s\n", dir->msglen, dir->msg);
break;
case STREAM_SHA512_DIGEST:
bin_to_base64(digest, sizeof(digest), (char *)sd->msg, CRYPTO_DIGEST_SHA512_SIZE, true);
Dmsg2(400, "send inx=%d SHA512=%s\n", jcr->JobFiles, digest);
dir->fsend("%d %d %s *SHA512-%d*", jcr->JobFiles, STREAM_SHA512_DIGEST,
digest, jcr->JobFiles);
Dmsg2(20, "bfiled>bdird: SHA512 len=%d: msg=%s\n", dir->msglen, dir->msg);
break;
/*
* Restore stream object is counted, but not restored here
*/
case STREAM_RESTORE_OBJECT:
jcr->lock();
jcr->JobFiles++;
jcr->num_files_examined++;
jcr->unlock();
break;
/* Ignore everything else */
default:
break;
} /* end switch */
} /* end while bnet_get */
jcr->setJobStatus(JS_Terminated);
goto ok_out;
bail_out:
jcr->setJobStatus(JS_ErrorTerminated);
ok_out:
if (jcr->compress_buf) {
free(jcr->compress_buf);
jcr->compress_buf = NULL;
}
free_pool_memory(fname);
free_pool_memory(lname);
Dmsg2(050, "End Verify-Vol. Files=%d Bytes=%" lld "\n", jcr->JobFiles,
jcr->JobBytes);
}
/* List just block information */
static void do_blocks(char *infname)
{
DEV_BLOCK *block = dcr->block;
char buf1[100], buf2[100];
for ( ;; ) {
if (!dcr->read_block_from_device(NO_BLOCK_NUMBER_CHECK)) {
Dmsg1(100, "!read_block(): ERR=%s\n", dev->bstrerror());
if (dev->at_eot()) {
if (!mount_next_read_volume(dcr)) {
Jmsg(jcr, M_INFO, 0, _("Got EOM at file %u on device %s, Volume \"%s\"\n"),
dev->file, dev->print_name(), dcr->VolumeName);
break;
}
/* Read and discard Volume label */
DEV_RECORD *record;
record = new_record();
dcr->read_block_from_device(NO_BLOCK_NUMBER_CHECK);
read_record_from_block(dcr, record);
get_session_record(dev, record, &sessrec);
free_record(record);
Jmsg(jcr, M_INFO, 0, _("Mounted Volume \"%s\".\n"), dcr->VolumeName);
} else if (dev->at_eof()) {
Jmsg(jcr, M_INFO, 0, _("End of file %u on device %s, Volume \"%s\"\n"),
dev->file, dev->print_name(), dcr->VolumeName);
Dmsg0(20, "read_record got eof. try again\n");
continue;
} else if (dev->is_short_block()) {
Jmsg(jcr, M_INFO, 0, "%s", dev->errmsg);
continue;
} else {
/* I/O error */
display_tape_error_status(jcr, dev);
break;
}
}
if (!match_bsr_block(bsr, block)) {
Dmsg5(100, "reject Blk=%u blen=%u bVer=%d SessId=%u SessTim=%u\n",
block->BlockNumber, block->block_len, block->BlockVer,
block->VolSessionId, block->VolSessionTime);
continue;
}
Dmsg5(100, "Blk=%u blen=%u bVer=%d SessId=%u SessTim=%u\n",
block->BlockNumber, block->block_len, block->BlockVer,
block->VolSessionId, block->VolSessionTime);
if (verbose == 1) {
read_record_from_block(dcr, rec);
Pmsg9(-1, _("File:blk=%u:%u blk_num=%u blen=%u First rec FI=%s SessId=%u SessTim=%u Strm=%s rlen=%d\n"),
dev->file, dev->block_num,
block->BlockNumber, block->block_len,
FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId, rec->VolSessionTime,
stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len);
rec->remainder = 0;
} else if (verbose > 1) {
dump_block(block, "");
} else {
printf(_("Block: %d size=%d\n"), block->BlockNumber, block->block_len);
}
}
return;
}
/*
* Read the header record
*/
static bool read_header(DCR *dcr, DEV_BLOCK *block, DEV_RECORD *rec)
{
ser_declare;
uint32_t VolSessionId;
uint32_t VolSessionTime;
int32_t FileIndex;
int32_t Stream;
uint32_t rhl;
char buf1[100], buf2[100];
Dmsg0(dbgep, "=== rpath 1 read_header\n");
/* Clear state flags */
rec->state_bits = 0;
if (block->dev->is_tape()) {
rec->state_bits |= REC_ISTAPE;
}
rec->Block = ((DEVICE *)block->dev)->EndBlock;
rec->File = ((DEVICE *)block->dev)->EndFile;
/*
* Get the header. There is always a full header,
* otherwise we find it in the next block.
*/
Dmsg3(read_dbglvl, "Block=%d Ver=%d block_len=%u\n",
block->BlockNumber, block->BlockVer, block->block_len);
if (block->BlockVer == 1) {
rhl = RECHDR1_LENGTH;
} else {
rhl = RECHDR2_LENGTH;
}
if (rec->remlen >= rhl) {
Dmsg0(dbgep, "=== rpath 2 begin unserial header\n");
Dmsg4(read_dbglvl, "read_header: remlen=%d data_len=%d rem=%d blkver=%d\n",
rec->remlen, rec->data_len, rec->remainder, block->BlockVer);
unser_begin(block->bufp, WRITE_RECHDR_LENGTH);
if (block->BlockVer == 1) {
unser_uint32(VolSessionId);
unser_uint32(VolSessionTime);
} else {
VolSessionId = block->VolSessionId;
VolSessionTime = block->VolSessionTime;
}
unser_int32(FileIndex);
unser_int32(Stream);
unser_uint32(rec->data_bytes);
block->bufp += rhl;
block->binbuf -= rhl;
rec->remlen -= rhl;
/* If we are looking for more (remainder!=0), we reject anything
* where the VolSessionId and VolSessionTime don't agree
*/
if (rec->remainder && (rec->VolSessionId != VolSessionId ||
rec->VolSessionTime != VolSessionTime)) {
rec->state_bits |= REC_NO_MATCH;
Dmsg0(read_dbglvl, "remainder and VolSession doesn't match\n");
Dmsg0(dbgep, "=== rpath 4 VolSession no match\n");
return false; /* This is from some other Session */
}
/* if Stream is negative, it means that this is a continuation
* of a previous partially written record.
*/
if (Stream < 0) { /* continuation record? */
Dmsg0(dbgep, "=== rpath 5 negative stream\n");
Dmsg1(read_dbglvl, "Got negative Stream => continuation. remainder=%d\n",
rec->remainder);
rec->state_bits |= REC_CONTINUATION;
if (!rec->remainder) { /* if we didn't read previously */
Dmsg0(dbgep, "=== rpath 6 no remainder\n");
rec->data_len = 0; /* return data as if no continuation */
} else if (rec->Stream != -Stream) {
Dmsg0(dbgep, "=== rpath 7 wrong cont stream\n");
rec->state_bits |= REC_NO_MATCH;
return false; /* This is from some other Session */
}
rec->Stream = -Stream; /* set correct Stream */
rec->maskedStream = rec->Stream & STREAMMASK_TYPE;
} else { /* Regular record */
Dmsg0(dbgep, "=== rpath 8 normal stream\n");
rec->Stream = Stream;
rec->maskedStream = rec->Stream & STREAMMASK_TYPE;
rec->data_len = 0; /* transfer to beginning of data */
}
rec->VolSessionId = VolSessionId;
rec->VolSessionTime = VolSessionTime;
rec->FileIndex = FileIndex;
if (FileIndex > 0) {
Dmsg0(dbgep, "=== rpath 9 FileIndex>0\n");
if (block->FirstIndex == 0) {
Dmsg0(dbgep, "=== rpath 10 FirstIndex\n");
block->FirstIndex = FileIndex;
}
block->LastIndex = rec->FileIndex;
}
Dmsg6(read_dbglvl, "read_header: FI=%s SessId=%d Strm=%s len=%u rec->remlen=%d data_len=%d\n",
FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId,
stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_bytes, rec->remlen,
rec->data_len);
} else {
Dmsg0(dbgep, "=== rpath 11a block out of records\n");
/*
* No more records in this block because the number
* of remaining bytes are less than a record header
* length, so return empty handed, but indicate that
* he must read again. By returning, we allow the
* higher level routine to fetch the next block and
* then reread.
*/
Dmsg0(read_dbglvl, "read_header: End of block\n");
rec->state_bits |= (REC_NO_HEADER | REC_BLOCK_EMPTY);
empty_block(block); /* mark block empty */
return false;
}
/* Sanity check */
if (rec->data_bytes >= MAX_BLOCK_LENGTH) {
Dmsg0(dbgep, "=== rpath 11b maxlen too big\n");
/*
* Something is wrong, force read of next block, abort
* continuing with this block.
*/
rec->state_bits |= (REC_NO_HEADER | REC_BLOCK_EMPTY);
empty_block(block);
Jmsg2(dcr->jcr, M_WARNING, 0, _("Sanity check failed. maxlen=%d datalen=%d. Block discarded.\n"),
MAX_BLOCK_LENGTH, rec->data_bytes);
return false;
}
rec->data = check_pool_memory_size(rec->data, rec->data_len+rec->data_bytes);
rec->rstate = st_data;
return true;
}
/* Write session label
* Returns: false on failure
* true on success
*/
bool write_session_label(DCR *dcr, int label)
{
JCR *jcr = dcr->jcr;
DEVICE *dev = dcr->dev;
DEV_RECORD *rec;
DEV_BLOCK *block = dcr->block;
char buf1[100], buf2[100];
rec = new_record();
Dmsg1(130, "session_label record=%x\n", rec);
switch (label) {
case SOS_LABEL:
set_start_vol_position(dcr);
break;
case EOS_LABEL:
if (dev->is_tape()) {
dcr->EndBlock = dev->EndBlock;
dcr->EndFile = dev->EndFile;
} else {
dcr->EndBlock = (uint32_t)dev->file_addr;
dcr->EndFile = (uint32_t)(dev->file_addr >> 32);
}
break;
default:
Jmsg1(jcr, M_ABORT, 0, _("Bad Volume session label = %d\n"), label);
break;
}
create_session_label(dcr, rec, label);
rec->FileIndex = label;
/*
* We guarantee that the session record can totally fit
* into a block. If not, write the block, and put it in
* the next block. Having the sesssion record totally in
* one block makes reading them much easier (no need to
* read the next block).
*/
if (!can_write_record_to_block(block, rec)) {
Dmsg0(150, "Cannot write session label to block.\n");
if (!dcr->write_block_to_device()) {
Dmsg0(130, "Got session label write_block_to_dev error.\n");
free_record(rec);
return false;
}
}
if (!write_record_to_block(dcr, rec)) {
free_record(rec);
return false;
}
Dmsg6(150, "Write sesson_label record JobId=%d FI=%s SessId=%d Strm=%s len=%d "
"remainder=%d\n", jcr->JobId,
FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId,
stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len,
rec->remainder);
free_record(rec);
Dmsg2(150, "Leave write_session_label Block=%ud File=%ud\n",
dev->get_block_num(), dev->get_file());
return true;
}
/*
* Write a Record to the block
*
* Returns: false on failure (none or partially written)
* true on success (all bytes written)
*
* and remainder returned in packet.
*
* We require enough room for the header, and we deal with
* two special cases. 1. Only part of the record may have
* been transferred the last time (when remainder is
* non-zero), and 2. The remaining bytes to write may not
* all fit into the block.
*/
bool write_record_to_block(DCR *dcr, DEV_RECORD *rec)
{
ssize_t n;
bool retval = false;
char buf1[100], buf2[100];
DEV_BLOCK *block = dcr->block;
/*
* After this point the record is in nrec not rec e.g. its either converted
* or is just a pointer to the same as the rec pointer being passed in.
*/
while (1) {
ASSERT(block->binbuf == (uint32_t)(block->bufp - block->buf));
ASSERT(block->buf_len >= block->binbuf);
Dmsg9(890, "%s() state=%d (%s) FI=%s SessId=%d Strm=%s len=%d "
"block_navail=%d remainder=%d\n",
__func__, rec->state, record_state_to_ascii(rec->state),
FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId,
stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len,
block_write_navail(block), rec->remainder);
switch (rec->state) {
case st_none:
/*
* Figure out what to do
*/
rec->state = st_header;
rec->remainder = rec->data_len; /* length of data remaining to write */
continue; /* goto st_header */
case st_header:
/*
* Write header
*/
n = write_header_to_block(block, rec, rec->Stream);
if (n < 0) {
/*
* the header did not fit into the block, so flush the current
* block and come back to st_header and try again on the next block.
*/
goto bail_out;
}
if (block_write_navail(block) == 0) {
/*
* The header fit, but no bytes of data will fit,
* so flush this block and start the next block with a
* continuation header.
*/
rec->state = st_header_cont;
goto bail_out;
}
/*
* The header fit, and at least one byte of data will fit,
* so move to the st_data state and start filling the block
* with data bytes
*/
rec->state = st_data;
continue;
case st_header_cont:
/*
* Write continuation header
*/
n = write_header_to_block(block, rec, -rec->Stream);
if (n < 0) {
/*
* The continuation header wouldn't fit, which is impossible
* unless something is broken
*/
Emsg0(M_ABORT, 0, _("couldn't write continuation header\n"));
}
/*
* After successfully writing a continuation header, we always start writing
* data, even if none will fit into this block.
*/
rec->state = st_data;
if (block_write_navail(block) == 0) {
/*
* The header fit, but no bytes of data will fit,
* so flush the block and start the next block with
* data bytes
*/
goto bail_out; /* Partial transfer */
}
continue;
case st_data:
/*
* Write normal data
*
* Part of it may have already been transferred, and we
* may not have enough room to transfer the whole this time.
*/
if (rec->remainder > 0) {
n = write_data_to_block(block, rec);
if (n < 0) {
/*
* error appending data to block should be impossible
* unless something is broken
*/
Emsg0(M_ABORT, 0, _("data write error\n"));
}
rec->remainder -= n;
if (rec->remainder > 0) {
/*
* Could not fit all of the data bytes into this block, so
* flush the current block, and start the next block with a
* continuation header
*/
rec->state = st_header_cont;
goto bail_out;
}
}
rec->remainder = 0; /* did whole transfer */
rec->state = st_none;
retval = true;
goto bail_out;
default:
Emsg1(M_ABORT, 0, _("Something went wrong. Unknown state %d.\n"), rec->state);
rec->state = st_none;
retval = true;
goto bail_out;
}
}
bail_out:
return retval;
}
/*
* Read a Record from the block
*
* Returns: false if nothing read or if the continuation record does not match.
* In both of these cases, a block read must be done.
* true if at least the record header was read, this
* routine may have to be called again with a new
* block if the entire record was not read.
*/
bool read_record_from_block(DCR *dcr, DEV_RECORD *rec)
{
ser_declare;
uint32_t remlen;
uint32_t VolSessionId;
uint32_t VolSessionTime;
int32_t FileIndex;
int32_t Stream;
uint32_t data_bytes;
uint32_t rhl;
char buf1[100], buf2[100];
remlen = dcr->block->binbuf;
/*
* Clear state flags
*/
clear_all_bits(REC_STATE_MAX, rec->state_bits);
if (dcr->block->dev->is_tape()) {
set_bit(REC_ISTAPE, rec->state_bits);
}
rec->Block = ((DEVICE *)(dcr->block->dev))->EndBlock;
rec->File = ((DEVICE *)(dcr->block->dev))->EndFile;
/*
* Get the header. There is always a full header, otherwise we find it in the next block.
*/
Dmsg3(450, "Block=%d Ver=%d size=%u\n", dcr->block->BlockNumber, dcr->block->BlockVer,
dcr->block->block_len);
if (dcr->block->BlockVer == 1) {
rhl = RECHDR1_LENGTH;
} else {
rhl = RECHDR2_LENGTH;
}
if (remlen >= rhl) {
Dmsg4(450, "Enter read_record_block: remlen=%d data_len=%d rem=%d blkver=%d\n",
remlen, rec->data_len, rec->remainder, dcr->block->BlockVer);
unser_begin(dcr->block->bufp, WRITE_RECHDR_LENGTH);
if (dcr->block->BlockVer == 1) {
unser_uint32(VolSessionId);
unser_uint32(VolSessionTime);
} else {
VolSessionId = dcr->block->VolSessionId;
VolSessionTime = dcr->block->VolSessionTime;
}
unser_int32(FileIndex);
unser_int32(Stream);
unser_uint32(data_bytes);
dcr->block->bufp += rhl;
dcr->block->binbuf -= rhl;
remlen -= rhl;
/*
* If we are looking for more (remainder!=0), we reject anything
* where the VolSessionId and VolSessionTime don't agree
*/
if (rec->remainder && (rec->VolSessionId != VolSessionId ||
rec->VolSessionTime != VolSessionTime)) {
set_bit(REC_NO_MATCH, rec->state_bits);
Dmsg0(450, "remainder and VolSession doesn't match\n");
return false; /* This is from some other Session */
}
/*
* If Stream is negative, it means that this is a continuation
* of a previous partially written record.
*/
if (Stream < 0) { /* continuation record? */
Dmsg1(500, "Got negative Stream => continuation. remainder=%d\n", rec->remainder);
set_bit(REC_CONTINUATION, rec->state_bits);
if (!rec->remainder) { /* if we didn't read previously */
rec->data_len = 0; /* return data as if no continuation */
} else if (rec->Stream != -Stream) {
set_bit(REC_NO_MATCH, rec->state_bits);
return false; /* This is from some other Session */
}
rec->Stream = -Stream; /* set correct Stream */
rec->maskedStream = rec->Stream & STREAMMASK_TYPE;
} else { /* Regular record */
rec->Stream = Stream;
rec->maskedStream = rec->Stream & STREAMMASK_TYPE;
rec->data_len = 0; /* transfer to beginning of data */
}
rec->VolSessionId = VolSessionId;
rec->VolSessionTime = VolSessionTime;
rec->FileIndex = FileIndex;
if (FileIndex > 0) {
if (dcr->block->FirstIndex == 0) {
dcr->block->FirstIndex = FileIndex;
}
dcr->block->LastIndex = FileIndex;
}
Dmsg6(450, "rd_rec_blk() got FI=%s SessId=%d Strm=%s len=%u\n"
"remlen=%d data_len=%d\n",
FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId,
stream_to_ascii(buf2, rec->Stream, rec->FileIndex), data_bytes, remlen,
rec->data_len);
} else {
/*
* No more records in this block because the number
* of remaining bytes are less than a record header
* length, so return empty handed, but indicate that
* he must read again. By returning, we allow the
* higher level routine to fetch the next block and
* then reread.
*/
Dmsg0(450, "read_record_block: nothing\n");
set_bit(REC_NO_HEADER, rec->state_bits);
set_bit(REC_BLOCK_EMPTY, rec->state_bits);
empty_block(dcr->block); /* mark block empty */
return false;
}
/* Sanity check */
if (data_bytes >= MAX_BLOCK_LENGTH) {
/*
* Something is wrong, force read of next block, abort
* continuing with this block.
*/
set_bit(REC_NO_HEADER, rec->state_bits);
set_bit(REC_BLOCK_EMPTY, rec->state_bits);
empty_block(dcr->block);
Jmsg2(dcr->jcr, M_WARNING, 0, _("Sanity check failed. maxlen=%d datalen=%d. Block discarded.\n"),
MAX_BLOCK_LENGTH, data_bytes);
return false;
}
rec->data = check_pool_memory_size(rec->data, rec->data_len + data_bytes);
/*
* At this point, we have read the header, now we
* must transfer as much of the data record as
* possible taking into account: 1. A partial
* data record may have previously been transferred,
* 2. The current block may not contain the whole data
* record.
*/
if (remlen >= data_bytes) {
/*
* Got whole record
*/
memcpy(rec->data+rec->data_len, dcr->block->bufp, data_bytes);
dcr->block->bufp += data_bytes;
dcr->block->binbuf -= data_bytes;
rec->data_len += data_bytes;
} else {
/*
* Partial record
*/
memcpy(rec->data+rec->data_len, dcr->block->bufp, remlen);
dcr->block->bufp += remlen;
dcr->block->binbuf -= remlen;
rec->data_len += remlen;
rec->remainder = 1; /* partial record transferred */
Dmsg1(450, "read_record_block: partial xfered=%d\n", rec->data_len);
set_bit(REC_PARTIAL_RECORD, rec->state_bits);
set_bit(REC_BLOCK_EMPTY, rec->state_bits);
return true;
}
rec->remainder = 0;
Dmsg4(450, "Rtn full rd_rec_blk FI=%s SessId=%d Strm=%s len=%d\n",
FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId,
stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len);
return true; /* transferred full record */
}
/*
* Called here for each record from read_records()
* This function is used when we do a internal clone of a Job e.g.
* this SD is both the reading and writing SD.
*
* Returns: true if OK
* false if error
*/
static bool clone_record_internally(DCR *dcr, DEV_RECORD *rec)
{
JCR *jcr = dcr->jcr;
DEVICE *dev = jcr->dcr->dev;
char buf1[100], buf2[100];
#ifdef xxx
Dmsg5(000, "on entry JobId=%d FI=%s SessId=%d Strm=%s len=%d\n",
jcr->JobId, FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId,
stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len);
#endif
/*
* If label and not for us, discard it
*/
if (rec->FileIndex < 0 && rec->match_stat <= 0) {
return true;
}
/*
* We want to write SOS_LABEL and EOS_LABEL discard all others
*/
switch (rec->FileIndex) {
case PRE_LABEL:
case VOL_LABEL:
case EOT_LABEL:
case EOM_LABEL:
return true; /* don't write vol labels */
}
// if (jcr->is_JobType(JT_BACKUP)) {
/*
* For normal migration jobs, FileIndex values are sequential because
* we are dealing with one job. However, for Vbackup (consolidation),
* we will be getting records from multiple jobs and writing them back
* out, so we need to ensure that the output FileIndex is sequential.
* We do so by detecting a FileIndex change and incrementing the
* JobFiles, which we then use as the output FileIndex.
*/
if (rec->FileIndex >= 0) {
/*
* If something changed, increment FileIndex
*/
if (rec->VolSessionId != rec->last_VolSessionId ||
rec->VolSessionTime != rec->last_VolSessionTime ||
rec->FileIndex != rec->last_FileIndex) {
jcr->JobFiles++;
rec->last_VolSessionId = rec->VolSessionId;
rec->last_VolSessionTime = rec->VolSessionTime;
rec->last_FileIndex = rec->FileIndex;
}
rec->FileIndex = jcr->JobFiles; /* set sequential output FileIndex */
}
// }
/*
* Modify record SessionId and SessionTime to correspond to output.
*/
rec->VolSessionId = jcr->VolSessionId;
rec->VolSessionTime = jcr->VolSessionTime;
Dmsg5(200, "before write JobId=%d FI=%s SessId=%d Strm=%s len=%d\n",
jcr->JobId, FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId,
stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len);
while (!write_record_to_block(jcr->dcr, rec)) {
Dmsg4(200, "!write_record_to_block blkpos=%u:%u len=%d rem=%d\n",
dev->file, dev->block_num, rec->data_len, rec->remainder);
if (!jcr->dcr->write_block_to_device()) {
Dmsg2(90, "Got write_block_to_dev error on device %s. %s\n",
dev->print_name(), dev->bstrerror());
Jmsg2(jcr, M_FATAL, 0, _("Fatal append error on device %s: ERR=%s\n"),
dev->print_name(), dev->bstrerror());
return false;
}
Dmsg2(200, "===== Wrote block new pos %u:%u\n", dev->file, dev->block_num);
}
/*
* Restore packet
*/
rec->VolSessionId = rec->last_VolSessionId;
rec->VolSessionTime = rec->last_VolSessionTime;
if (rec->FileIndex < 0) {
return true; /* don't send LABELs to Dir */
}
jcr->JobBytes += rec->data_len; /* increment bytes of this job */
Dmsg5(500, "wrote_record JobId=%d FI=%s SessId=%d Strm=%s len=%d\n",
jcr->JobId, FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId,
stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len);
send_attrs_to_dir(jcr, rec);
return true;
}
/*
* Called here by find() for each file.
*
* Find the file, compute the MD5 or SHA1 and send it back to the Director
*/
static int verify_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
{
POOL_MEM attribs(PM_NAME),
attribsEx(PM_NAME);
int digest_stream = STREAM_NONE;
int status;
DIGEST *digest = NULL;
BSOCK *dir;
if (job_canceled(jcr)) {
return 0;
}
dir = jcr->dir_bsock;
jcr->num_files_examined++; /* bump total file count */
switch (ff_pkt->type) {
case FT_LNKSAVED: /* Hard linked, file already saved */
Dmsg2(30, "FT_LNKSAVED saving: %s => %s\n", ff_pkt->fname, ff_pkt->link);
break;
case FT_REGE:
Dmsg1(30, "FT_REGE saving: %s\n", ff_pkt->fname);
break;
case FT_REG:
Dmsg1(30, "FT_REG saving: %s\n", ff_pkt->fname);
break;
case FT_LNK:
Dmsg2(30, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
break;
case FT_DIRBEGIN:
jcr->num_files_examined--; /* correct file count */
return 1; /* ignored */
case FT_REPARSE:
case FT_JUNCTION:
case FT_DIREND:
Dmsg1(30, "FT_DIR saving: %s\n", ff_pkt->fname);
break;
case FT_SPEC:
Dmsg1(30, "FT_SPEC saving: %s\n", ff_pkt->fname);
break;
case FT_RAW:
Dmsg1(30, "FT_RAW saving: %s\n", ff_pkt->fname);
break;
case FT_FIFO:
Dmsg1(30, "FT_FIFO saving: %s\n", ff_pkt->fname);
break;
case FT_NOACCESS: {
berrno be;
be.set_errno(ff_pkt->ff_errno);
Jmsg(jcr, M_NOTSAVED, 1, _(" Could not access %s: ERR=%s\n"), ff_pkt->fname, be.bstrerror());
jcr->JobErrors++;
return 1;
}
case FT_NOFOLLOW: {
berrno be;
be.set_errno(ff_pkt->ff_errno);
Jmsg(jcr, M_NOTSAVED, 1, _(" Could not follow link %s: ERR=%s\n"), ff_pkt->fname, be.bstrerror());
jcr->JobErrors++;
return 1;
}
case FT_NOSTAT: {
berrno be;
be.set_errno(ff_pkt->ff_errno);
Jmsg(jcr, M_NOTSAVED, 1, _(" Could not stat %s: ERR=%s\n"), ff_pkt->fname, be.bstrerror());
jcr->JobErrors++;
return 1;
}
case FT_DIRNOCHG:
case FT_NOCHG:
Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
return 1;
case FT_ISARCH:
Jmsg(jcr, M_SKIPPED, 1, _(" Archive file skipped: %s\n"), ff_pkt->fname);
return 1;
case FT_NORECURSE:
Jmsg(jcr, M_SKIPPED, 1, _(" Recursion turned off. Directory skipped: %s\n"), ff_pkt->fname);
ff_pkt->type = FT_DIREND; /* directory entry was backed up */
break;
case FT_NOFSCHG:
Jmsg(jcr, M_SKIPPED, 1, _(" File system change prohibited. Directory skipped: %s\n"), ff_pkt->fname);
return 1;
case FT_PLUGIN_CONFIG:
case FT_RESTORE_FIRST:
return 1; /* silently skip */
case FT_NOOPEN: {
berrno be;
be.set_errno(ff_pkt->ff_errno);
Jmsg(jcr, M_NOTSAVED, 1, _(" Could not open directory %s: ERR=%s\n"), ff_pkt->fname, be.bstrerror());
jcr->JobErrors++;
return 1;
}
default:
Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d: %s\n"), ff_pkt->type, ff_pkt->fname);
jcr->JobErrors++;
return 1;
}
/* Encode attributes and possibly extend them */
encode_stat(attribs.c_str(), &ff_pkt->statp, sizeof(ff_pkt->statp), ff_pkt->LinkFI, 0);
encode_attribsEx(jcr, attribsEx.c_str(), ff_pkt);
jcr->lock();
jcr->JobFiles++; /* increment number of files sent */
pm_strcpy(jcr->last_fname, ff_pkt->fname);
jcr->unlock();
/*
* Send file attributes to Director
* File_index
* Stream
* Verify Options
* Filename (full path)
* Encoded attributes
* Link name (if type==FT_LNK)
* For a directory, link is the same as fname, but with trailing
* slash. For a linked file, link is the link.
*/
/* Send file attributes to Director (note different format than for Storage) */
Dmsg2(400, "send ATTR inx=%d fname=%s\n", jcr->JobFiles, ff_pkt->fname);
if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
status = dir->fsend("%d %d %s %s%c%s%c%s%c", jcr->JobFiles,
STREAM_UNIX_ATTRIBUTES, ff_pkt->VerifyOpts, ff_pkt->fname,
0, attribs.c_str(), 0, ff_pkt->link, 0);
} else if (ff_pkt->type == FT_DIREND || ff_pkt->type == FT_REPARSE ||
ff_pkt->type == FT_JUNCTION) {
/* Here link is the canonical filename (i.e. with trailing slash) */
status = dir->fsend("%d %d %s %s%c%s%c%c", jcr->JobFiles,
STREAM_UNIX_ATTRIBUTES, ff_pkt->VerifyOpts, ff_pkt->link,
0, attribs.c_str(), 0, 0);
} else {
status = dir->fsend("%d %d %s %s%c%s%c%c", jcr->JobFiles,
STREAM_UNIX_ATTRIBUTES, ff_pkt->VerifyOpts, ff_pkt->fname,
0, attribs.c_str(), 0, 0);
}
Dmsg2(20, "filed>dir: attribs len=%d: msg=%s\n", dir->msglen, dir->msg);
if (!status) {
Jmsg(jcr, M_FATAL, 0, _("Network error in send to Director: ERR=%s\n"), bnet_strerror(dir));
return 0;
}
/*
* The remainder of the function is all about getting the checksum.
* First we initialise, then we read files, other streams and Finder Info.
*/
if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
ff_pkt->flags & (FO_MD5|FO_SHA1|FO_SHA256|FO_SHA512))) {
/*
* Create our digest context. If this fails, the digest will be set to NULL
* and not used.
*/
if (ff_pkt->flags & FO_MD5) {
digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
digest_stream = STREAM_MD5_DIGEST;
} else if (ff_pkt->flags & FO_SHA1) {
digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
digest_stream = STREAM_SHA1_DIGEST;
} else if (ff_pkt->flags & FO_SHA256) {
digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
digest_stream = STREAM_SHA256_DIGEST;
} else if (ff_pkt->flags & FO_SHA512) {
digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
digest_stream = STREAM_SHA512_DIGEST;
}
/* Did digest initialization fail? */
if (digest_stream != STREAM_NONE && digest == NULL) {
Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
stream_to_ascii(digest_stream));
}
/* compute MD5 or SHA1 hash */
if (digest) {
char md[CRYPTO_DIGEST_MAX_SIZE];
uint32_t size;
size = sizeof(md);
if (digest_file(jcr, ff_pkt, digest) != 0) {
jcr->JobErrors++;
goto good_rtn;
}
if (crypto_digest_finalize(digest, (uint8_t *)md, &size)) {
char *digest_buf;
const char *digest_name;
digest_buf = (char *)malloc(BASE64_SIZE(size));
digest_name = crypto_digest_name(digest);
bin_to_base64(digest_buf, BASE64_SIZE(size), md, size, true);
Dmsg3(400, "send inx=%d %s=%s\n", jcr->JobFiles, digest_name, digest_buf);
dir->fsend("%d %d %s *%s-%d*", jcr->JobFiles, digest_stream, digest_buf,
digest_name, jcr->JobFiles);
Dmsg3(20, "filed>dir: %s len=%d: msg=%s\n", digest_name,
dir->msglen, dir->msg);
free(digest_buf);
}
}
}
good_rtn:
if (digest) {
crypto_digest_free(digest);
}
return 1;
}
/*
* Called here for each record from read_records()
* This function is used when we do a external clone of a Job e.g.
* this SD is the reading SD. And a remote SD is the writing SD.
*
* Returns: true if OK
* false if error
*/
static bool clone_record_to_remote_sd(DCR *dcr, DEV_RECORD *rec)
{
POOLMEM *msgsave;
JCR *jcr = dcr->jcr;
char buf1[100], buf2[100];
BSOCK *sd = jcr->store_bsock;
bool send_eod, send_header;
#ifdef xxx
Dmsg5(000, "on entry JobId=%d FI=%s SessId=%d Strm=%s len=%d\n",
jcr->JobId, FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId,
stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len);
#endif
/*
* If label discard it
*/
if (rec->FileIndex < 0) {
return true;
}
/*
* See if this is the first record being processed.
*/
if (rec->last_FileIndex == 0) {
/*
* Initialize the last counters so we can compare
* things in the next run through here.
*/
rec->last_VolSessionId = rec->VolSessionId;
rec->last_VolSessionTime = rec->VolSessionTime;
rec->last_FileIndex = rec->FileIndex;
rec->last_Stream = rec->Stream;
jcr->JobFiles = 1;
/*
* Need to send both a new header only.
*/
send_eod = false;
send_header = true;
} else {
/*
* See if we are changing file or stream type.
*/
if (rec->VolSessionId != rec->last_VolSessionId ||
rec->VolSessionTime != rec->last_VolSessionTime ||
rec->FileIndex != rec->last_FileIndex ||
rec->Stream != rec->last_Stream) {
/*
* See if we are changing the FileIndex e.g.
* start processing the next file in the backup stream.
*/
if (rec->FileIndex != rec->last_FileIndex) {
jcr->JobFiles++;
}
/*
* Keep track of the new state.
*/
rec->last_VolSessionId = rec->VolSessionId;
rec->last_VolSessionTime = rec->VolSessionTime;
rec->last_FileIndex = rec->FileIndex;
rec->last_Stream = rec->Stream;
/*
* Need to send both a EOD and a new header.
*/
send_eod = true;
send_header = true;
} else {
send_eod = false;
send_header = false;
}
}
/*
* Send a EOD when needed.
*/
if (send_eod) {
if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
if (!jcr->is_job_canceled()) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
sd->bstrerror());
}
return false;
}
}
/*
* Send a header when needed.
*/
if (send_header) {
if (!sd->fsend("%ld %d 0", rec->FileIndex, rec->Stream)) {
if (!jcr->is_job_canceled()) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
sd->bstrerror());
}
return false;
}
}
/*
* Send the record data.
* We don't want to copy the data from the record to the socket structure
* so we save the original msg pointer and use the record data pointer for
* sending and restore the original msg pointer when done.
*/
msgsave = sd->msg;
sd->msg = rec->data;
sd->msglen = rec->data_len;
if (!sd->send()) {
sd->msg = msgsave;
sd->msglen = 0;
if (!jcr->is_job_canceled()) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
sd->bstrerror());
}
return false;
}
jcr->JobBytes += sd->msglen;
sd->msg = msgsave;
Dmsg5(500, "wrote_record JobId=%d FI=%s SessId=%d Strm=%s len=%d\n",
jcr->JobId, FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId,
stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len);
return true;
}
/*
* Write a Record to the block
*
* Returns: false on failure (none or partially written)
* true on success (all bytes written)
*
* and remainder returned in packet.
*
* We require enough room for the header, and we deal with
* two special cases. 1. Only part of the record may have
* been transferred the last time (when remainder is
* non-zero), and 2. The remaining bytes to write may not
* all fit into the block.
*/
bool write_record_to_block(DCR *dcr, DEV_RECORD *rec)
{
bool retval = false;
char buf1[100], buf2[100];
DEV_BLOCK *block = dcr->block;
/*
* After this point the record is in nrec not rec e.g. its either converted
* or is just a pointer to the same as the rec pointer being passed in.
*/
while (1) {
ASSERT(block->binbuf == (uint32_t)(block->bufp - block->buf));
ASSERT(block->buf_len >= block->binbuf);
Dmsg6(890, "write_record_to_block() FI=%s SessId=%d Strm=%s len=%d\n"
"rem=%d remainder=%d\n",
FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId,
stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len,
rec->remlen, rec->remainder);
switch (rec->state) {
case st_none:
/*
* Figure out what to do
*/
rec->state = st_header;
continue; /* go to next state */
case st_header:
/*
* Write header
*
* If rec->remainder is non-zero, we have been called a
* second (or subsequent) time to finish writing a record
* that did not previously fit into the block.
*/
if (!write_header_to_block(block, rec)) {
goto bail_out; /* write block then come back here */
}
rec->state = st_data; /* after header, now write data */
continue;
/*
* Write continuation header
*/
case st_header_cont:
write_continue_header_to_block(block, rec);
rec->state = st_data;
if (rec->remlen == 0) {
goto bail_out; /* Partial transfer */
}
continue;
/*
* Write normal data
*/
case st_data:
/*
* Write data
*
* Part of it may have already been transferred, and we
* may not have enough room to transfer the whole this time.
*/
if (rec->remainder > 0) {
if (!write_data_to_block(block, rec)) {
rec->state = st_header_cont;
goto bail_out;
}
}
rec->remainder = 0; /* did whole transfer */
rec->state = st_none;
retval = true;
goto bail_out;
default:
Dmsg0(000, "Something went wrong. Default state.\n");
rec->state = st_none;
retval = true;
goto bail_out;
}
}
bail_out:
return retval;
}
/**
* Setup for digest handling. If this fails, the digest will be set to NULL
* and not used. Note, the digest (file hash) can be any one of the four
* algorithms below.
*
* The signing digest is a single algorithm depending on
* whether or not we have SHA2.
* ****FIXME**** the signing algorithm should really be
* determined a different way!!!!!! What happens if
* sha2 was available during backup but not restore?
*/
static inline bool setup_encryption_digests(b_save_ctx &bsctx)
{
bool retval = false;
// TODO landonf: Allow the user to specify the digest algorithm
#ifdef HAVE_SHA2
crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
#else
crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
#endif
if (bit_is_set(FO_MD5, bsctx.ff_pkt->flags)) {
bsctx.digest = crypto_digest_new(bsctx.jcr, CRYPTO_DIGEST_MD5);
bsctx.digest_stream = STREAM_MD5_DIGEST;
} else if (bit_is_set(FO_SHA1, bsctx.ff_pkt->flags)) {
bsctx.digest = crypto_digest_new(bsctx.jcr, CRYPTO_DIGEST_SHA1);
bsctx.digest_stream = STREAM_SHA1_DIGEST;
} else if (bit_is_set(FO_SHA256, bsctx.ff_pkt->flags)) {
bsctx.digest = crypto_digest_new(bsctx.jcr, CRYPTO_DIGEST_SHA256);
bsctx.digest_stream = STREAM_SHA256_DIGEST;
} else if (bit_is_set(FO_SHA512, bsctx.ff_pkt->flags)) {
bsctx.digest = crypto_digest_new(bsctx.jcr, CRYPTO_DIGEST_SHA512);
bsctx.digest_stream = STREAM_SHA512_DIGEST;
}
/*
* Did digest initialization fail?
*/
if (bsctx.digest_stream != STREAM_NONE && bsctx.digest == NULL) {
Jmsg(bsctx.jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
stream_to_ascii(bsctx.digest_stream));
}
/**
* Set up signature digest handling. If this fails, the signature digest
* will be set to NULL and not used.
*/
/* TODO landonf: We should really only calculate the digest once, for
* both verification and signing.
*/
if (bsctx.jcr->crypto.pki_sign) {
bsctx.signing_digest = crypto_digest_new(bsctx.jcr, signing_algorithm);
/*
* Full-stop if a failure occurred initializing the signature digest
*/
if (bsctx.signing_digest == NULL) {
Jmsg(bsctx.jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
stream_to_ascii(signing_algorithm));
bsctx.jcr->JobErrors++;
goto bail_out;
}
}
/*
* Enable encryption
*/
if (bsctx.jcr->crypto.pki_encrypt) {
set_bit(FO_ENCRYPT, bsctx.ff_pkt->flags);
}
retval = true;
bail_out:
return retval;
}
/*
* Append Data sent from Client (FD/SD)
*
*/
bool do_append_data(JCR *jcr)
{
int32_t n;
int32_t file_index, stream, last_file_index;
uint64_t stream_len;
BSOCK *fd = jcr->file_bsock;
bool ok = true;
DEV_RECORD rec;
char buf1[100], buf2[100];
DCR *dcr = jcr->dcr;
DEVICE *dev;
char ec[50];
POOLMEM *eblock = NULL;
POOL_MEM errmsg(PM_EMSG);
if (!dcr) {
pm_strcpy(jcr->errmsg, _("DCR is NULL!!!\n"));
Jmsg0(jcr, M_FATAL, 0, jcr->errmsg);
return false;
}
dev = dcr->dev;
if (!dev) {
pm_strcpy(jcr->errmsg, _("DEVICE is NULL!!!\n"));
Jmsg0(jcr, M_FATAL, 0, jcr->errmsg);
return false;
}
Dmsg1(100, "Start append data. res=%d\n", dev->num_reserved());
memset(&rec, 0, sizeof(rec));
if (!fd->set_buffer_size(dcr->device->max_network_buffer_size, BNET_SETBUF_WRITE)) {
jcr->setJobStatus(JS_ErrorTerminated);
pm_strcpy(jcr->errmsg, _("Unable to set network buffer size.\n"));
Jmsg0(jcr, M_FATAL, 0, jcr->errmsg);
return false;
}
if (!acquire_device_for_append(dcr)) {
jcr->setJobStatus(JS_ErrorTerminated);
return false;
}
jcr->sendJobStatus(JS_Running);
//ASSERT(dev->VolCatInfo.VolCatName[0]);
if (dev->VolCatInfo.VolCatName[0] == 0) {
Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
}
Dmsg1(50, "Begin append device=%s\n", dev->print_name());
begin_data_spool(dcr);
begin_attribute_spool(jcr);
Dmsg0(100, "Just after acquire_device_for_append\n");
//ASSERT(dev->VolCatInfo.VolCatName[0]);
if (dev->VolCatInfo.VolCatName[0] == 0) {
Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
}
/*
* Write Begin Session Record
*/
if (!write_session_label(dcr, SOS_LABEL)) {
Jmsg1(jcr, M_FATAL, 0, _("Write session label failed. ERR=%s\n"),
dev->bstrerror());
jcr->setJobStatus(JS_ErrorTerminated);
ok = false;
}
//ASSERT(dev->VolCatInfo.VolCatName[0]);
if (dev->VolCatInfo.VolCatName[0] == 0) {
Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
}
/* Tell File daemon to send data */
if (!fd->fsend(OK_data)) {
berrno be;
Jmsg1(jcr, M_FATAL, 0, _("Network send error to FD. ERR=%s\n"),
be.bstrerror(fd->b_errno));
ok = false;
}
/*
* Get Data from File daemon, write to device. To clarify what is
* going on here. We expect:
* - A stream header
* - Multiple records of data
* - EOD record
*
* The Stream header is just used to synchronize things, and
* none of the stream header is written to tape.
* The Multiple records of data, contain first the Attributes,
* then after another stream header, the file data, then
* after another stream header, the MD5 data if any.
*
* So we get the (stream header, data, EOD) three time for each
* file. 1. for the Attributes, 2. for the file data if any,
* and 3. for the MD5 if any.
*/
dcr->VolFirstIndex = dcr->VolLastIndex = 0;
jcr->run_time = time(NULL); /* start counting time for rates */
GetMsg *qfd;
qfd = New(GetMsg(jcr, fd, NULL, GETMSG_MAX_MSG_SIZE));
qfd->start_read_sock();
for (last_file_index = 0; ok && !jcr->is_job_canceled(); ) {
/* Read Stream header from the File daemon.
* The stream header consists of the following:
* file_index (sequential Bacula file index, base 1)
* stream (Bacula number to distinguish parts of data)
* stream_len (Expected length of this stream. This
* will be the size backed up if the file does not
* grow during the backup.
*/
n = qfd->bget_msg(NULL);
if (n <= 0) {
if (n == BNET_SIGNAL && qfd->msglen == BNET_EOD) {
Dmsg0(200, "Got EOD on reading header.\n");
break; /* end of data */
}
Jmsg3(jcr, M_FATAL, 0, _("Error reading data header from FD. n=%d msglen=%d ERR=%s\n"),
n, qfd->msglen, fd->bstrerror());
// ASX TODO the fd->bstrerror() can be related to the wrong error, I should Queue the error too
possible_incomplete_job(jcr, last_file_index);
ok = false;
break;
}
if (sscanf(qfd->msg, "%ld %ld %lld", &file_index, &stream, &stream_len) != 3) {
// TODO ASX already done in bufmsg, should reuse the values
char buf[256];
Jmsg1(jcr, M_FATAL, 0, _("Malformed data header from FD: %s\n"), asciidump(qfd->msg, qfd->msglen, buf, sizeof(buf)));
ok = false;
possible_incomplete_job(jcr, last_file_index);
break;
}
Dmsg3(890, "<filed: Header FilInx=%d stream=%d stream_len=%lld\n",
file_index, stream, stream_len);
/*
* We make sure the file_index is advancing sequentially.
* An incomplete job can start the file_index at any number.
* otherwise, it must start at 1.
*/
if (jcr->rerunning && file_index > 0 && last_file_index == 0) {
goto fi_checked;
}
Dmsg2(400, "file_index=%d last_file_index=%d\n", file_index, last_file_index);
if (file_index > 0 && (file_index == last_file_index ||
file_index == last_file_index + 1)) {
goto fi_checked;
}
Jmsg2(jcr, M_FATAL, 0, _("FI=%d from FD not positive or last_FI=%d\n"),
file_index, last_file_index);
possible_incomplete_job(jcr, last_file_index);
ok = false;
break;
fi_checked:
if (file_index != last_file_index) {
jcr->JobFiles = file_index;
last_file_index = file_index;
}
/* Read data stream from the File daemon.
* The data stream is just raw bytes
*/
while ((n=qfd->bget_msg(NULL)) > 0 && !jcr->is_job_canceled()) {
rec.VolSessionId = jcr->VolSessionId;
rec.VolSessionTime = jcr->VolSessionTime;
rec.FileIndex = file_index;
rec.Stream = stream;
rec.StreamLen = stream_len;
rec.maskedStream = stream & STREAMMASK_TYPE; /* strip high bits */
rec.data_len = qfd->msglen;
rec.data = qfd->msg; /* use message buffer */
Dmsg4(850, "before writ_rec FI=%d SessId=%d Strm=%s len=%d\n",
rec.FileIndex, rec.VolSessionId,
stream_to_ascii(buf1, rec.Stream,rec.FileIndex),
rec.data_len);
ok = dcr->write_record(&rec);
if (!ok) {
Dmsg2(90, "Got write_block_to_dev error on device %s. %s\n",
dcr->dev->print_name(), dcr->dev->bstrerror());
break;
}
jcr->JobBytes += rec.data_len; /* increment bytes this job */
jcr->JobBytes += qfd->bmsg->jobbytes; // if the block as been downloaded, count it
Dmsg4(850, "write_record FI=%s SessId=%d Strm=%s len=%d\n",
FI_to_ascii(buf1, rec.FileIndex), rec.VolSessionId,
stream_to_ascii(buf2, rec.Stream, rec.FileIndex), rec.data_len);
send_attrs_to_dir(jcr, &rec);
Dmsg0(650, "Enter bnet_get\n");
}
Dmsg2(650, "End read loop with FD. JobFiles=%d Stat=%d\n", jcr->JobFiles, n);
if (fd->is_error()) {
if (!jcr->is_job_canceled()) {
Dmsg1(350, "Network read error from FD. ERR=%s\n", fd->bstrerror());
Jmsg1(jcr, M_FATAL, 0, _("Network error reading from FD. ERR=%s\n"),
fd->bstrerror());
possible_incomplete_job(jcr, last_file_index);
}
ok = false;
break;
}
}
qfd->wait_read_sock();
free_GetMsg(qfd);
if (eblock != NULL) {
free_pool_memory(eblock);
}
/* Create Job status for end of session label */
jcr->setJobStatus(ok?JS_Terminated:JS_ErrorTerminated);
if (ok) {
/* Terminate connection with Client */
fd->fsend(OK_append);
do_client_commands(jcr); /* finish dialog with Client */
} else {
fd->fsend("3999 Failed append\n");
}
Dmsg1(200, "Write EOS label JobStatus=%c\n", jcr->JobStatus);
/*
* Check if we can still write. This may not be the case
* if we are at the end of the tape or we got a fatal I/O error.
*/
if (ok || dev->can_write()) {
if (!write_session_label(dcr, EOS_LABEL)) {
/* Print only if ok and not cancelled to avoid spurious messages */
if (ok && !jcr->is_job_canceled()) {
Jmsg1(jcr, M_FATAL, 0, _("Error writing end session label. ERR=%s\n"),
dev->bstrerror());
possible_incomplete_job(jcr, last_file_index);
}
jcr->setJobStatus(JS_ErrorTerminated);
ok = false;
}
/* Flush out final partial block of this session */
if (!dcr->write_final_block_to_device()) {
/* Print only if ok and not cancelled to avoid spurious messages */
if (ok && !jcr->is_job_canceled()) {
Jmsg2(jcr, M_FATAL, 0, _("Fatal append error on device %s: ERR=%s\n"),
dev->print_name(), dev->bstrerror());
Dmsg0(100, _("Set ok=FALSE after write_final_block_to_device.\n"));
possible_incomplete_job(jcr, last_file_index);
}
jcr->setJobStatus(JS_ErrorTerminated);
ok = false;
}
}
flush_jobmedia_queue(jcr);
if (!ok && !jcr->is_JobStatus(JS_Incomplete)) {
discard_data_spool(dcr);
} else {
/* Note: if commit is OK, the device will remain blocked */
commit_data_spool(dcr);
}
/*
* Don't use time_t for job_elapsed as time_t can be 32 or 64 bits,
* and the subsequent Jmsg() editing will break
*/
int32_t job_elapsed = time(NULL) - jcr->run_time;
if (job_elapsed <= 0) {
job_elapsed = 1;
}
Jmsg(dcr->jcr, M_INFO, 0, _("Elapsed time=%02d:%02d:%02d, Transfer rate=%s Bytes/second\n"),
job_elapsed / 3600, job_elapsed % 3600 / 60, job_elapsed % 60,
edit_uint64_with_suffix(jcr->JobBytes / job_elapsed, ec));
/*
* Release the device -- and send final Vol info to DIR
* and unlock it.
*/
release_device(dcr);
if ((!ok || jcr->is_job_canceled()) && !jcr->is_JobStatus(JS_Incomplete)) {
discard_attribute_spool(jcr);
} else {
commit_attribute_spool(jcr);
}
jcr->sendJobStatus(); /* update director */
Dmsg1(100, "return from do_append_data() ok=%d\n", ok);
return ok;
}
/*
* This routine is called only during a Verify
*/
void get_attributes_and_compare_to_catalog(JCR *jcr, JobId_t JobId)
{
BSOCK *fd;
int n, len;
FILE_DBR fdbr;
struct stat statf; /* file stat */
struct stat statc; /* catalog stat */
char buf[MAXSTRING];
POOLMEM *fname = get_pool_memory(PM_MESSAGE);
int do_Digest = CRYPTO_DIGEST_NONE;
int32_t file_index = 0;
memset(&fdbr, 0, sizeof(FILE_DBR));
fd = jcr->file_bsock;
fdbr.JobId = JobId;
jcr->FileIndex = 0;
Dmsg0(20, "bdird: waiting to receive file attributes\n");
/*
* Get Attributes and Signature from File daemon
* We expect:
* FileIndex
* Stream
* Options or Digest (MD5/SHA1)
* Filename
* Attributes
* Link name ???
*/
while ((n=bget_dirmsg(fd)) >= 0 && !job_canceled(jcr)) {
int32_t stream, full_stream;
char *attr, *p, *fn;
char Opts_Digest[MAXSTRING]; /* Verify Opts or MD5/SHA1 digest */
if (job_canceled(jcr)) {
free_pool_memory(fname);
return;
}
fname = check_pool_memory_size(fname, fd->msglen);
jcr->fname = check_pool_memory_size(jcr->fname, fd->msglen);
Dmsg1(200, "Atts+Digest=%s\n", fd->msg);
if ((len = sscanf(fd->msg, "%ld %d %100s", &file_index, &full_stream,
fname)) != 3) {
Jmsg3(jcr, M_FATAL, 0, _("bird<filed: bad attributes, expected 3 fields got %d\n"
" mslen=%d msg=%s\n"), len, fd->msglen, fd->msg);
free_pool_memory(fname);
return;
}
stream = full_stream & STREAMMASK_TYPE;
Dmsg4(30, "Got hdr: FilInx=%d FullStream=%d Stream=%d fname=%s.\n", file_index, full_stream, stream, fname);
/*
* We read the Options or Signature into fname
* to prevent overrun, now copy it to proper location.
*/
bstrncpy(Opts_Digest, fname, sizeof(Opts_Digest));
p = fd->msg;
skip_nonspaces(&p); /* skip FileIndex */
skip_spaces(&p);
skip_nonspaces(&p); /* skip Stream */
skip_spaces(&p);
skip_nonspaces(&p); /* skip Opts_Digest */
p++; /* skip space */
fn = fname;
while (*p != 0) {
*fn++ = *p++; /* copy filename */
}
*fn = *p++; /* term filename and point to attribs */
attr = p;
/*
* Got attributes stream, decode it
*/
if (stream == STREAM_UNIX_ATTRIBUTES || stream == STREAM_UNIX_ATTRIBUTES_EX) {
int32_t LinkFIf, LinkFIc;
Dmsg2(400, "file_index=%d attr=%s\n", file_index, attr);
jcr->JobFiles++;
jcr->FileIndex = file_index; /* remember attribute file_index */
jcr->previous_jr.FileIndex = file_index;
decode_stat(attr, &statf, sizeof(statf), &LinkFIf); /* decode file stat packet */
do_Digest = CRYPTO_DIGEST_NONE;
jcr->fn_printed = false;
pm_strcpy(jcr->fname, fname); /* move filename into JCR */
Dmsg2(040, "dird<filed: stream=%d %s\n", stream, jcr->fname);
Dmsg1(020, "dird<filed: attr=%s\n", attr);
/*
* Find equivalent record in the database
*/
fdbr.FileId = 0;
if (!db_get_file_attributes_record(jcr, jcr->db, jcr->fname,
&jcr->previous_jr, &fdbr)) {
Jmsg(jcr, M_INFO, 0, _("New file: %s\n"), jcr->fname);
Dmsg1(020, _("File not in catalog: %s\n"), jcr->fname);
jcr->setJobStatus(JS_Differences);
continue;
} else {
/*
* mark file record as visited by stuffing the
* current JobId, which is unique, into the MarkId field.
*/
db_mark_file_record(jcr, jcr->db, fdbr.FileId, jcr->JobId);
}
Dmsg3(400, "Found %s in catalog. inx=%d Opts=%s\n", jcr->fname,
file_index, Opts_Digest);
decode_stat(fdbr.LStat, &statc, sizeof(statc), &LinkFIc); /* decode catalog stat */
/*
* Loop over options supplied by user and verify the
* fields he requests.
*/
for (p=Opts_Digest; *p; p++) {
char ed1[30], ed2[30];
switch (*p) {
case 'i': /* compare INODEs */
if (statc.st_ino != statf.st_ino) {
prt_fname(jcr);
Jmsg(jcr, M_INFO, 0, _(" st_ino differ. Cat: %s File: %s\n"),
edit_uint64((uint64_t)statc.st_ino, ed1),
edit_uint64((uint64_t)statf.st_ino, ed2));
jcr->setJobStatus(JS_Differences);
}
break;
case 'p': /* permissions bits */
if (statc.st_mode != statf.st_mode) {
prt_fname(jcr);
Jmsg(jcr, M_INFO, 0, _(" st_mode differ. Cat: %x File: %x\n"),
(uint32_t)statc.st_mode, (uint32_t)statf.st_mode);
jcr->setJobStatus(JS_Differences);
}
break;
case 'n': /* number of links */
if (statc.st_nlink != statf.st_nlink) {
prt_fname(jcr);
Jmsg(jcr, M_INFO, 0, _(" st_nlink differ. Cat: %d File: %d\n"),
(uint32_t)statc.st_nlink, (uint32_t)statf.st_nlink);
jcr->setJobStatus(JS_Differences);
}
break;
case 'u': /* user id */
if (statc.st_uid != statf.st_uid) {
prt_fname(jcr);
Jmsg(jcr, M_INFO, 0, _(" st_uid differ. Cat: %u File: %u\n"),
(uint32_t)statc.st_uid, (uint32_t)statf.st_uid);
jcr->setJobStatus(JS_Differences);
}
break;
case 'g': /* group id */
if (statc.st_gid != statf.st_gid) {
prt_fname(jcr);
Jmsg(jcr, M_INFO, 0, _(" st_gid differ. Cat: %u File: %u\n"),
(uint32_t)statc.st_gid, (uint32_t)statf.st_gid);
jcr->setJobStatus(JS_Differences);
}
break;
case 's': /* size */
if (statc.st_size != statf.st_size) {
prt_fname(jcr);
Jmsg(jcr, M_INFO, 0, _(" st_size differ. Cat: %s File: %s\n"),
edit_uint64((uint64_t)statc.st_size, ed1),
edit_uint64((uint64_t)statf.st_size, ed2));
jcr->setJobStatus(JS_Differences);
}
break;
case 'a': /* access time */
if (statc.st_atime != statf.st_atime) {
prt_fname(jcr);
Jmsg(jcr, M_INFO, 0, _(" st_atime differs\n"));
jcr->setJobStatus(JS_Differences);
}
break;
case 'm':
if (statc.st_mtime != statf.st_mtime) {
prt_fname(jcr);
Jmsg(jcr, M_INFO, 0, _(" st_mtime differs\n"));
jcr->setJobStatus(JS_Differences);
}
break;
case 'c': /* ctime */
if (statc.st_ctime != statf.st_ctime) {
prt_fname(jcr);
Jmsg(jcr, M_INFO, 0, _(" st_ctime differs\n"));
jcr->setJobStatus(JS_Differences);
}
break;
case 'd': /* file size decrease */
if (statc.st_size > statf.st_size) {
prt_fname(jcr);
Jmsg(jcr, M_INFO, 0, _(" st_size decrease. Cat: %s File: %s\n"),
edit_uint64((uint64_t)statc.st_size, ed1),
edit_uint64((uint64_t)statf.st_size, ed2));
jcr->setJobStatus(JS_Differences);
}
break;
case '5': /* compare MD5 */
Dmsg1(500, "set Do_MD5 for %s\n", jcr->fname);
do_Digest = CRYPTO_DIGEST_MD5;
break;
case '1': /* compare SHA1 */
do_Digest = CRYPTO_DIGEST_SHA1;
break;
case ':':
case 'V':
default:
break;
}
}
/*
* Got Digest Signature from Storage daemon
* It came across in the Opts_Digest field.
*/
} else if (crypto_digest_stream_type(stream) != CRYPTO_DIGEST_NONE) {
Dmsg2(400, "stream=Digest inx=%d Digest=%s\n", file_index, Opts_Digest);
/*
* When ever we get a digest it MUST have been
* preceded by an attributes record, which sets attr_file_index
*/
if (jcr->FileIndex != (uint32_t)file_index) {
Jmsg2(jcr, M_FATAL, 0, _("MD5/SHA1 index %d not same as attributes %d\n"),
file_index, jcr->FileIndex);
free_pool_memory(fname);
return;
}
if (do_Digest != CRYPTO_DIGEST_NONE) {
db_escape_string(jcr, jcr->db, buf, Opts_Digest, strlen(Opts_Digest));
if (strcmp(buf, fdbr.Digest) != 0) {
prt_fname(jcr);
Jmsg(jcr, M_INFO, 0, _(" %s differs. File=%s Cat=%s\n"),
stream_to_ascii(stream), buf, fdbr.Digest);
jcr->setJobStatus(JS_Differences);
}
do_Digest = CRYPTO_DIGEST_NONE;
}
}
jcr->JobFiles = file_index;
}
if (fd->is_error()) {
berrno be;
Jmsg2(jcr, M_FATAL, 0, _("bdird<filed: bad attributes from filed n=%d : %s\n"),
n, be.bstrerror());
free_pool_memory(fname);
return;
}
/* Now find all the files that are missing -- i.e. all files in
* the database where the MarkId != current JobId
*/
jcr->fn_printed = false;
bsnprintf(buf, sizeof(buf),
"SELECT Path.Path,Filename.Name FROM File,Path,Filename "
"WHERE File.JobId=%d AND File.FileIndex > 0 "
"AND File.MarkId!=%d AND File.PathId=Path.PathId "
"AND File.FilenameId=Filename.FilenameId",
JobId, jcr->JobId);
/* missing_handler is called for each file found */
db_sql_query(jcr->db, buf, missing_handler, (void *)jcr);
if (jcr->fn_printed) {
jcr->setJobStatus(JS_Differences);
}
free_pool_memory(fname);
}
/*
* Note, we receive the whole attribute record, but we select out only the stat
* packet, VolSessionId, VolSessionTime, FileIndex, file type, and file name to
* store in the catalog.
*/
static void update_attribute(JCR *jcr, char *msg, int32_t msglen)
{
unser_declare;
uint32_t VolSessionId, VolSessionTime;
int32_t Stream;
uint32_t FileIndex;
char *p;
int len;
char *fname, *attr;
ATTR_DBR *ar = NULL;
uint32_t reclen;
/*
* Start transaction allocates jcr->attr and jcr->ar if needed
*/
db_start_transaction(jcr, jcr->db); /* start transaction if not already open */
ar = jcr->ar;
/*
* Start by scanning directly in the message buffer to get Stream
* there may be a cached attr so we cannot yet write into
* jcr->attr or jcr->ar
*/
p = msg;
skip_nonspaces(&p); /* UpdCat */
skip_spaces(&p);
skip_nonspaces(&p); /* Job=nnn */
skip_spaces(&p);
skip_nonspaces(&p); /* "FileAttributes" */
p += 1;
/*
* The following "SD header" fields are serialized
*/
unser_begin(p, 0);
unser_uint32(VolSessionId); /* VolSessionId */
unser_uint32(VolSessionTime); /* VolSessionTime */
unser_int32(FileIndex); /* FileIndex */
unser_int32(Stream); /* Stream */
unser_uint32(reclen); /* Record length */
p += unser_length(p); /* Raw record follows */
/**
* At this point p points to the raw record, which varies according
* to what kind of a record (Stream) was sent. Note, the integer
* fields at the beginning of these "raw" records are in ASCII with
* spaces between them so one can use scanf or manual scanning to
* extract the fields.
*
* File Attributes
* File_index
* File type
* Filename (full path)
* Encoded attributes
* Link name (if type==FT_LNK or FT_LNKSAVED)
* Encoded extended-attributes (for Win32)
* Delta sequence number (32 bit int)
*
* Restore Object
* File_index
* File_type
* Object_index
* Object_len (possibly compressed)
* Object_full_len (not compressed)
* Object_compression
* Plugin_name
* Object_name
* Binary Object data
*/
Dmsg1(400, "UpdCat msg=%s\n", msg);
Dmsg5(400, "UpdCat VolSessId=%d VolSessT=%d FI=%d Strm=%d reclen=%d\n",
VolSessionId, VolSessionTime, FileIndex, Stream, reclen);
jcr->SDJobBytes += reclen; /* update number of bytes transferred for quotas */
/*
* Depending on the stream we are handling dispatch.
*/
switch (Stream) {
case STREAM_UNIX_ATTRIBUTES:
case STREAM_UNIX_ATTRIBUTES_EX:
if (jcr->cached_attribute) {
Dmsg2(400, "Cached attr. Stream=%d fname=%s\n", ar->Stream, ar->fname);
if (!db_create_attributes_record(jcr, jcr->db, ar)) {
Jmsg1(jcr, M_FATAL, 0, _("Attribute create error: ERR=%s"), db_strerror(jcr->db));
}
jcr->cached_attribute = false;
}
/*
* Any cached attr is flushed so we can reuse jcr->attr and jcr->ar
*/
jcr->attr = check_pool_memory_size(jcr->attr, msglen);
memcpy(jcr->attr, msg, msglen);
p = jcr->attr - msg + p; /* point p into jcr->attr */
skip_nonspaces(&p); /* skip FileIndex */
skip_spaces(&p);
ar->FileType = str_to_int32(p);
skip_nonspaces(&p); /* skip FileType */
skip_spaces(&p);
fname = p;
len = strlen(fname); /* length before attributes */
attr = &fname[len+1];
ar->DeltaSeq = 0;
if (ar->FileType == FT_REG) {
p = attr + strlen(attr) + 1; /* point to link */
p = p + strlen(p) + 1; /* point to extended attributes */
p = p + strlen(p) + 1; /* point to delta sequence */
/*
* Older FDs don't have a delta sequence, so check if it is there
*/
if (p - jcr->attr < msglen) {
ar->DeltaSeq = str_to_int32(p); /* delta_seq */
}
}
Dmsg2(400, "dird<stored: stream=%d %s\n", Stream, fname);
Dmsg1(400, "dird<stored: attr=%s\n", attr);
ar->attr = attr;
ar->fname = fname;
if (ar->FileType == FT_DELETED) {
ar->FileIndex = 0; /* special value */
} else {
ar->FileIndex = FileIndex;
}
ar->Stream = Stream;
ar->link = NULL;
if (jcr->mig_jcr) {
ar->JobId = jcr->mig_jcr->JobId;
} else {
ar->JobId = jcr->JobId;
}
ar->Digest = NULL;
ar->DigestType = CRYPTO_DIGEST_NONE;
jcr->cached_attribute = true;
Dmsg2(400, "dird<filed: stream=%d %s\n", Stream, fname);
Dmsg1(400, "dird<filed: attr=%s\n", attr);
break;
case STREAM_RESTORE_OBJECT: {
ROBJECT_DBR ro;
memset(&ro, 0, sizeof(ro));
ro.Stream = Stream;
ro.FileIndex = FileIndex;
if (jcr->mig_jcr) {
ro.JobId = jcr->mig_jcr->JobId;
} else {
ro.JobId = jcr->JobId;
}
Dmsg1(100, "Robj=%s\n", p);
skip_nonspaces(&p); /* skip FileIndex */
skip_spaces(&p);
ro.FileType = str_to_int32(p); /* FileType */
skip_nonspaces(&p);
skip_spaces(&p);
ro.object_index = str_to_int32(p); /* Object Index */
skip_nonspaces(&p);
skip_spaces(&p);
ro.object_len = str_to_int32(p); /* object length possibly compressed */
skip_nonspaces(&p);
skip_spaces(&p);
ro.object_full_len = str_to_int32(p); /* uncompressed object length */
skip_nonspaces(&p);
skip_spaces(&p);
ro.object_compression = str_to_int32(p); /* compression */
skip_nonspaces(&p);
skip_spaces(&p);
ro.plugin_name = p; /* point to plugin name */
len = strlen(ro.plugin_name);
ro.object_name = &ro.plugin_name[len+1]; /* point to object name */
len = strlen(ro.object_name);
ro.object = &ro.object_name[len+1]; /* point to object */
ro.object[ro.object_len] = 0; /* add zero for those who attempt printing */
Dmsg7(100, "oname=%s stream=%d FT=%d FI=%d JobId=%d, obj_len=%d\nobj=\"%s\"\n",
ro.object_name, ro.Stream, ro.FileType, ro.FileIndex, ro.JobId,
ro.object_len, ro.object);
/*
* Store it.
*/
if (!db_create_restore_object_record(jcr, jcr->db, &ro)) {
Jmsg1(jcr, M_FATAL, 0, _("Restore object create error. %s"), db_strerror(jcr->db));
}
break;
}
default:
if (crypto_digest_stream_type(Stream) != CRYPTO_DIGEST_NONE) {
fname = p;
if (ar->FileIndex != FileIndex) {
Jmsg3(jcr, M_WARNING, 0, _("%s not same File=%d as attributes=%d\n"),
stream_to_ascii(Stream), FileIndex, ar->FileIndex);
} else {
/*
* Update digest in catalog
*/
char digestbuf[BASE64_SIZE(CRYPTO_DIGEST_MAX_SIZE)];
int len = 0;
int type = CRYPTO_DIGEST_NONE;
switch(Stream) {
case STREAM_MD5_DIGEST:
len = CRYPTO_DIGEST_MD5_SIZE;
type = CRYPTO_DIGEST_MD5;
break;
case STREAM_SHA1_DIGEST:
len = CRYPTO_DIGEST_SHA1_SIZE;
type = CRYPTO_DIGEST_SHA1;
break;
case STREAM_SHA256_DIGEST:
len = CRYPTO_DIGEST_SHA256_SIZE;
type = CRYPTO_DIGEST_SHA256;
break;
case STREAM_SHA512_DIGEST:
len = CRYPTO_DIGEST_SHA512_SIZE;
type = CRYPTO_DIGEST_SHA512;
break;
default:
/*
* Never reached ...
*/
Jmsg(jcr, M_ERROR, 0, _("Catalog error updating file digest. Unsupported digest stream type: %d"), Stream);
}
bin_to_base64(digestbuf, sizeof(digestbuf), fname, len, true);
Dmsg3(400, "DigestLen=%d Digest=%s type=%d\n", strlen(digestbuf),
digestbuf, Stream);
if (jcr->cached_attribute) {
ar->Digest = digestbuf;
ar->DigestType = type;
Dmsg2(400, "Cached attr with digest. Stream=%d fname=%s\n", ar->Stream, ar->fname);
/*
* Update BaseFile table
*/
if (!db_create_attributes_record(jcr, jcr->db, ar)) {
Jmsg1(jcr, M_FATAL, 0, _("attribute create error. %s"), db_strerror(jcr->db));
}
jcr->cached_attribute = false;
} else {
if (!db_add_digest_to_file_record(jcr, jcr->db, ar->FileId, digestbuf, type)) {
Jmsg(jcr, M_ERROR, 0, _("Catalog error updating file digest. %s"), db_strerror(jcr->db));
}
}
}
}
break;
}
}
/*
* Called here for each record from read_records()
*/
static bool record_cb(DCR *dcr, DEV_RECORD *rec)
{
int status;
JCR *jcr = dcr->jcr;
if (rec->FileIndex < 0) {
return true; /* we don't want labels */
}
/* File Attributes stream */
switch (rec->maskedStream) {
case STREAM_UNIX_ATTRIBUTES:
case STREAM_UNIX_ATTRIBUTES_EX:
/* If extracting, it was from previous stream, so
* close the output file.
*/
if (extract) {
if (!is_bopen(&bfd)) {
Emsg0(M_ERROR, 0, _("Logic error output file should be open but is not.\n"));
}
set_attributes(jcr, attr, &bfd);
extract = false;
}
if (!unpack_attributes_record(jcr, rec->Stream, rec->data, rec->data_len, attr)) {
Emsg0(M_ERROR_TERM, 0, _("Cannot continue.\n"));
}
if (file_is_included(ff, attr->fname) && !file_is_excluded(ff, attr->fname)) {
attr->data_stream = decode_stat(attr->attr, &attr->statp, sizeof(attr->statp), &attr->LinkFI);
if (!is_restore_stream_supported(attr->data_stream)) {
if (!non_support_data++) {
Jmsg(jcr, M_ERROR, 0, _("%s stream not supported on this Client.\n"),
stream_to_ascii(attr->data_stream));
}
extract = false;
return true;
}
build_attr_output_fnames(jcr, attr);
if (attr->type == FT_DELETED) { /* TODO: choose the right fname/ofname */
Jmsg(jcr, M_INFO, 0, _("%s was deleted.\n"), attr->fname);
extract = false;
return true;
}
extract = false;
status = create_file(jcr, attr, &bfd, REPLACE_ALWAYS);
switch (status) {
case CF_ERROR:
case CF_SKIP:
break;
case CF_EXTRACT:
extract = true;
print_ls_output(jcr, attr);
num_files++;
fileAddr = 0;
break;
case CF_CREATED:
set_attributes(jcr, attr, &bfd);
print_ls_output(jcr, attr);
num_files++;
fileAddr = 0;
break;
}
}
break;
case STREAM_RESTORE_OBJECT:
/* nothing to do */
break;
/* Data stream and extracting */
case STREAM_FILE_DATA:
case STREAM_SPARSE_DATA:
case STREAM_WIN32_DATA:
if (extract) {
if (rec->maskedStream == STREAM_SPARSE_DATA) {
ser_declare;
uint64_t faddr;
wbuf = rec->data + OFFSET_FADDR_SIZE;
wsize = rec->data_len - OFFSET_FADDR_SIZE;
ser_begin(rec->data, OFFSET_FADDR_SIZE);
unser_uint64(faddr);
if (fileAddr != faddr) {
fileAddr = faddr;
if (blseek(&bfd, (boffset_t)fileAddr, SEEK_SET) < 0) {
berrno be;
Emsg2(M_ERROR_TERM, 0, _("Seek error on %s: %s\n"),
attr->ofname, be.bstrerror());
}
}
} else {
wbuf = rec->data;
wsize = rec->data_len;
}
total += wsize;
Dmsg2(8, "Write %u bytes, total=%u\n", wsize, total);
store_data(&bfd, wbuf, wsize);
fileAddr += wsize;
}
break;
/* GZIP data stream */
case STREAM_GZIP_DATA:
case STREAM_SPARSE_GZIP_DATA:
case STREAM_WIN32_GZIP_DATA:
#ifdef HAVE_LIBZ
if (extract) {
uLong compress_len = compress_buf_size;
int status = Z_BUF_ERROR;
if (rec->maskedStream == STREAM_SPARSE_GZIP_DATA) {
ser_declare;
uint64_t faddr;
char ec1[50];
wbuf = rec->data + OFFSET_FADDR_SIZE;
wsize = rec->data_len - OFFSET_FADDR_SIZE;
ser_begin(rec->data, OFFSET_FADDR_SIZE);
unser_uint64(faddr);
if (fileAddr != faddr) {
fileAddr = faddr;
if (blseek(&bfd, (boffset_t)fileAddr, SEEK_SET) < 0) {
berrno be;
Emsg3(M_ERROR, 0, _("Seek to %s error on %s: ERR=%s\n"),
edit_uint64(fileAddr, ec1), attr->ofname, be.bstrerror());
extract = false;
return true;
}
}
} else {
wbuf = rec->data;
wsize = rec->data_len;
}
while (compress_len < 10000000 && (status = uncompress((Byte *)compress_buf, &compress_len,
(const Byte *)wbuf, (uLong)wsize)) == Z_BUF_ERROR) {
/* The buffer size is too small, try with a bigger one */
compress_len = 2 * compress_len;
compress_buf = check_pool_memory_size(compress_buf,
compress_len);
}
if (status != Z_OK) {
Emsg1(M_ERROR, 0, _("Uncompression error. ERR=%d\n"), status);
extract = false;
return true;
}
Dmsg2(100, "Write uncompressed %d bytes, total before write=%d\n", compress_len, total);
store_data(&bfd, compress_buf, compress_len);
total += compress_len;
fileAddr += compress_len;
Dmsg2(100, "Compress len=%d uncompressed=%d\n", rec->data_len,
compress_len);
}
#else
if (extract) {
Emsg0(M_ERROR, 0, _("GZIP data stream found, but GZIP not configured!\n"));
extract = false;
return true;
}
#endif
break;
/* Compressed data stream */
case STREAM_COMPRESSED_DATA:
case STREAM_SPARSE_COMPRESSED_DATA:
case STREAM_WIN32_COMPRESSED_DATA:
if (extract) {
uint32_t comp_magic, comp_len;
uint16_t comp_level, comp_version;
#ifdef HAVE_LZO
lzo_uint compress_len;
const unsigned char *cbuf;
int r, real_compress_len;
#endif
if (rec->maskedStream == STREAM_SPARSE_COMPRESSED_DATA) {
ser_declare;
uint64_t faddr;
char ec1[50];
wbuf = rec->data + OFFSET_FADDR_SIZE;
wsize = rec->data_len - OFFSET_FADDR_SIZE;
ser_begin(rec->data, OFFSET_FADDR_SIZE);
unser_uint64(faddr);
if (fileAddr != faddr) {
fileAddr = faddr;
if (blseek(&bfd, (boffset_t)fileAddr, SEEK_SET) < 0) {
berrno be;
Emsg3(M_ERROR, 0, _("Seek to %s error on %s: ERR=%s\n"),
edit_uint64(fileAddr, ec1), attr->ofname, be.bstrerror());
extract = false;
return true;
}
}
} else {
wbuf = rec->data;
wsize = rec->data_len;
}
/* read compress header */
unser_declare;
unser_begin(wbuf, sizeof(comp_stream_header));
unser_uint32(comp_magic);
unser_uint32(comp_len);
unser_uint16(comp_level);
unser_uint16(comp_version);
Dmsg4(200, "Compressed data stream found: magic=0x%x, len=%d, level=%d, ver=0x%x\n", comp_magic, comp_len,
comp_level, comp_version);
/* version check */
if (comp_version != COMP_HEAD_VERSION) {
Emsg1(M_ERROR, 0, _("Compressed header version error. version=0x%x\n"), comp_version);
return false;
}
/* size check */
if (comp_len + sizeof(comp_stream_header) != wsize) {
Emsg2(M_ERROR, 0, _("Compressed header size error. comp_len=%d, msglen=%d\n"),
comp_len, wsize);
return false;
}
switch(comp_magic) {
#ifdef HAVE_LZO
case COMPRESS_LZO1X:
compress_len = compress_buf_size;
cbuf = (const unsigned char*) wbuf + sizeof(comp_stream_header);
real_compress_len = wsize - sizeof(comp_stream_header);
Dmsg2(200, "Comp_len=%d msglen=%d\n", compress_len, wsize);
while ((r=lzo1x_decompress_safe(cbuf, real_compress_len,
(unsigned char *)compress_buf, &compress_len, NULL)) == LZO_E_OUTPUT_OVERRUN)
{
/* The buffer size is too small, try with a bigger one */
compress_len = 2 * compress_len;
compress_buf = check_pool_memory_size(compress_buf,
compress_len);
}
if (r != LZO_E_OK) {
Emsg1(M_ERROR, 0, _("LZO uncompression error. ERR=%d\n"), r);
extract = false;
return true;
}
Dmsg2(100, "Write uncompressed %d bytes, total before write=%d\n", compress_len, total);
store_data(&bfd, compress_buf, compress_len);
total += compress_len;
fileAddr += compress_len;
Dmsg2(100, "Compress len=%d uncompressed=%d\n", rec->data_len, compress_len);
break;
#endif
default:
Emsg1(M_ERROR, 0, _("Compression algorithm 0x%x found, but not supported!\n"), comp_magic);
extract = false;
return true;
}
}
break;
case STREAM_MD5_DIGEST:
case STREAM_SHA1_DIGEST:
case STREAM_SHA256_DIGEST:
case STREAM_SHA512_DIGEST:
break;
case STREAM_SIGNED_DIGEST:
case STREAM_ENCRYPTED_SESSION_DATA:
// TODO landonf: Investigate crypto support in the storage daemon
break;
case STREAM_PROGRAM_NAMES:
case STREAM_PROGRAM_DATA:
if (!prog_name_msg) {
Pmsg0(000, _("Got Program Name or Data Stream. Ignored.\n"));
prog_name_msg++;
}
break;
case STREAM_UNIX_ACCESS_ACL: /* Deprecated Standard ACL attributes on UNIX */
case STREAM_UNIX_DEFAULT_ACL: /* Deprecated Default ACL attributes on UNIX */
case STREAM_ACL_AIX_TEXT:
case STREAM_ACL_DARWIN_ACCESS_ACL:
case STREAM_ACL_FREEBSD_DEFAULT_ACL:
case STREAM_ACL_FREEBSD_ACCESS_ACL:
case STREAM_ACL_HPUX_ACL_ENTRY:
case STREAM_ACL_IRIX_DEFAULT_ACL:
case STREAM_ACL_IRIX_ACCESS_ACL:
case STREAM_ACL_LINUX_DEFAULT_ACL:
case STREAM_ACL_LINUX_ACCESS_ACL:
case STREAM_ACL_TRU64_DEFAULT_ACL:
case STREAM_ACL_TRU64_DEFAULT_DIR_ACL:
case STREAM_ACL_TRU64_ACCESS_ACL:
case STREAM_ACL_SOLARIS_ACLENT:
case STREAM_ACL_SOLARIS_ACE:
case STREAM_ACL_AFS_TEXT:
case STREAM_ACL_AIX_AIXC:
case STREAM_ACL_AIX_NFS4:
case STREAM_ACL_FREEBSD_NFS4_ACL:
case STREAM_ACL_HURD_DEFAULT_ACL:
case STREAM_ACL_HURD_ACCESS_ACL:
if (extract) {
wbuf = rec->data;
wsize = rec->data_len;
pm_strcpy(acl_data.last_fname, attr->fname);
parse_acl_streams(jcr, &acl_data, rec->maskedStream, wbuf, wsize);
}
break;
case STREAM_XATTR_HURD:
case STREAM_XATTR_IRIX:
case STREAM_XATTR_TRU64:
case STREAM_XATTR_AIX:
case STREAM_XATTR_OPENBSD:
case STREAM_XATTR_SOLARIS_SYS:
case STREAM_XATTR_SOLARIS:
case STREAM_XATTR_DARWIN:
case STREAM_XATTR_FREEBSD:
case STREAM_XATTR_LINUX:
case STREAM_XATTR_NETBSD:
if (extract) {
wbuf = rec->data;
wsize = rec->data_len;
pm_strcpy(xattr_data.last_fname, attr->fname);
parse_xattr_streams(jcr, &xattr_data, rec->maskedStream, wbuf, wsize);
}
break;
case STREAM_NDMP_SEPERATOR:
break;
default:
/* If extracting, weird stream (not 1 or 2), close output file anyway */
if (extract) {
if (!is_bopen(&bfd)) {
Emsg0(M_ERROR, 0, _("Logic error output file should be open but is not.\n"));
}
set_attributes(jcr, attr, &bfd);
extract = false;
}
Jmsg(jcr, M_ERROR, 0, _("Unknown stream=%d ignored. This shouldn't happen!\n"),
rec->Stream);
break;
} /* end switch */
return true;
}
