/*
* Append Data sent from File daemon
*
*/
bool do_append_data(JCR *jcr)
{
int32_t n;
int32_t file_index, stream, last_file_index;
BSOCK *fd = jcr->file_bsock;
bool ok = true;
DEV_RECORD rec;
char buf1[100], buf2[100];
DCR *dcr = jcr->dcr;
DEVICE *dev;
char ec[50];
if (!dcr) {
Jmsg0(jcr, M_FATAL, 0, _("DCR is NULL!!!\n"));
return false;
}
dev = dcr->dev;
if (!dev) {
Jmsg0(jcr, M_FATAL, 0, _("DEVICE is NULL!!!\n"));
return false;
}
Dmsg1(100, "Start append data. res=%d\n", dev->num_reserved());
memset(&rec, 0, sizeof(rec));
if (!fd->set_buffer_size(dcr->device->max_network_buffer_size, BNET_SETBUF_WRITE)) {
jcr->setJobStatus(JS_ErrorTerminated);
Jmsg0(jcr, M_FATAL, 0, _("Unable to set network buffer size.\n"));
return false;
}
if (!acquire_device_for_append(dcr)) {
jcr->setJobStatus(JS_ErrorTerminated);
return false;
}
jcr->setJobStatus(JS_Running);
dir_send_job_status(jcr);
if (dev->VolCatInfo.VolCatName[0] == 0) {
Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
}
Dmsg1(50, "Begin append device=%s\n", dev->print_name());
begin_data_spool(dcr);
begin_attribute_spool(jcr);
Dmsg0(100, "Just after acquire_device_for_append\n");
if (dev->VolCatInfo.VolCatName[0] == 0) {
Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
}
/*
* Write Begin Session Record
*/
if (!write_session_label(dcr, SOS_LABEL)) {
Jmsg1(jcr, M_FATAL, 0, _("Write session label failed. ERR=%s\n"),
dev->bstrerror());
jcr->setJobStatus(JS_ErrorTerminated);
ok = false;
}
if (dev->VolCatInfo.VolCatName[0] == 0) {
Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
}
/* Tell File daemon to send data */
if (!fd->fsend(OK_data)) {
berrno be;
Jmsg1(jcr, M_FATAL, 0, _("Network send error to FD. ERR=%s\n"),
be.bstrerror(fd->b_errno));
ok = false;
}
/*
* Get Data from File daemon, write to device. To clarify what is
* going on here. We expect:
* - A stream header
* - Multiple records of data
* - EOD record
*
* The Stream header is just used to sychronize things, and
* none of the stream header is written to tape.
* The Multiple records of data, contain first the Attributes,
* then after another stream header, the file data, then
* after another stream header, the MD5 data if any.
*
* So we get the (stream header, data, EOD) three time for each
* file. 1. for the Attributes, 2. for the file data if any,
* and 3. for the MD5 if any.
*/
dcr->VolFirstIndex = dcr->VolLastIndex = 0;
jcr->run_time = time(NULL); /* start counting time for rates */
for (last_file_index = 0; ok && !jcr->is_job_canceled(); ) {
/* Read Stream header from the File daemon.
* The stream header consists of the following:
* file_index (sequential Bacula file index, base 1)
* stream (Bacula number to distinguish parts of data)
* info (Info for Storage daemon -- compressed, encrypted, ...)
* info is not currently used, so is read, but ignored!
*/
if ((n=bget_msg(fd)) <= 0) {
if (n == BNET_SIGNAL && fd->msglen == BNET_EOD) {
break; /* end of data */
}
Jmsg1(jcr, M_FATAL, 0, _("Error reading data header from FD. ERR=%s\n"),
fd->bstrerror());
possible_incomplete_job(jcr, last_file_index);
ok = false;
break;
}
if (sscanf(fd->msg, "%ld %ld", &file_index, &stream) != 2) {
Jmsg1(jcr, M_FATAL, 0, _("Malformed data header from FD: %s\n"), fd->msg);
ok = false;
possible_incomplete_job(jcr, last_file_index);
break;
}
Dmsg2(890, "<filed: Header FilInx=%d stream=%d\n", file_index, stream);
/*
* We make sure the file_index is advancing sequentially.
* An incomplete job can start the file_index at any number.
* otherwise, it must start at 1.
*/
if (jcr->rerunning && file_index > 0 && last_file_index == 0) {
goto fi_checked;
}
if (file_index > 0 && (file_index == last_file_index ||
file_index == last_file_index + 1)) {
goto fi_checked;
}
Jmsg2(jcr, M_FATAL, 0, _("FI=%d from FD not positive or sequential=%d\n"),
file_index, last_file_index);
possible_incomplete_job(jcr, last_file_index);
ok = false;
break;
fi_checked:
if (file_index != last_file_index) {
jcr->JobFiles = file_index;
last_file_index = file_index;
}
/* Read data stream from the File daemon.
* The data stream is just raw bytes
*/
while ((n=bget_msg(fd)) > 0 && !jcr->is_job_canceled()) {
rec.VolSessionId = jcr->VolSessionId;
rec.VolSessionTime = jcr->VolSessionTime;
rec.FileIndex = file_index;
rec.Stream = stream;
rec.maskedStream = stream & STREAMMASK_TYPE; /* strip high bits */
rec.data_len = fd->msglen;
rec.data = fd->msg; /* use message buffer */
Dmsg4(850, "before writ_rec FI=%d SessId=%d Strm=%s len=%d\n",
rec.FileIndex, rec.VolSessionId,
stream_to_ascii(buf1, rec.Stream,rec.FileIndex),
rec.data_len);
while (!write_record_to_block(dcr->block, &rec)) {
Dmsg2(850, "!write_record_to_block data_len=%d rem=%d\n", rec.data_len,
rec.remainder);
if (!write_block_to_device(dcr)) {
Dmsg2(90, "Got write_block_to_dev error on device %s. %s\n",
dev->print_name(), dev->bstrerror());
ok = false;
break;
}
}
if (!ok) {
Dmsg0(400, "Not OK\n");
break;
}
jcr->JobBytes += rec.data_len; /* increment bytes this job */
Dmsg4(850, "write_record FI=%s SessId=%d Strm=%s len=%d\n",
FI_to_ascii(buf1, rec.FileIndex), rec.VolSessionId,
stream_to_ascii(buf2, rec.Stream, rec.FileIndex), rec.data_len);
send_attrs_to_dir(jcr, &rec);
Dmsg0(650, "Enter bnet_get\n");
}
Dmsg1(650, "End read loop with FD. Stat=%d\n", n);
if (fd->is_error()) {
if (!jcr->is_job_canceled()) {
Dmsg1(350, "Network read error from FD. ERR=%s\n", fd->bstrerror());
Jmsg1(jcr, M_FATAL, 0, _("Network error reading from FD. ERR=%s\n"),
fd->bstrerror());
possible_incomplete_job(jcr, last_file_index);
}
ok = false;
break;
}
}
/* Create Job status for end of session label */
jcr->setJobStatus(ok?JS_Terminated:JS_ErrorTerminated);
if (ok) {
/* Terminate connection with FD */
fd->fsend(OK_append);
do_fd_commands(jcr); /* finish dialog with FD */
} else {
fd->fsend("3999 Failed append\n");
}
/*
* Don't use time_t for job_elapsed as time_t can be 32 or 64 bits,
* and the subsequent Jmsg() editing will break
*/
int32_t job_elapsed = time(NULL) - jcr->run_time;
if (job_elapsed <= 0) {
job_elapsed = 1;
}
Jmsg(dcr->jcr, M_INFO, 0, _("Job write elapsed time = %02d:%02d:%02d, Transfer rate = %s Bytes/second\n"),
job_elapsed / 3600, job_elapsed % 3600 / 60, job_elapsed % 60,
edit_uint64_with_suffix(jcr->JobBytes / job_elapsed, ec));
Dmsg1(200, "Write EOS label JobStatus=%c\n", jcr->JobStatus);
/*
* Check if we can still write. This may not be the case
* if we are at the end of the tape or we got a fatal I/O error.
*/
if (ok || dev->can_write()) {
if (!write_session_label(dcr, EOS_LABEL)) {
/* Print only if ok and not cancelled to avoid spurious messages */
if (ok && !jcr->is_job_canceled()) {
Jmsg1(jcr, M_FATAL, 0, _("Error writing end session label. ERR=%s\n"),
dev->bstrerror());
possible_incomplete_job(jcr, last_file_index);
}
jcr->setJobStatus(JS_ErrorTerminated);
ok = false;
}
if (dev->VolCatInfo.VolCatName[0] == 0) {
Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
Dmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
}
Dmsg0(90, "back from write_end_session_label()\n");
/* Flush out final partial block of this session */
if (!write_block_to_device(dcr)) {
/* Print only if ok and not cancelled to avoid spurious messages */
if (ok && !jcr->is_job_canceled()) {
Jmsg2(jcr, M_FATAL, 0, _("Fatal append error on device %s: ERR=%s\n"),
dev->print_name(), dev->bstrerror());
Dmsg0(100, _("Set ok=FALSE after write_block_to_device.\n"));
possible_incomplete_job(jcr, last_file_index);
}
jcr->setJobStatus(JS_ErrorTerminated);
ok = false;
}
}
if (!ok && !jcr->is_JobStatus(JS_Incomplete)) {
discard_data_spool(dcr);
} else {
/* Note: if commit is OK, the device will remain blocked */
commit_data_spool(dcr);
}
if (ok) {
ok = dvd_close_job(dcr); /* do DVD cleanup if any */
}
/*
* Release the device -- and send final Vol info to DIR
* and unlock it.
*/
release_device(dcr);
if ((!ok || jcr->is_job_canceled()) && !jcr->is_JobStatus(JS_Incomplete)) {
discard_attribute_spool(jcr);
} else {
commit_attribute_spool(jcr);
}
dir_send_job_status(jcr); /* update director */
Dmsg1(100, "return from do_append_data() ok=%d\n", ok);
return ok;
}
/**
* Find all the requested files and send them
* to the Storage daemon.
*
* Note, we normally carry on a one-way
* conversation from this point on with the SD, simply blasting
* data to him. To properly know what is going on, we
* also run a "heartbeat" monitor which reads the socket and
* reacts accordingly (at the moment it has nothing to do
* except echo the heartbeat to the Director).
*/
bool blast_data_to_storage_daemon(JCR *jcr, char *addr, crypto_cipher_t cipher)
{
BSOCK *sd;
bool ok = true;
sd = jcr->store_bsock;
jcr->setJobStatus(JS_Running);
Dmsg1(300, "filed: opened data connection %d to stored\n", sd->m_fd);
LockRes();
CLIENTRES *client = (CLIENTRES *)GetNextRes(R_CLIENT, NULL);
UnlockRes();
uint32_t buf_size;
if (client) {
buf_size = client->max_network_buffer_size;
} else {
buf_size = 0; /* use default */
}
if (!sd->set_buffer_size(buf_size, BNET_SETBUF_WRITE)) {
jcr->setJobStatus(JS_ErrorTerminated);
Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
return false;
}
jcr->buf_size = sd->msglen;
if (!adjust_compression_buffers(jcr)) {
return false;
}
if (!crypto_session_start(jcr, cipher)) {
return false;
}
set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
/**
* In accurate mode, we overload the find_one check function
*/
if (jcr->accurate) {
set_find_changed_function((FF_PKT *)jcr->ff, accurate_check_file);
}
start_heartbeat_monitor(jcr);
if (have_acl) {
jcr->acl_data = (acl_data_t *)malloc(sizeof(acl_data_t));
memset(jcr->acl_data, 0, sizeof(acl_data_t));
jcr->acl_data->u.build = (acl_build_data_t *)malloc(sizeof(acl_build_data_t));
memset(jcr->acl_data->u.build, 0, sizeof(acl_build_data_t));
jcr->acl_data->u.build->content = get_pool_memory(PM_MESSAGE);
}
if (have_xattr) {
jcr->xattr_data = (xattr_data_t *)malloc(sizeof(xattr_data_t));
memset(jcr->xattr_data, 0, sizeof(xattr_data_t));
jcr->xattr_data->u.build = (xattr_build_data_t *)malloc(sizeof(xattr_build_data_t));
memset(jcr->xattr_data->u.build, 0, sizeof(xattr_build_data_t));
jcr->xattr_data->u.build->content = get_pool_memory(PM_MESSAGE);
}
/**
* Subroutine save_file() is called for each file
*/
if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, plugin_save)) {
ok = false; /* error */
jcr->setJobStatus(JS_ErrorTerminated);
}
if (have_acl && jcr->acl_data->u.build->nr_errors > 0) {
Jmsg(jcr, M_WARNING, 0, _("Encountered %ld acl errors while doing backup\n"),
jcr->acl_data->u.build->nr_errors);
}
if (have_xattr && jcr->xattr_data->u.build->nr_errors > 0) {
Jmsg(jcr, M_WARNING, 0, _("Encountered %ld xattr errors while doing backup\n"),
jcr->xattr_data->u.build->nr_errors);
}
close_vss_backup_session(jcr);
accurate_finish(jcr); /* send deleted or base file list to SD */
stop_heartbeat_monitor(jcr);
sd->signal(BNET_EOD); /* end of sending data */
if (have_acl && jcr->acl_data) {
free_pool_memory(jcr->acl_data->u.build->content);
free(jcr->acl_data->u.build);
free(jcr->acl_data);
jcr->acl_data = NULL;
}
if (have_xattr && jcr->xattr_data) {
free_pool_memory(jcr->xattr_data->u.build->content);
free(jcr->xattr_data->u.build);
free(jcr->xattr_data);
jcr->xattr_data = NULL;
}
if (jcr->big_buf) {
free(jcr->big_buf);
jcr->big_buf = NULL;
}
cleanup_compression(jcr);
crypto_session_end(jcr);
Dmsg1(100, "end blast_data ok=%d\n", ok);
return ok;
}
/*
* Read Data and commit to new job.
*/
bool do_mac_run(JCR *jcr)
{
DEVICE *dev;
char ec1[50];
const char *Type;
bool ok = true;
BSOCK *dir = jcr->dir_bsock;
switch(jcr->getJobType()) {
case JT_MIGRATE:
Type = "Migration";
break;
case JT_ARCHIVE:
Type = "Archive";
break;
case JT_COPY:
Type = "Copy";
break;
case JT_BACKUP:
Type = "Virtual Backup";
break;
default:
Type = "Unknown";
break;
}
Dmsg0(20, "Start read data.\n");
if (jcr->NumReadVolumes == 0) {
Jmsg(jcr, M_FATAL, 0, _("No Volume names found for %s.\n"), Type);
goto bail_out;
}
/*
* Check autoinflation/autodeflation settings.
*/
check_auto_xflation(jcr);
/*
* See if we perform both read and write or read only.
*/
if (jcr->remote_replicate) {
BSOCK *sd;
if (!jcr->read_dcr) {
Jmsg(jcr, M_FATAL, 0, _("Read device not properly initialized.\n"));
goto bail_out;
}
Dmsg1(100, "read_dcr=%p\n", jcr->read_dcr);
Dmsg3(200, "Found %d volumes names for %s. First=%s\n",
jcr->NumReadVolumes, Type, jcr->VolList->VolumeName);
/*
* Ready devices for reading.
*/
if (!acquire_device_for_read(jcr->read_dcr)) {
ok = false;
goto bail_out;
}
Dmsg2(200, "===== After acquire pos %u:%u\n", jcr->read_dcr->dev->file, jcr->read_dcr->dev->block_num);
jcr->sendJobStatus(JS_Running);
/*
* Set network buffering.
*/
sd = jcr->store_bsock;
if (!sd->set_buffer_size(me->max_network_buffer_size, BNET_SETBUF_WRITE)) {
Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size SD->SD.\n"));
ok = false;
goto bail_out;
}
/*
* Let the remote SD know we are about to start the replication.
*/
sd->fsend(start_replicate);
Dmsg1(110, ">stored: %s", sd->msg);
/*
* Expect to receive back the Ticket number.
*/
if (bget_msg(sd) >= 0) {
Dmsg1(110, "<stored: %s", sd->msg);
if (sscanf(sd->msg, OK_start_replicate, &jcr->Ticket) != 1) {
Jmsg(jcr, M_FATAL, 0, _("Bad response to start replicate: %s\n"), sd->msg);
goto bail_out;
}
Dmsg1(110, "Got Ticket=%d\n", jcr->Ticket);
} else {
Jmsg(jcr, M_FATAL, 0, _("Bad response from stored to start replicate command\n"));
goto bail_out;
}
/*
* Let the remote SD know we are now really going to send the data.
*/
sd->fsend(replicate_data, jcr->Ticket);
Dmsg1(110, ">stored: %s", sd->msg);
/*
* Expect to get response to the replicate data cmd from Storage daemon
*/
if (!response(jcr, sd, OK_data, "replicate data")) {
ok = false;
goto bail_out;
}
/*
* Read all data and send it to remote SD.
*/
ok = read_records(jcr->read_dcr, clone_record_to_remote_sd, mount_next_read_volume);
/*
* Send the last EOD to close the last data transfer and a next EOD to
* signal the remote we are done.
*/
if (!sd->signal(BNET_EOD) || !sd->signal(BNET_EOD)) {
if (!jcr->is_job_canceled()) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
sd->bstrerror());
}
goto bail_out;
}
/*
* Expect to get response that the replicate data succeeded.
*/
if (!response(jcr, sd, OK_replicate, "replicate data")) {
ok = false;
goto bail_out;
}
/*
* End replicate session.
*/
sd->fsend(end_replicate);
Dmsg1(110, ">stored: %s", sd->msg);
/*
* Expect to get response to the end replicate cmd from Storage daemon
*/
if (!response(jcr, sd, OK_end_replicate, "end replicate")) {
ok = false;
goto bail_out;
}
/* Inform Storage daemon that we are done */
sd->signal(BNET_TERMINATE);
} else {
if (!jcr->read_dcr || !jcr->dcr) {
Jmsg(jcr, M_FATAL, 0, _("Read and write devices not properly initialized.\n"));
goto bail_out;
}
Dmsg2(100, "read_dcr=%p write_dcr=%p\n", jcr->read_dcr, jcr->dcr);
Dmsg3(200, "Found %d volumes names for %s. First=%s\n",
jcr->NumReadVolumes, Type, jcr->VolList->VolumeName);
/*
* Ready devices for reading and writing.
*/
if (!acquire_device_for_read(jcr->read_dcr) ||
!acquire_device_for_append(jcr->dcr)) {
ok = false;
goto bail_out;
}
Dmsg2(200, "===== After acquire pos %u:%u\n", jcr->dcr->dev->file, jcr->dcr->dev->block_num);
jcr->sendJobStatus(JS_Running);
if (!begin_data_spool(jcr->dcr) ) {
ok = false;
goto bail_out;
}
if (!begin_attribute_spool(jcr)) {
ok = false;
goto bail_out;
}
jcr->dcr->VolFirstIndex = jcr->dcr->VolLastIndex = 0;
jcr->run_time = time(NULL);
set_start_vol_position(jcr->dcr);
jcr->JobFiles = 0;
/*
* Read all data and make a local clone of it.
*/
ok = read_records(jcr->read_dcr, clone_record_internally, mount_next_read_volume);
}
bail_out:
if (!ok) {
jcr->setJobStatus(JS_ErrorTerminated);
}
if (!jcr->remote_replicate && jcr->dcr) {
/*
* Don't use time_t for job_elapsed as time_t can be 32 or 64 bits,
* and the subsequent Jmsg() editing will break
*/
int32_t job_elapsed;
dev = jcr->dcr->dev;
Dmsg1(100, "ok=%d\n", ok);
if (ok || dev->can_write()) {
/*
* Flush out final partial block of this session
*/
if (!jcr->dcr->write_block_to_device()) {
Jmsg2(jcr, M_FATAL, 0, _("Fatal append error on device %s: ERR=%s\n"),
dev->print_name(), dev->bstrerror());
Dmsg0(100, _("Set ok=FALSE after write_block_to_device.\n"));
ok = false;
}
Dmsg2(200, "Flush block to device pos %u:%u\n", dev->file, dev->block_num);
}
if (!ok) {
discard_data_spool(jcr->dcr);
} else {
/*
* Note: if commit is OK, the device will remain blocked
*/
commit_data_spool(jcr->dcr);
}
job_elapsed = time(NULL) - jcr->run_time;
if (job_elapsed <= 0) {
job_elapsed = 1;
}
Jmsg(jcr, M_INFO, 0, _("Elapsed time=%02d:%02d:%02d, Transfer rate=%s Bytes/second\n"),
job_elapsed / 3600, job_elapsed % 3600 / 60, job_elapsed % 60,
edit_uint64_with_suffix(jcr->JobBytes / job_elapsed, ec1));
/*
* Release the device -- and send final Vol info to DIR
*/
release_device(jcr->dcr);
if (!ok || job_canceled(jcr)) {
discard_attribute_spool(jcr);
} else {
commit_attribute_spool(jcr);
}
}
if (jcr->read_dcr) {
if (!release_device(jcr->read_dcr)) {
ok = false;
}
}
jcr->sendJobStatus(); /* update director */
Dmsg0(30, "Done reading.\n");
jcr->end_time = time(NULL);
dequeue_messages(jcr); /* send any queued messages */
if (ok) {
jcr->setJobStatus(JS_Terminated);
}
generate_plugin_event(jcr, bsdEventJobEnd);
dir->fsend(Job_end, jcr->Job, jcr->JobStatus, jcr->JobFiles,
edit_uint64(jcr->JobBytes, ec1), jcr->JobErrors);
Dmsg4(100, Job_end, jcr->Job, jcr->JobStatus, jcr->JobFiles, ec1);
dir->signal(BNET_EOD); /* send EOD to Director daemon */
free_plugins(jcr); /* release instantiated plugins */
return false; /* Continue DIR session ? */
}
/*
* Append Data sent from Client (FD/SD)
*
*/
bool do_append_data(JCR *jcr)
{
int32_t n;
int32_t file_index, stream, last_file_index;
uint64_t stream_len;
BSOCK *fd = jcr->file_bsock;
bool ok = true;
DEV_RECORD rec;
char buf1[100], buf2[100];
DCR *dcr = jcr->dcr;
DEVICE *dev;
char ec[50];
POOLMEM *eblock = NULL;
POOL_MEM errmsg(PM_EMSG);
if (!dcr) {
pm_strcpy(jcr->errmsg, _("DCR is NULL!!!\n"));
Jmsg0(jcr, M_FATAL, 0, jcr->errmsg);
return false;
}
dev = dcr->dev;
if (!dev) {
pm_strcpy(jcr->errmsg, _("DEVICE is NULL!!!\n"));
Jmsg0(jcr, M_FATAL, 0, jcr->errmsg);
return false;
}
Dmsg1(100, "Start append data. res=%d\n", dev->num_reserved());
memset(&rec, 0, sizeof(rec));
if (!fd->set_buffer_size(dcr->device->max_network_buffer_size, BNET_SETBUF_WRITE)) {
jcr->setJobStatus(JS_ErrorTerminated);
pm_strcpy(jcr->errmsg, _("Unable to set network buffer size.\n"));
Jmsg0(jcr, M_FATAL, 0, jcr->errmsg);
return false;
}
if (!acquire_device_for_append(dcr)) {
jcr->setJobStatus(JS_ErrorTerminated);
return false;
}
jcr->sendJobStatus(JS_Running);
//ASSERT(dev->VolCatInfo.VolCatName[0]);
if (dev->VolCatInfo.VolCatName[0] == 0) {
Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
}
Dmsg1(50, "Begin append device=%s\n", dev->print_name());
begin_data_spool(dcr);
begin_attribute_spool(jcr);
Dmsg0(100, "Just after acquire_device_for_append\n");
//ASSERT(dev->VolCatInfo.VolCatName[0]);
if (dev->VolCatInfo.VolCatName[0] == 0) {
Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
}
/*
* Write Begin Session Record
*/
if (!write_session_label(dcr, SOS_LABEL)) {
Jmsg1(jcr, M_FATAL, 0, _("Write session label failed. ERR=%s\n"),
dev->bstrerror());
jcr->setJobStatus(JS_ErrorTerminated);
ok = false;
}
//ASSERT(dev->VolCatInfo.VolCatName[0]);
if (dev->VolCatInfo.VolCatName[0] == 0) {
Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
}
/* Tell File daemon to send data */
if (!fd->fsend(OK_data)) {
berrno be;
Jmsg1(jcr, M_FATAL, 0, _("Network send error to FD. ERR=%s\n"),
be.bstrerror(fd->b_errno));
ok = false;
}
/*
* Get Data from File daemon, write to device. To clarify what is
* going on here. We expect:
* - A stream header
* - Multiple records of data
* - EOD record
*
* The Stream header is just used to synchronize things, and
* none of the stream header is written to tape.
* The Multiple records of data, contain first the Attributes,
* then after another stream header, the file data, then
* after another stream header, the MD5 data if any.
*
* So we get the (stream header, data, EOD) three time for each
* file. 1. for the Attributes, 2. for the file data if any,
* and 3. for the MD5 if any.
*/
dcr->VolFirstIndex = dcr->VolLastIndex = 0;
jcr->run_time = time(NULL); /* start counting time for rates */
GetMsg *qfd;
qfd = New(GetMsg(jcr, fd, NULL, GETMSG_MAX_MSG_SIZE));
qfd->start_read_sock();
for (last_file_index = 0; ok && !jcr->is_job_canceled(); ) {
/* Read Stream header from the File daemon.
* The stream header consists of the following:
* file_index (sequential Bacula file index, base 1)
* stream (Bacula number to distinguish parts of data)
* stream_len (Expected length of this stream. This
* will be the size backed up if the file does not
* grow during the backup.
*/
n = qfd->bget_msg(NULL);
if (n <= 0) {
if (n == BNET_SIGNAL && qfd->msglen == BNET_EOD) {
Dmsg0(200, "Got EOD on reading header.\n");
break; /* end of data */
}
Jmsg3(jcr, M_FATAL, 0, _("Error reading data header from FD. n=%d msglen=%d ERR=%s\n"),
n, qfd->msglen, fd->bstrerror());
// ASX TODO the fd->bstrerror() can be related to the wrong error, I should Queue the error too
possible_incomplete_job(jcr, last_file_index);
ok = false;
break;
}
if (sscanf(qfd->msg, "%ld %ld %lld", &file_index, &stream, &stream_len) != 3) {
// TODO ASX already done in bufmsg, should reuse the values
char buf[256];
Jmsg1(jcr, M_FATAL, 0, _("Malformed data header from FD: %s\n"), asciidump(qfd->msg, qfd->msglen, buf, sizeof(buf)));
ok = false;
possible_incomplete_job(jcr, last_file_index);
break;
}
Dmsg3(890, "<filed: Header FilInx=%d stream=%d stream_len=%lld\n",
file_index, stream, stream_len);
/*
* We make sure the file_index is advancing sequentially.
* An incomplete job can start the file_index at any number.
* otherwise, it must start at 1.
*/
if (jcr->rerunning && file_index > 0 && last_file_index == 0) {
goto fi_checked;
}
Dmsg2(400, "file_index=%d last_file_index=%d\n", file_index, last_file_index);
if (file_index > 0 && (file_index == last_file_index ||
file_index == last_file_index + 1)) {
goto fi_checked;
}
Jmsg2(jcr, M_FATAL, 0, _("FI=%d from FD not positive or last_FI=%d\n"),
file_index, last_file_index);
possible_incomplete_job(jcr, last_file_index);
ok = false;
break;
fi_checked:
if (file_index != last_file_index) {
jcr->JobFiles = file_index;
last_file_index = file_index;
}
/* Read data stream from the File daemon.
* The data stream is just raw bytes
*/
while ((n=qfd->bget_msg(NULL)) > 0 && !jcr->is_job_canceled()) {
rec.VolSessionId = jcr->VolSessionId;
rec.VolSessionTime = jcr->VolSessionTime;
rec.FileIndex = file_index;
rec.Stream = stream;
rec.StreamLen = stream_len;
rec.maskedStream = stream & STREAMMASK_TYPE; /* strip high bits */
rec.data_len = qfd->msglen;
rec.data = qfd->msg; /* use message buffer */
Dmsg4(850, "before writ_rec FI=%d SessId=%d Strm=%s len=%d\n",
rec.FileIndex, rec.VolSessionId,
stream_to_ascii(buf1, rec.Stream,rec.FileIndex),
rec.data_len);
ok = dcr->write_record(&rec);
if (!ok) {
Dmsg2(90, "Got write_block_to_dev error on device %s. %s\n",
dcr->dev->print_name(), dcr->dev->bstrerror());
break;
}
jcr->JobBytes += rec.data_len; /* increment bytes this job */
jcr->JobBytes += qfd->bmsg->jobbytes; // if the block as been downloaded, count it
Dmsg4(850, "write_record FI=%s SessId=%d Strm=%s len=%d\n",
FI_to_ascii(buf1, rec.FileIndex), rec.VolSessionId,
stream_to_ascii(buf2, rec.Stream, rec.FileIndex), rec.data_len);
send_attrs_to_dir(jcr, &rec);
Dmsg0(650, "Enter bnet_get\n");
}
Dmsg2(650, "End read loop with FD. JobFiles=%d Stat=%d\n", jcr->JobFiles, n);
if (fd->is_error()) {
if (!jcr->is_job_canceled()) {
Dmsg1(350, "Network read error from FD. ERR=%s\n", fd->bstrerror());
Jmsg1(jcr, M_FATAL, 0, _("Network error reading from FD. ERR=%s\n"),
fd->bstrerror());
possible_incomplete_job(jcr, last_file_index);
}
ok = false;
break;
}
}
qfd->wait_read_sock();
free_GetMsg(qfd);
if (eblock != NULL) {
free_pool_memory(eblock);
}
/* Create Job status for end of session label */
jcr->setJobStatus(ok?JS_Terminated:JS_ErrorTerminated);
if (ok) {
/* Terminate connection with Client */
fd->fsend(OK_append);
do_client_commands(jcr); /* finish dialog with Client */
} else {
fd->fsend("3999 Failed append\n");
}
Dmsg1(200, "Write EOS label JobStatus=%c\n", jcr->JobStatus);
/*
* Check if we can still write. This may not be the case
* if we are at the end of the tape or we got a fatal I/O error.
*/
if (ok || dev->can_write()) {
if (!write_session_label(dcr, EOS_LABEL)) {
/* Print only if ok and not cancelled to avoid spurious messages */
if (ok && !jcr->is_job_canceled()) {
Jmsg1(jcr, M_FATAL, 0, _("Error writing end session label. ERR=%s\n"),
dev->bstrerror());
possible_incomplete_job(jcr, last_file_index);
}
jcr->setJobStatus(JS_ErrorTerminated);
ok = false;
}
/* Flush out final partial block of this session */
if (!dcr->write_final_block_to_device()) {
/* Print only if ok and not cancelled to avoid spurious messages */
if (ok && !jcr->is_job_canceled()) {
Jmsg2(jcr, M_FATAL, 0, _("Fatal append error on device %s: ERR=%s\n"),
dev->print_name(), dev->bstrerror());
Dmsg0(100, _("Set ok=FALSE after write_final_block_to_device.\n"));
possible_incomplete_job(jcr, last_file_index);
}
jcr->setJobStatus(JS_ErrorTerminated);
ok = false;
}
}
flush_jobmedia_queue(jcr);
if (!ok && !jcr->is_JobStatus(JS_Incomplete)) {
discard_data_spool(dcr);
} else {
/* Note: if commit is OK, the device will remain blocked */
commit_data_spool(dcr);
}
/*
* Don't use time_t for job_elapsed as time_t can be 32 or 64 bits,
* and the subsequent Jmsg() editing will break
*/
int32_t job_elapsed = time(NULL) - jcr->run_time;
if (job_elapsed <= 0) {
job_elapsed = 1;
}
Jmsg(dcr->jcr, M_INFO, 0, _("Elapsed time=%02d:%02d:%02d, Transfer rate=%s Bytes/second\n"),
job_elapsed / 3600, job_elapsed % 3600 / 60, job_elapsed % 60,
edit_uint64_with_suffix(jcr->JobBytes / job_elapsed, ec));
/*
* Release the device -- and send final Vol info to DIR
* and unlock it.
*/
release_device(dcr);
if ((!ok || jcr->is_job_canceled()) && !jcr->is_JobStatus(JS_Incomplete)) {
discard_attribute_spool(jcr);
} else {
commit_attribute_spool(jcr);
}
jcr->sendJobStatus(); /* update director */
Dmsg1(100, "return from do_append_data() ok=%d\n", ok);
return ok;
}
/*
* Find all the requested files and send them
* to the Storage daemon.
*
* Note, we normally carry on a one-way
* conversation from this point on with the SD, simply blasting
* data to him. To properly know what is going on, we
* also run a "heartbeat" monitor which reads the socket and
* reacts accordingly (at the moment it has nothing to do
* except echo the heartbeat to the Director).
*
*/
bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
{
BSOCK *sd;
bool ok = true;
// TODO landonf: Allow user to specify encryption algorithm
sd = jcr->store_bsock;
set_jcr_job_status(jcr, JS_Running);
Dmsg1(300, "bfiled: opened data connection %d to stored\n", sd->m_fd);
LockRes();
CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
UnlockRes();
uint32_t buf_size;
if (client) {
buf_size = client->max_network_buffer_size;
} else {
buf_size = 0; /* use default */
}
if (!sd->set_buffer_size(buf_size, BNET_SETBUF_WRITE)) {
set_jcr_job_status(jcr, JS_ErrorTerminated);
Jmsg(jcr, M_FATAL, 0, _("Cannot set buffer size FD->SD.\n"));
return false;
}
jcr->buf_size = sd->msglen;
/* Adjust for compression so that output buffer is
* 12 bytes + 0.1% larger than input buffer plus 18 bytes.
* This gives a bit extra plus room for the sparse addr if any.
* Note, we adjust the read size to be smaller so that the
* same output buffer can be used without growing it.
*
* The zlib compression workset is initialized here to minimize
* the "per file" load. The jcr member is only set, if the init
* was successful.
*/
jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30;
jcr->compress_buf = get_memory(jcr->compress_buf_size);
#ifdef HAVE_LIBZ
z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream));
if (pZlibStream) {
pZlibStream->zalloc = Z_NULL;
pZlibStream->zfree = Z_NULL;
pZlibStream->opaque = Z_NULL;
pZlibStream->state = Z_NULL;
if (deflateInit(pZlibStream, Z_DEFAULT_COMPRESSION) == Z_OK) {
jcr->pZLIB_compress_workset = pZlibStream;
} else {
free (pZlibStream);
}
}
#endif
if (!crypto_session_start(jcr)) {
return false;
}
set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime);
/* in accurate mode, we overwrite the find_one check function */
if (jcr->accurate) {
set_find_changed_function((FF_PKT *)jcr->ff, accurate_check_file);
}
start_heartbeat_monitor(jcr);
jcr->acl_data = get_pool_memory(PM_MESSAGE);
jcr->xattr_data = get_pool_memory(PM_MESSAGE);
/* Subroutine save_file() is called for each file */
if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, plugin_save)) {
ok = false; /* error */
set_jcr_job_status(jcr, JS_ErrorTerminated);
}
accurate_send_deleted_list(jcr); /* send deleted list to SD */
stop_heartbeat_monitor(jcr);
sd->signal(BNET_EOD); /* end of sending data */
if (jcr->acl_data) {
free_pool_memory(jcr->acl_data);
jcr->acl_data = NULL;
}
if (jcr->xattr_data) {
free_pool_memory(jcr->xattr_data);
jcr->xattr_data = NULL;
}
if (jcr->big_buf) {
free(jcr->big_buf);
jcr->big_buf = NULL;
}
if (jcr->compress_buf) {
free_pool_memory(jcr->compress_buf);
jcr->compress_buf = NULL;
}
if (jcr->pZLIB_compress_workset) {
/* Free the zlib stream */
#ifdef HAVE_LIBZ
deflateEnd((z_stream *)jcr->pZLIB_compress_workset);
#endif
free (jcr->pZLIB_compress_workset);
jcr->pZLIB_compress_workset = NULL;
}
crypto_session_end(jcr);
Dmsg1(100, "end blast_data ok=%d\n", ok);
return ok;
}