/*
* Append Data sent from File daemon
*
*/
bool do_append_data(JCR *jcr)
{
int32_t n;
int32_t file_index, stream, last_file_index;
BSOCK *fd = jcr->file_bsock;
bool ok = true;
DEV_RECORD rec;
char buf1[100], buf2[100];
DCR *dcr = jcr->dcr;
DEVICE *dev;
char ec[50];
if (!dcr) {
Jmsg0(jcr, M_FATAL, 0, _("DCR is NULL!!!\n"));
return false;
}
dev = dcr->dev;
if (!dev) {
Jmsg0(jcr, M_FATAL, 0, _("DEVICE is NULL!!!\n"));
return false;
}
Dmsg1(100, "Start append data. res=%d\n", dev->num_reserved());
memset(&rec, 0, sizeof(rec));
if (!fd->set_buffer_size(dcr->device->max_network_buffer_size, BNET_SETBUF_WRITE)) {
jcr->setJobStatus(JS_ErrorTerminated);
Jmsg0(jcr, M_FATAL, 0, _("Unable to set network buffer size.\n"));
return false;
}
if (!acquire_device_for_append(dcr)) {
jcr->setJobStatus(JS_ErrorTerminated);
return false;
}
jcr->setJobStatus(JS_Running);
dir_send_job_status(jcr);
if (dev->VolCatInfo.VolCatName[0] == 0) {
Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
}
Dmsg1(50, "Begin append device=%s\n", dev->print_name());
begin_data_spool(dcr);
begin_attribute_spool(jcr);
Dmsg0(100, "Just after acquire_device_for_append\n");
if (dev->VolCatInfo.VolCatName[0] == 0) {
Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
}
/*
* Write Begin Session Record
*/
if (!write_session_label(dcr, SOS_LABEL)) {
Jmsg1(jcr, M_FATAL, 0, _("Write session label failed. ERR=%s\n"),
dev->bstrerror());
jcr->setJobStatus(JS_ErrorTerminated);
ok = false;
}
if (dev->VolCatInfo.VolCatName[0] == 0) {
Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
}
/* Tell File daemon to send data */
if (!fd->fsend(OK_data)) {
berrno be;
Jmsg1(jcr, M_FATAL, 0, _("Network send error to FD. ERR=%s\n"),
be.bstrerror(fd->b_errno));
ok = false;
}
/*
* Get Data from File daemon, write to device. To clarify what is
* going on here. We expect:
* - A stream header
* - Multiple records of data
* - EOD record
*
* The Stream header is just used to sychronize things, and
* none of the stream header is written to tape.
* The Multiple records of data, contain first the Attributes,
* then after another stream header, the file data, then
* after another stream header, the MD5 data if any.
*
* So we get the (stream header, data, EOD) three time for each
* file. 1. for the Attributes, 2. for the file data if any,
* and 3. for the MD5 if any.
*/
dcr->VolFirstIndex = dcr->VolLastIndex = 0;
jcr->run_time = time(NULL); /* start counting time for rates */
for (last_file_index = 0; ok && !jcr->is_job_canceled(); ) {
/* Read Stream header from the File daemon.
* The stream header consists of the following:
* file_index (sequential Bacula file index, base 1)
* stream (Bacula number to distinguish parts of data)
* info (Info for Storage daemon -- compressed, encrypted, ...)
* info is not currently used, so is read, but ignored!
*/
if ((n=bget_msg(fd)) <= 0) {
if (n == BNET_SIGNAL && fd->msglen == BNET_EOD) {
break; /* end of data */
}
Jmsg1(jcr, M_FATAL, 0, _("Error reading data header from FD. ERR=%s\n"),
fd->bstrerror());
possible_incomplete_job(jcr, last_file_index);
ok = false;
break;
}
if (sscanf(fd->msg, "%ld %ld", &file_index, &stream) != 2) {
Jmsg1(jcr, M_FATAL, 0, _("Malformed data header from FD: %s\n"), fd->msg);
ok = false;
possible_incomplete_job(jcr, last_file_index);
break;
}
Dmsg2(890, "<filed: Header FilInx=%d stream=%d\n", file_index, stream);
/*
* We make sure the file_index is advancing sequentially.
* An incomplete job can start the file_index at any number.
* otherwise, it must start at 1.
*/
if (jcr->rerunning && file_index > 0 && last_file_index == 0) {
goto fi_checked;
}
if (file_index > 0 && (file_index == last_file_index ||
file_index == last_file_index + 1)) {
goto fi_checked;
}
Jmsg2(jcr, M_FATAL, 0, _("FI=%d from FD not positive or sequential=%d\n"),
file_index, last_file_index);
possible_incomplete_job(jcr, last_file_index);
ok = false;
break;
fi_checked:
if (file_index != last_file_index) {
jcr->JobFiles = file_index;
last_file_index = file_index;
}
/* Read data stream from the File daemon.
* The data stream is just raw bytes
*/
while ((n=bget_msg(fd)) > 0 && !jcr->is_job_canceled()) {
rec.VolSessionId = jcr->VolSessionId;
rec.VolSessionTime = jcr->VolSessionTime;
rec.FileIndex = file_index;
rec.Stream = stream;
rec.maskedStream = stream & STREAMMASK_TYPE; /* strip high bits */
rec.data_len = fd->msglen;
rec.data = fd->msg; /* use message buffer */
Dmsg4(850, "before writ_rec FI=%d SessId=%d Strm=%s len=%d\n",
rec.FileIndex, rec.VolSessionId,
stream_to_ascii(buf1, rec.Stream,rec.FileIndex),
rec.data_len);
while (!write_record_to_block(dcr->block, &rec)) {
Dmsg2(850, "!write_record_to_block data_len=%d rem=%d\n", rec.data_len,
rec.remainder);
if (!write_block_to_device(dcr)) {
Dmsg2(90, "Got write_block_to_dev error on device %s. %s\n",
dev->print_name(), dev->bstrerror());
ok = false;
break;
}
}
if (!ok) {
Dmsg0(400, "Not OK\n");
break;
}
jcr->JobBytes += rec.data_len; /* increment bytes this job */
Dmsg4(850, "write_record FI=%s SessId=%d Strm=%s len=%d\n",
FI_to_ascii(buf1, rec.FileIndex), rec.VolSessionId,
stream_to_ascii(buf2, rec.Stream, rec.FileIndex), rec.data_len);
send_attrs_to_dir(jcr, &rec);
Dmsg0(650, "Enter bnet_get\n");
}
Dmsg1(650, "End read loop with FD. Stat=%d\n", n);
if (fd->is_error()) {
if (!jcr->is_job_canceled()) {
Dmsg1(350, "Network read error from FD. ERR=%s\n", fd->bstrerror());
Jmsg1(jcr, M_FATAL, 0, _("Network error reading from FD. ERR=%s\n"),
fd->bstrerror());
possible_incomplete_job(jcr, last_file_index);
}
ok = false;
break;
}
}
/* Create Job status for end of session label */
jcr->setJobStatus(ok?JS_Terminated:JS_ErrorTerminated);
if (ok) {
/* Terminate connection with FD */
fd->fsend(OK_append);
do_fd_commands(jcr); /* finish dialog with FD */
} else {
fd->fsend("3999 Failed append\n");
}
/*
* Don't use time_t for job_elapsed as time_t can be 32 or 64 bits,
* and the subsequent Jmsg() editing will break
*/
int32_t job_elapsed = time(NULL) - jcr->run_time;
if (job_elapsed <= 0) {
job_elapsed = 1;
}
Jmsg(dcr->jcr, M_INFO, 0, _("Job write elapsed time = %02d:%02d:%02d, Transfer rate = %s Bytes/second\n"),
job_elapsed / 3600, job_elapsed % 3600 / 60, job_elapsed % 60,
edit_uint64_with_suffix(jcr->JobBytes / job_elapsed, ec));
Dmsg1(200, "Write EOS label JobStatus=%c\n", jcr->JobStatus);
/*
* Check if we can still write. This may not be the case
* if we are at the end of the tape or we got a fatal I/O error.
*/
if (ok || dev->can_write()) {
if (!write_session_label(dcr, EOS_LABEL)) {
/* Print only if ok and not cancelled to avoid spurious messages */
if (ok && !jcr->is_job_canceled()) {
Jmsg1(jcr, M_FATAL, 0, _("Error writing end session label. ERR=%s\n"),
dev->bstrerror());
possible_incomplete_job(jcr, last_file_index);
}
jcr->setJobStatus(JS_ErrorTerminated);
ok = false;
}
if (dev->VolCatInfo.VolCatName[0] == 0) {
Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
Dmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n"));
}
Dmsg0(90, "back from write_end_session_label()\n");
/* Flush out final partial block of this session */
if (!write_block_to_device(dcr)) {
/* Print only if ok and not cancelled to avoid spurious messages */
if (ok && !jcr->is_job_canceled()) {
Jmsg2(jcr, M_FATAL, 0, _("Fatal append error on device %s: ERR=%s\n"),
dev->print_name(), dev->bstrerror());
Dmsg0(100, _("Set ok=FALSE after write_block_to_device.\n"));
possible_incomplete_job(jcr, last_file_index);
}
jcr->setJobStatus(JS_ErrorTerminated);
ok = false;
}
}
if (!ok && !jcr->is_JobStatus(JS_Incomplete)) {
discard_data_spool(dcr);
} else {
/* Note: if commit is OK, the device will remain blocked */
commit_data_spool(dcr);
}
if (ok) {
ok = dvd_close_job(dcr); /* do DVD cleanup if any */
}
/*
* Release the device -- and send final Vol info to DIR
* and unlock it.
*/
release_device(dcr);
if ((!ok || jcr->is_job_canceled()) && !jcr->is_JobStatus(JS_Incomplete)) {
discard_attribute_spool(jcr);
} else {
commit_attribute_spool(jcr);
}
dir_send_job_status(jcr); /* update director */
Dmsg1(100, "return from do_append_data() ok=%d\n", ok);
return ok;
}
/*
* Write a Record to the block
*
* Returns: false means the block could not be written to tape/disk.
* true on success (all bytes written to the block).
*/
bool DCR::write_record()
{
bool retval = false;
bool translated_record = false;
char buf1[100], buf2[100];
/*
* Perform record translations.
*/
before_rec = rec;
after_rec = NULL;
if (generate_plugin_event(jcr, bsdEventWriteRecordTranslation, this) != bRC_OK) {
goto bail_out;
}
/*
* The record got translated when we got an after_rec pointer after calling the
* bsdEventWriteRecordTranslation plugin event. If no translation has taken place
* we just point the after_rec pointer to same DEV_RECORD as in the before_rec pointer.
*/
if (!after_rec) {
after_rec = before_rec;
} else {
translated_record = true;
}
while (!write_record_to_block(this, after_rec)) {
Dmsg2(850, "!write_record_to_block data_len=%d rem=%d\n",
after_rec->data_len, after_rec->remainder);
if (!write_block_to_device()) {
Dmsg2(90, "Got write_block_to_dev error on device %s. %s\n",
dev->print_name(), dev->bstrerror());
goto bail_out;
}
}
jcr->JobBytes += after_rec->data_len; /* increment bytes this job */
if (jcr->RemainingQuota && jcr->JobBytes > jcr->RemainingQuota) {
Jmsg0(jcr, M_FATAL, 0, _("Quota Exceeded. Job Terminated.\n"));
goto bail_out;
}
Dmsg4(850, "write_record FI=%s SessId=%d Strm=%s len=%d\n",
FI_to_ascii(buf1, after_rec->FileIndex), after_rec->VolSessionId,
stream_to_ascii(buf2, after_rec->Stream, after_rec->FileIndex), after_rec->data_len);
retval = true;
bail_out:
if (translated_record) {
copy_record_state(before_rec, after_rec);
free_record(after_rec);
after_rec = NULL;
}
return retval;
}
/*
* read_records() calls back here for each record it gets
*/
static bool record_cb(DCR *in_dcr, DEV_RECORD *rec)
{
if (list_records) {
Pmsg5(000, _("Record: SessId=%u SessTim=%u FileIndex=%d Stream=%d len=%u\n"),
rec->VolSessionId, rec->VolSessionTime, rec->FileIndex,
rec->Stream, rec->data_len);
}
/*
* Check for Start or End of Session Record
*
*/
if (rec->FileIndex < 0) {
get_session_record(in_dcr->dev, rec, &sessrec);
if (verbose > 1) {
dump_label_record(in_dcr->dev, rec, 1);
}
switch (rec->FileIndex) {
case PRE_LABEL:
Pmsg0(000, _("Volume is prelabeled. This volume cannot be copied.\n"));
return false;
case VOL_LABEL:
Pmsg0(000, _("Volume label not copied.\n"));
return true;
case SOS_LABEL:
if (bsr && rec->match_stat < 1) {
/* Skipping record, because does not match BSR filter */
if (verbose) {
Pmsg0(-1, _("Copy skipped. Record does not match BSR filter.\n"));
}
} else {
jobs++;
}
break;
case EOS_LABEL:
if (bsr && rec->match_stat < 1) {
/* Skipping record, because does not match BSR filter */
return true;
}
while (!write_record_to_block(out_block, rec)) {
Dmsg2(150, "!write_record_to_block data_len=%d rem=%d\n", rec->data_len,
rec->remainder);
if (!write_block_to_device(out_jcr->dcr)) {
Dmsg2(90, "Got write_block_to_dev error on device %s: ERR=%s\n",
out_dev->print_name(), out_dev->bstrerror());
Jmsg(out_jcr, M_FATAL, 0, _("Cannot fixup device error. %s\n"),
out_dev->bstrerror());
return false;
}
}
if (!write_block_to_device(out_jcr->dcr)) {
Dmsg2(90, "Got write_block_to_dev error on device %s: ERR=%s\n",
out_dev->print_name(), out_dev->bstrerror());
Jmsg(out_jcr, M_FATAL, 0, _("Cannot fixup device error. %s\n"),
out_dev->bstrerror());
return false;
}
return true;
case EOM_LABEL:
Pmsg0(000, _("EOM label not copied.\n"));
return true;
case EOT_LABEL: /* end of all tapes */
Pmsg0(000, _("EOT label not copied.\n"));
return true;
default:
return true;
}
}
/* Write record */
if (bsr && rec->match_stat < 1) {
/* Skipping record, because does not match BSR filter */
return true;
}
records++;
while (!write_record_to_block(out_block, rec)) {
Dmsg2(150, "!write_record_to_block data_len=%d rem=%d\n", rec->data_len,
rec->remainder);
if (!write_block_to_device(out_jcr->dcr)) {
Dmsg2(90, "Got write_block_to_dev error on device %s: ERR=%s\n",
out_dev->print_name(), out_dev->bstrerror());
Jmsg(out_jcr, M_FATAL, 0, _("Cannot fixup device error. %s\n"),
out_dev->bstrerror());
return false;
}
}
return true;
}
int main (int argc, char *argv[])
{
int ch;
char *iVolumeName = NULL;
char *oVolumeName = NULL;
bool ignore_label_errors = false;
bool ok;
setlocale(LC_ALL, "");
bindtextdomain("bacula", LOCALEDIR);
textdomain("bacula");
init_stack_dump();
my_name_is(argc, argv, "bcopy");
init_msg(NULL, NULL);
while ((ch = getopt(argc, argv, "b:c:d:i:o:pvw:?")) != -1) {
switch (ch) {
case 'b':
bsr = parse_bsr(NULL, optarg);
break;
case 'c': /* specify config file */
if (configfile != NULL) {
free(configfile);
}
configfile = bstrdup(optarg);
break;
case 'd': /* debug level */
if (*optarg == 't') {
dbg_timestamp = true;
} else {
debug_level = atoi(optarg);
if (debug_level <= 0) {
debug_level = 1;
}
}
break;
case 'i': /* input Volume name */
iVolumeName = optarg;
break;
case 'o': /* output Volume name */
oVolumeName = optarg;
break;
case 'p':
ignore_label_errors = true;
forge_on = true;
break;
case 'v':
verbose++;
break;
case 'w':
wd = optarg;
break;
case '?':
default:
usage();
}
}
argc -= optind;
argv += optind;
if (argc != 2) {
Pmsg0(0, _("Wrong number of arguments: \n"));
usage();
}
OSDependentInit();
working_directory = wd;
if (configfile == NULL) {
configfile = bstrdup(CONFIG_FILE);
}
config = new_config_parser();
parse_sd_config(config, configfile, M_ERROR_TERM);
/* Setup and acquire input device for reading */
Dmsg0(100, "About to setup input jcr\n");
in_jcr = setup_jcr("bcopy", argv[0], bsr, iVolumeName, 1); /* read device */
if (!in_jcr) {
exit(1);
}
in_jcr->ignore_label_errors = ignore_label_errors;
in_dev = in_jcr->dcr->dev;
if (!in_dev) {
exit(1);
}
/* Setup output device for writing */
Dmsg0(100, "About to setup output jcr\n");
out_jcr = setup_jcr("bcopy", argv[1], bsr, oVolumeName, 0); /* no acquire */
if (!out_jcr) {
exit(1);
}
out_dev = out_jcr->dcr->dev;
if (!out_dev) {
exit(1);
}
Dmsg0(100, "About to acquire device for writing\n");
/* For we must now acquire the device for writing */
out_dev->r_dlock();
if (out_dev->open(out_jcr->dcr, OPEN_READ_WRITE) < 0) {
Emsg1(M_FATAL, 0, _("dev open failed: %s\n"), out_dev->errmsg);
out_dev->dunlock();
exit(1);
}
out_dev->dunlock();
if (!acquire_device_for_append(out_jcr->dcr)) {
free_jcr(in_jcr);
exit(1);
}
out_block = out_jcr->dcr->block;
ok = read_records(in_jcr->dcr, record_cb, mount_next_read_volume);
if (ok || out_dev->can_write()) {
if (!write_block_to_device(out_jcr->dcr)) {
Pmsg0(000, _("Write of last block failed.\n"));
}
}
Pmsg2(000, _("%u Jobs copied. %u records copied.\n"), jobs, records);
free_jcr(in_jcr);
free_jcr(out_jcr);
in_dev->term();
out_dev->term();
return 0;
}
/*
* NB! This routine locks the device, but if committing will
* not unlock it. If not committing, it will be unlocked.
*/
static bool despool_data(DCR *dcr, bool commit)
{
DEVICE *rdev;
DCR *rdcr;
bool ok = true;
DEV_BLOCK *block;
JCR *jcr = dcr->jcr;
int stat;
char ec1[50];
Dmsg0(100, "Despooling data\n");
/*
* Commit means that the job is done, so we commit, otherwise, we
* are despooling because of user spool size max or some error
* (e.g. filesystem full).
*/
if (commit) {
Jmsg(jcr, M_INFO, 0, _("Committing spooled data to Volume \"%s\". Despooling %s bytes ...\n"),
jcr->dcr->VolumeName,
edit_uint64_with_commas(jcr->dcr->job_spool_size, ec1));
set_jcr_job_status(jcr, JS_DataCommitting);
} else {
Jmsg(jcr, M_INFO, 0, _("Writing spooled data to Volume. Despooling %s bytes ...\n"),
edit_uint64_with_commas(jcr->dcr->job_spool_size, ec1));
set_jcr_job_status(jcr, JS_DataDespooling);
}
set_jcr_job_status(jcr, JS_DataDespooling);
dir_send_job_status(jcr);
dcr->despool_wait = true;
dcr->spooling = false;
/*
* We work with device blocked, but not locked so that
* other threads -- e.g. reservations can lock the device
* structure.
*/
dcr->dblock(BST_DESPOOLING);
dcr->despool_wait = false;
dcr->despooling = true;
/*
* This is really quite kludgy and should be fixed some time.
* We create a dev structure to read from the spool file
* in rdev and rdcr.
*/
rdev = (DEVICE *)malloc(sizeof(DEVICE));
memset(rdev, 0, sizeof(DEVICE));
rdev->dev_name = get_memory(strlen(spool_name)+1);
bstrncpy(rdev->dev_name, spool_name, sizeof(rdev->dev_name));
rdev->errmsg = get_pool_memory(PM_EMSG);
*rdev->errmsg = 0;
rdev->max_block_size = dcr->dev->max_block_size;
rdev->min_block_size = dcr->dev->min_block_size;
rdev->device = dcr->dev->device;
rdcr = new_dcr(jcr, NULL, rdev);
rdcr->spool_fd = dcr->spool_fd;
block = dcr->block; /* save block */
dcr->block = rdcr->block; /* make read and write block the same */
Dmsg1(800, "read/write block size = %d\n", block->buf_len);
lseek(rdcr->spool_fd, 0, SEEK_SET); /* rewind */
#if defined(HAVE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED)
posix_fadvise(rdcr->spool_fd, 0, 0, POSIX_FADV_WILLNEED);
#endif
/* Add run time, to get current wait time */
int32_t despool_start = time(NULL) - jcr->run_time;
set_new_file_parameters(dcr);
for ( ; ok; ) {
if (job_canceled(jcr)) {
ok = false;
break;
}
stat = read_block_from_spool_file(rdcr);
if (stat == RB_EOT) {
break;
} else if (stat == RB_ERROR) {
ok = false;
break;
}
ok = write_block_to_device(dcr);
if (!ok) {
Jmsg2(jcr, M_FATAL, 0, _("Fatal append error on device %s: ERR=%s\n"),
dcr->dev->print_name(), dcr->dev->bstrerror());
Dmsg2(000, "Fatal append error on device %s: ERR=%s\n",
dcr->dev->print_name(), dcr->dev->bstrerror());
}
Dmsg3(800, "Write block ok=%d FI=%d LI=%d\n", ok, block->FirstIndex, block->LastIndex);
}
if (!dir_create_jobmedia_record(dcr)) {
Jmsg2(jcr, M_FATAL, 0, _("Could not create JobMedia record for Volume=\"%s\" Job=%s\n"),
dcr->VolCatInfo.VolCatName, jcr->Job);
}
/* Set new file/block parameters for current dcr */
set_new_file_parameters(dcr);
/*
* Subtracting run_time give us elapsed time - wait_time since
* we started despooling. Note, don't use time_t as it is 32 or 64
* bits depending on the OS and doesn't edit with %d
*/
int32_t despool_elapsed = time(NULL) - despool_start - jcr->run_time;
if (despool_elapsed <= 0) {
despool_elapsed = 1;
}
Jmsg(dcr->jcr, M_INFO, 0, _("Despooling elapsed time = %02d:%02d:%02d, Transfer rate = %s bytes/second\n"),
despool_elapsed / 3600, despool_elapsed % 3600 / 60, despool_elapsed % 60,
edit_uint64_with_suffix(jcr->dcr->job_spool_size / despool_elapsed, ec1));
dcr->block = block; /* reset block */
lseek(rdcr->spool_fd, 0, SEEK_SET); /* rewind */
if (ftruncate(rdcr->spool_fd, 0) != 0) {
berrno be;
Jmsg(dcr->jcr, M_ERROR, 0, _("Ftruncate spool file failed: ERR=%s\n"),
be.bstrerror());
/* Note, try continuing despite ftruncate problem */
}
P(mutex);
if (spool_stats.data_size < dcr->job_spool_size) {
spool_stats.data_size = 0;
} else {
spool_stats.data_size -= dcr->job_spool_size;
}
V(mutex);
P(dcr->dev->spool_mutex);
dcr->dev->spool_size -= dcr->job_spool_size;
dcr->job_spool_size = 0; /* zap size in input dcr */
V(dcr->dev->spool_mutex);
free_memory(rdev->dev_name);
free_pool_memory(rdev->errmsg);
/* Be careful to NULL the jcr and free rdev after free_dcr() */
rdcr->jcr = NULL;
rdcr->dev = NULL;
free_dcr(rdcr);
free(rdev);
dcr->spooling = true; /* turn on spooling again */
dcr->despooling = false;
/*
* We are done, so unblock the device, but if we have done a
* commit, leave it locked so that the job cleanup does not
* need to wait to release the device (no re-acquire of the lock).
*/
dcr->dlock();
unblock_device(dcr->dev);
/* If doing a commit, leave the device locked -- unlocked in release_device() */
if (!commit) {
dcr->dunlock();
}
set_jcr_job_status(jcr, JS_Running);
dir_send_job_status(jcr);
return ok;
}
