/* * Append Data sent from File daemon * */ bool do_append_data(JCR *jcr) { int32_t n; int32_t file_index, stream, last_file_index; BSOCK *fd = jcr->file_bsock; bool ok = true; DEV_RECORD rec; char buf1[100], buf2[100]; DCR *dcr = jcr->dcr; DEVICE *dev; char ec[50]; if (!dcr) { Jmsg0(jcr, M_FATAL, 0, _("DCR is NULL!!!\n")); return false; } dev = dcr->dev; if (!dev) { Jmsg0(jcr, M_FATAL, 0, _("DEVICE is NULL!!!\n")); return false; } Dmsg1(100, "Start append data. res=%d\n", dev->num_reserved()); memset(&rec, 0, sizeof(rec)); if (!fd->set_buffer_size(dcr->device->max_network_buffer_size, BNET_SETBUF_WRITE)) { jcr->setJobStatus(JS_ErrorTerminated); Jmsg0(jcr, M_FATAL, 0, _("Unable to set network buffer size.\n")); return false; } if (!acquire_device_for_append(dcr)) { jcr->setJobStatus(JS_ErrorTerminated); return false; } jcr->setJobStatus(JS_Running); dir_send_job_status(jcr); if (dev->VolCatInfo.VolCatName[0] == 0) { Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n")); } Dmsg1(50, "Begin append device=%s\n", dev->print_name()); begin_data_spool(dcr); begin_attribute_spool(jcr); Dmsg0(100, "Just after acquire_device_for_append\n"); if (dev->VolCatInfo.VolCatName[0] == 0) { Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n")); } /* * Write Begin Session Record */ if (!write_session_label(dcr, SOS_LABEL)) { Jmsg1(jcr, M_FATAL, 0, _("Write session label failed. ERR=%s\n"), dev->bstrerror()); jcr->setJobStatus(JS_ErrorTerminated); ok = false; } if (dev->VolCatInfo.VolCatName[0] == 0) { Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n")); } /* Tell File daemon to send data */ if (!fd->fsend(OK_data)) { berrno be; Jmsg1(jcr, M_FATAL, 0, _("Network send error to FD. ERR=%s\n"), be.bstrerror(fd->b_errno)); ok = false; } /* * Get Data from File daemon, write to device. To clarify what is * going on here. We expect: * - A stream header * - Multiple records of data * - EOD record * * The Stream header is just used to sychronize things, and * none of the stream header is written to tape. * The Multiple records of data, contain first the Attributes, * then after another stream header, the file data, then * after another stream header, the MD5 data if any. * * So we get the (stream header, data, EOD) three time for each * file. 1. for the Attributes, 2. for the file data if any, * and 3. for the MD5 if any. */ dcr->VolFirstIndex = dcr->VolLastIndex = 0; jcr->run_time = time(NULL); /* start counting time for rates */ for (last_file_index = 0; ok && !jcr->is_job_canceled(); ) { /* Read Stream header from the File daemon. * The stream header consists of the following: * file_index (sequential Bacula file index, base 1) * stream (Bacula number to distinguish parts of data) * info (Info for Storage daemon -- compressed, encrypted, ...) * info is not currently used, so is read, but ignored! */ if ((n=bget_msg(fd)) <= 0) { if (n == BNET_SIGNAL && fd->msglen == BNET_EOD) { break; /* end of data */ } Jmsg1(jcr, M_FATAL, 0, _("Error reading data header from FD. ERR=%s\n"), fd->bstrerror()); possible_incomplete_job(jcr, last_file_index); ok = false; break; } if (sscanf(fd->msg, "%ld %ld", &file_index, &stream) != 2) { Jmsg1(jcr, M_FATAL, 0, _("Malformed data header from FD: %s\n"), fd->msg); ok = false; possible_incomplete_job(jcr, last_file_index); break; } Dmsg2(890, "<filed: Header FilInx=%d stream=%d\n", file_index, stream); /* * We make sure the file_index is advancing sequentially. * An incomplete job can start the file_index at any number. * otherwise, it must start at 1. */ if (jcr->rerunning && file_index > 0 && last_file_index == 0) { goto fi_checked; } if (file_index > 0 && (file_index == last_file_index || file_index == last_file_index + 1)) { goto fi_checked; } Jmsg2(jcr, M_FATAL, 0, _("FI=%d from FD not positive or sequential=%d\n"), file_index, last_file_index); possible_incomplete_job(jcr, last_file_index); ok = false; break; fi_checked: if (file_index != last_file_index) { jcr->JobFiles = file_index; last_file_index = file_index; } /* Read data stream from the File daemon. * The data stream is just raw bytes */ while ((n=bget_msg(fd)) > 0 && !jcr->is_job_canceled()) { rec.VolSessionId = jcr->VolSessionId; rec.VolSessionTime = jcr->VolSessionTime; rec.FileIndex = file_index; rec.Stream = stream; rec.maskedStream = stream & STREAMMASK_TYPE; /* strip high bits */ rec.data_len = fd->msglen; rec.data = fd->msg; /* use message buffer */ Dmsg4(850, "before writ_rec FI=%d SessId=%d Strm=%s len=%d\n", rec.FileIndex, rec.VolSessionId, stream_to_ascii(buf1, rec.Stream,rec.FileIndex), rec.data_len); while (!write_record_to_block(dcr->block, &rec)) { Dmsg2(850, "!write_record_to_block data_len=%d rem=%d\n", rec.data_len, rec.remainder); if (!write_block_to_device(dcr)) { Dmsg2(90, "Got write_block_to_dev error on device %s. %s\n", dev->print_name(), dev->bstrerror()); ok = false; break; } } if (!ok) { Dmsg0(400, "Not OK\n"); break; } jcr->JobBytes += rec.data_len; /* increment bytes this job */ Dmsg4(850, "write_record FI=%s SessId=%d Strm=%s len=%d\n", FI_to_ascii(buf1, rec.FileIndex), rec.VolSessionId, stream_to_ascii(buf2, rec.Stream, rec.FileIndex), rec.data_len); send_attrs_to_dir(jcr, &rec); Dmsg0(650, "Enter bnet_get\n"); } Dmsg1(650, "End read loop with FD. Stat=%d\n", n); if (fd->is_error()) { if (!jcr->is_job_canceled()) { Dmsg1(350, "Network read error from FD. ERR=%s\n", fd->bstrerror()); Jmsg1(jcr, M_FATAL, 0, _("Network error reading from FD. ERR=%s\n"), fd->bstrerror()); possible_incomplete_job(jcr, last_file_index); } ok = false; break; } } /* Create Job status for end of session label */ jcr->setJobStatus(ok?JS_Terminated:JS_ErrorTerminated); if (ok) { /* Terminate connection with FD */ fd->fsend(OK_append); do_fd_commands(jcr); /* finish dialog with FD */ } else { fd->fsend("3999 Failed append\n"); } /* * Don't use time_t for job_elapsed as time_t can be 32 or 64 bits, * and the subsequent Jmsg() editing will break */ int32_t job_elapsed = time(NULL) - jcr->run_time; if (job_elapsed <= 0) { job_elapsed = 1; } Jmsg(dcr->jcr, M_INFO, 0, _("Job write elapsed time = %02d:%02d:%02d, Transfer rate = %s Bytes/second\n"), job_elapsed / 3600, job_elapsed % 3600 / 60, job_elapsed % 60, edit_uint64_with_suffix(jcr->JobBytes / job_elapsed, ec)); Dmsg1(200, "Write EOS label JobStatus=%c\n", jcr->JobStatus); /* * Check if we can still write. This may not be the case * if we are at the end of the tape or we got a fatal I/O error. */ if (ok || dev->can_write()) { if (!write_session_label(dcr, EOS_LABEL)) { /* Print only if ok and not cancelled to avoid spurious messages */ if (ok && !jcr->is_job_canceled()) { Jmsg1(jcr, M_FATAL, 0, _("Error writing end session label. ERR=%s\n"), dev->bstrerror()); possible_incomplete_job(jcr, last_file_index); } jcr->setJobStatus(JS_ErrorTerminated); ok = false; } if (dev->VolCatInfo.VolCatName[0] == 0) { Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n")); Dmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n")); } Dmsg0(90, "back from write_end_session_label()\n"); /* Flush out final partial block of this session */ if (!write_block_to_device(dcr)) { /* Print only if ok and not cancelled to avoid spurious messages */ if (ok && !jcr->is_job_canceled()) { Jmsg2(jcr, M_FATAL, 0, _("Fatal append error on device %s: ERR=%s\n"), dev->print_name(), dev->bstrerror()); Dmsg0(100, _("Set ok=FALSE after write_block_to_device.\n")); possible_incomplete_job(jcr, last_file_index); } jcr->setJobStatus(JS_ErrorTerminated); ok = false; } } if (!ok && !jcr->is_JobStatus(JS_Incomplete)) { discard_data_spool(dcr); } else { /* Note: if commit is OK, the device will remain blocked */ commit_data_spool(dcr); } if (ok) { ok = dvd_close_job(dcr); /* do DVD cleanup if any */ } /* * Release the device -- and send final Vol info to DIR * and unlock it. */ release_device(dcr); if ((!ok || jcr->is_job_canceled()) && !jcr->is_JobStatus(JS_Incomplete)) { discard_attribute_spool(jcr); } else { commit_attribute_spool(jcr); } dir_send_job_status(jcr); /* update director */ Dmsg1(100, "return from do_append_data() ok=%d\n", ok); return ok; }
/* * Write a Record to the block * * Returns: false means the block could not be written to tape/disk. * true on success (all bytes written to the block). */ bool DCR::write_record() { bool retval = false; bool translated_record = false; char buf1[100], buf2[100]; /* * Perform record translations. */ before_rec = rec; after_rec = NULL; if (generate_plugin_event(jcr, bsdEventWriteRecordTranslation, this) != bRC_OK) { goto bail_out; } /* * The record got translated when we got an after_rec pointer after calling the * bsdEventWriteRecordTranslation plugin event. If no translation has taken place * we just point the after_rec pointer to same DEV_RECORD as in the before_rec pointer. */ if (!after_rec) { after_rec = before_rec; } else { translated_record = true; } while (!write_record_to_block(this, after_rec)) { Dmsg2(850, "!write_record_to_block data_len=%d rem=%d\n", after_rec->data_len, after_rec->remainder); if (!write_block_to_device()) { Dmsg2(90, "Got write_block_to_dev error on device %s. %s\n", dev->print_name(), dev->bstrerror()); goto bail_out; } } jcr->JobBytes += after_rec->data_len; /* increment bytes this job */ if (jcr->RemainingQuota && jcr->JobBytes > jcr->RemainingQuota) { Jmsg0(jcr, M_FATAL, 0, _("Quota Exceeded. Job Terminated.\n")); goto bail_out; } Dmsg4(850, "write_record FI=%s SessId=%d Strm=%s len=%d\n", FI_to_ascii(buf1, after_rec->FileIndex), after_rec->VolSessionId, stream_to_ascii(buf2, after_rec->Stream, after_rec->FileIndex), after_rec->data_len); retval = true; bail_out: if (translated_record) { copy_record_state(before_rec, after_rec); free_record(after_rec); after_rec = NULL; } return retval; }
/* * read_records() calls back here for each record it gets */ static bool record_cb(DCR *in_dcr, DEV_RECORD *rec) { if (list_records) { Pmsg5(000, _("Record: SessId=%u SessTim=%u FileIndex=%d Stream=%d len=%u\n"), rec->VolSessionId, rec->VolSessionTime, rec->FileIndex, rec->Stream, rec->data_len); } /* * Check for Start or End of Session Record * */ if (rec->FileIndex < 0) { get_session_record(in_dcr->dev, rec, &sessrec); if (verbose > 1) { dump_label_record(in_dcr->dev, rec, 1); } switch (rec->FileIndex) { case PRE_LABEL: Pmsg0(000, _("Volume is prelabeled. This volume cannot be copied.\n")); return false; case VOL_LABEL: Pmsg0(000, _("Volume label not copied.\n")); return true; case SOS_LABEL: if (bsr && rec->match_stat < 1) { /* Skipping record, because does not match BSR filter */ if (verbose) { Pmsg0(-1, _("Copy skipped. Record does not match BSR filter.\n")); } } else { jobs++; } break; case EOS_LABEL: if (bsr && rec->match_stat < 1) { /* Skipping record, because does not match BSR filter */ return true; } while (!write_record_to_block(out_block, rec)) { Dmsg2(150, "!write_record_to_block data_len=%d rem=%d\n", rec->data_len, rec->remainder); if (!write_block_to_device(out_jcr->dcr)) { Dmsg2(90, "Got write_block_to_dev error on device %s: ERR=%s\n", out_dev->print_name(), out_dev->bstrerror()); Jmsg(out_jcr, M_FATAL, 0, _("Cannot fixup device error. %s\n"), out_dev->bstrerror()); return false; } } if (!write_block_to_device(out_jcr->dcr)) { Dmsg2(90, "Got write_block_to_dev error on device %s: ERR=%s\n", out_dev->print_name(), out_dev->bstrerror()); Jmsg(out_jcr, M_FATAL, 0, _("Cannot fixup device error. %s\n"), out_dev->bstrerror()); return false; } return true; case EOM_LABEL: Pmsg0(000, _("EOM label not copied.\n")); return true; case EOT_LABEL: /* end of all tapes */ Pmsg0(000, _("EOT label not copied.\n")); return true; default: return true; } } /* Write record */ if (bsr && rec->match_stat < 1) { /* Skipping record, because does not match BSR filter */ return true; } records++; while (!write_record_to_block(out_block, rec)) { Dmsg2(150, "!write_record_to_block data_len=%d rem=%d\n", rec->data_len, rec->remainder); if (!write_block_to_device(out_jcr->dcr)) { Dmsg2(90, "Got write_block_to_dev error on device %s: ERR=%s\n", out_dev->print_name(), out_dev->bstrerror()); Jmsg(out_jcr, M_FATAL, 0, _("Cannot fixup device error. %s\n"), out_dev->bstrerror()); return false; } } return true; }
int main (int argc, char *argv[]) { int ch; char *iVolumeName = NULL; char *oVolumeName = NULL; bool ignore_label_errors = false; bool ok; setlocale(LC_ALL, ""); bindtextdomain("bacula", LOCALEDIR); textdomain("bacula"); init_stack_dump(); my_name_is(argc, argv, "bcopy"); init_msg(NULL, NULL); while ((ch = getopt(argc, argv, "b:c:d:i:o:pvw:?")) != -1) { switch (ch) { case 'b': bsr = parse_bsr(NULL, optarg); break; case 'c': /* specify config file */ if (configfile != NULL) { free(configfile); } configfile = bstrdup(optarg); break; case 'd': /* debug level */ if (*optarg == 't') { dbg_timestamp = true; } else { debug_level = atoi(optarg); if (debug_level <= 0) { debug_level = 1; } } break; case 'i': /* input Volume name */ iVolumeName = optarg; break; case 'o': /* output Volume name */ oVolumeName = optarg; break; case 'p': ignore_label_errors = true; forge_on = true; break; case 'v': verbose++; break; case 'w': wd = optarg; break; case '?': default: usage(); } } argc -= optind; argv += optind; if (argc != 2) { Pmsg0(0, _("Wrong number of arguments: \n")); usage(); } OSDependentInit(); working_directory = wd; if (configfile == NULL) { configfile = bstrdup(CONFIG_FILE); } config = new_config_parser(); parse_sd_config(config, configfile, M_ERROR_TERM); /* Setup and acquire input device for reading */ Dmsg0(100, "About to setup input jcr\n"); in_jcr = setup_jcr("bcopy", argv[0], bsr, iVolumeName, 1); /* read device */ if (!in_jcr) { exit(1); } in_jcr->ignore_label_errors = ignore_label_errors; in_dev = in_jcr->dcr->dev; if (!in_dev) { exit(1); } /* Setup output device for writing */ Dmsg0(100, "About to setup output jcr\n"); out_jcr = setup_jcr("bcopy", argv[1], bsr, oVolumeName, 0); /* no acquire */ if (!out_jcr) { exit(1); } out_dev = out_jcr->dcr->dev; if (!out_dev) { exit(1); } Dmsg0(100, "About to acquire device for writing\n"); /* For we must now acquire the device for writing */ out_dev->r_dlock(); if (out_dev->open(out_jcr->dcr, OPEN_READ_WRITE) < 0) { Emsg1(M_FATAL, 0, _("dev open failed: %s\n"), out_dev->errmsg); out_dev->dunlock(); exit(1); } out_dev->dunlock(); if (!acquire_device_for_append(out_jcr->dcr)) { free_jcr(in_jcr); exit(1); } out_block = out_jcr->dcr->block; ok = read_records(in_jcr->dcr, record_cb, mount_next_read_volume); if (ok || out_dev->can_write()) { if (!write_block_to_device(out_jcr->dcr)) { Pmsg0(000, _("Write of last block failed.\n")); } } Pmsg2(000, _("%u Jobs copied. %u records copied.\n"), jobs, records); free_jcr(in_jcr); free_jcr(out_jcr); in_dev->term(); out_dev->term(); return 0; }
/* * NB! This routine locks the device, but if committing will * not unlock it. If not committing, it will be unlocked. */ static bool despool_data(DCR *dcr, bool commit) { DEVICE *rdev; DCR *rdcr; bool ok = true; DEV_BLOCK *block; JCR *jcr = dcr->jcr; int stat; char ec1[50]; Dmsg0(100, "Despooling data\n"); /* * Commit means that the job is done, so we commit, otherwise, we * are despooling because of user spool size max or some error * (e.g. filesystem full). */ if (commit) { Jmsg(jcr, M_INFO, 0, _("Committing spooled data to Volume \"%s\". Despooling %s bytes ...\n"), jcr->dcr->VolumeName, edit_uint64_with_commas(jcr->dcr->job_spool_size, ec1)); set_jcr_job_status(jcr, JS_DataCommitting); } else { Jmsg(jcr, M_INFO, 0, _("Writing spooled data to Volume. Despooling %s bytes ...\n"), edit_uint64_with_commas(jcr->dcr->job_spool_size, ec1)); set_jcr_job_status(jcr, JS_DataDespooling); } set_jcr_job_status(jcr, JS_DataDespooling); dir_send_job_status(jcr); dcr->despool_wait = true; dcr->spooling = false; /* * We work with device blocked, but not locked so that * other threads -- e.g. reservations can lock the device * structure. */ dcr->dblock(BST_DESPOOLING); dcr->despool_wait = false; dcr->despooling = true; /* * This is really quite kludgy and should be fixed some time. * We create a dev structure to read from the spool file * in rdev and rdcr. */ rdev = (DEVICE *)malloc(sizeof(DEVICE)); memset(rdev, 0, sizeof(DEVICE)); rdev->dev_name = get_memory(strlen(spool_name)+1); bstrncpy(rdev->dev_name, spool_name, sizeof(rdev->dev_name)); rdev->errmsg = get_pool_memory(PM_EMSG); *rdev->errmsg = 0; rdev->max_block_size = dcr->dev->max_block_size; rdev->min_block_size = dcr->dev->min_block_size; rdev->device = dcr->dev->device; rdcr = new_dcr(jcr, NULL, rdev); rdcr->spool_fd = dcr->spool_fd; block = dcr->block; /* save block */ dcr->block = rdcr->block; /* make read and write block the same */ Dmsg1(800, "read/write block size = %d\n", block->buf_len); lseek(rdcr->spool_fd, 0, SEEK_SET); /* rewind */ #if defined(HAVE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED) posix_fadvise(rdcr->spool_fd, 0, 0, POSIX_FADV_WILLNEED); #endif /* Add run time, to get current wait time */ int32_t despool_start = time(NULL) - jcr->run_time; set_new_file_parameters(dcr); for ( ; ok; ) { if (job_canceled(jcr)) { ok = false; break; } stat = read_block_from_spool_file(rdcr); if (stat == RB_EOT) { break; } else if (stat == RB_ERROR) { ok = false; break; } ok = write_block_to_device(dcr); if (!ok) { Jmsg2(jcr, M_FATAL, 0, _("Fatal append error on device %s: ERR=%s\n"), dcr->dev->print_name(), dcr->dev->bstrerror()); Dmsg2(000, "Fatal append error on device %s: ERR=%s\n", dcr->dev->print_name(), dcr->dev->bstrerror()); } Dmsg3(800, "Write block ok=%d FI=%d LI=%d\n", ok, block->FirstIndex, block->LastIndex); } if (!dir_create_jobmedia_record(dcr)) { Jmsg2(jcr, M_FATAL, 0, _("Could not create JobMedia record for Volume=\"%s\" Job=%s\n"), dcr->VolCatInfo.VolCatName, jcr->Job); } /* Set new file/block parameters for current dcr */ set_new_file_parameters(dcr); /* * Subtracting run_time give us elapsed time - wait_time since * we started despooling. Note, don't use time_t as it is 32 or 64 * bits depending on the OS and doesn't edit with %d */ int32_t despool_elapsed = time(NULL) - despool_start - jcr->run_time; if (despool_elapsed <= 0) { despool_elapsed = 1; } Jmsg(dcr->jcr, M_INFO, 0, _("Despooling elapsed time = %02d:%02d:%02d, Transfer rate = %s bytes/second\n"), despool_elapsed / 3600, despool_elapsed % 3600 / 60, despool_elapsed % 60, edit_uint64_with_suffix(jcr->dcr->job_spool_size / despool_elapsed, ec1)); dcr->block = block; /* reset block */ lseek(rdcr->spool_fd, 0, SEEK_SET); /* rewind */ if (ftruncate(rdcr->spool_fd, 0) != 0) { berrno be; Jmsg(dcr->jcr, M_ERROR, 0, _("Ftruncate spool file failed: ERR=%s\n"), be.bstrerror()); /* Note, try continuing despite ftruncate problem */ } P(mutex); if (spool_stats.data_size < dcr->job_spool_size) { spool_stats.data_size = 0; } else { spool_stats.data_size -= dcr->job_spool_size; } V(mutex); P(dcr->dev->spool_mutex); dcr->dev->spool_size -= dcr->job_spool_size; dcr->job_spool_size = 0; /* zap size in input dcr */ V(dcr->dev->spool_mutex); free_memory(rdev->dev_name); free_pool_memory(rdev->errmsg); /* Be careful to NULL the jcr and free rdev after free_dcr() */ rdcr->jcr = NULL; rdcr->dev = NULL; free_dcr(rdcr); free(rdev); dcr->spooling = true; /* turn on spooling again */ dcr->despooling = false; /* * We are done, so unblock the device, but if we have done a * commit, leave it locked so that the job cleanup does not * need to wait to release the device (no re-acquire of the lock). */ dcr->dlock(); unblock_device(dcr->dev); /* If doing a commit, leave the device locked -- unlocked in release_device() */ if (!commit) { dcr->dunlock(); } set_jcr_job_status(jcr, JS_Running); dir_send_job_status(jcr); return ok; }