/* * Append Data sent from File daemon * */ bool do_append_data(JCR *jcr) { int32_t n; int32_t file_index, stream, last_file_index; BSOCK *fd = jcr->file_bsock; bool ok = true; DEV_RECORD rec; char buf1[100], buf2[100]; DCR *dcr = jcr->dcr; DEVICE *dev; char ec[50]; if (!dcr) { Jmsg0(jcr, M_FATAL, 0, _("DCR is NULL!!!\n")); return false; } dev = dcr->dev; if (!dev) { Jmsg0(jcr, M_FATAL, 0, _("DEVICE is NULL!!!\n")); return false; } Dmsg1(100, "Start append data. res=%d\n", dev->num_reserved()); memset(&rec, 0, sizeof(rec)); if (!fd->set_buffer_size(dcr->device->max_network_buffer_size, BNET_SETBUF_WRITE)) { jcr->setJobStatus(JS_ErrorTerminated); Jmsg0(jcr, M_FATAL, 0, _("Unable to set network buffer size.\n")); return false; } if (!acquire_device_for_append(dcr)) { jcr->setJobStatus(JS_ErrorTerminated); return false; } jcr->setJobStatus(JS_Running); dir_send_job_status(jcr); if (dev->VolCatInfo.VolCatName[0] == 0) { Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n")); } Dmsg1(50, "Begin append device=%s\n", dev->print_name()); begin_data_spool(dcr); begin_attribute_spool(jcr); Dmsg0(100, "Just after acquire_device_for_append\n"); if (dev->VolCatInfo.VolCatName[0] == 0) { Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n")); } /* * Write Begin Session Record */ if (!write_session_label(dcr, SOS_LABEL)) { Jmsg1(jcr, M_FATAL, 0, _("Write session label failed. ERR=%s\n"), dev->bstrerror()); jcr->setJobStatus(JS_ErrorTerminated); ok = false; } if (dev->VolCatInfo.VolCatName[0] == 0) { Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n")); } /* Tell File daemon to send data */ if (!fd->fsend(OK_data)) { berrno be; Jmsg1(jcr, M_FATAL, 0, _("Network send error to FD. ERR=%s\n"), be.bstrerror(fd->b_errno)); ok = false; } /* * Get Data from File daemon, write to device. To clarify what is * going on here. We expect: * - A stream header * - Multiple records of data * - EOD record * * The Stream header is just used to sychronize things, and * none of the stream header is written to tape. * The Multiple records of data, contain first the Attributes, * then after another stream header, the file data, then * after another stream header, the MD5 data if any. * * So we get the (stream header, data, EOD) three time for each * file. 1. for the Attributes, 2. for the file data if any, * and 3. for the MD5 if any. */ dcr->VolFirstIndex = dcr->VolLastIndex = 0; jcr->run_time = time(NULL); /* start counting time for rates */ for (last_file_index = 0; ok && !jcr->is_job_canceled(); ) { /* Read Stream header from the File daemon. * The stream header consists of the following: * file_index (sequential Bacula file index, base 1) * stream (Bacula number to distinguish parts of data) * info (Info for Storage daemon -- compressed, encrypted, ...) * info is not currently used, so is read, but ignored! */ if ((n=bget_msg(fd)) <= 0) { if (n == BNET_SIGNAL && fd->msglen == BNET_EOD) { break; /* end of data */ } Jmsg1(jcr, M_FATAL, 0, _("Error reading data header from FD. ERR=%s\n"), fd->bstrerror()); possible_incomplete_job(jcr, last_file_index); ok = false; break; } if (sscanf(fd->msg, "%ld %ld", &file_index, &stream) != 2) { Jmsg1(jcr, M_FATAL, 0, _("Malformed data header from FD: %s\n"), fd->msg); ok = false; possible_incomplete_job(jcr, last_file_index); break; } Dmsg2(890, "<filed: Header FilInx=%d stream=%d\n", file_index, stream); /* * We make sure the file_index is advancing sequentially. * An incomplete job can start the file_index at any number. * otherwise, it must start at 1. */ if (jcr->rerunning && file_index > 0 && last_file_index == 0) { goto fi_checked; } if (file_index > 0 && (file_index == last_file_index || file_index == last_file_index + 1)) { goto fi_checked; } Jmsg2(jcr, M_FATAL, 0, _("FI=%d from FD not positive or sequential=%d\n"), file_index, last_file_index); possible_incomplete_job(jcr, last_file_index); ok = false; break; fi_checked: if (file_index != last_file_index) { jcr->JobFiles = file_index; last_file_index = file_index; } /* Read data stream from the File daemon. * The data stream is just raw bytes */ while ((n=bget_msg(fd)) > 0 && !jcr->is_job_canceled()) { rec.VolSessionId = jcr->VolSessionId; rec.VolSessionTime = jcr->VolSessionTime; rec.FileIndex = file_index; rec.Stream = stream; rec.maskedStream = stream & STREAMMASK_TYPE; /* strip high bits */ rec.data_len = fd->msglen; rec.data = fd->msg; /* use message buffer */ Dmsg4(850, "before writ_rec FI=%d SessId=%d Strm=%s len=%d\n", rec.FileIndex, rec.VolSessionId, stream_to_ascii(buf1, rec.Stream,rec.FileIndex), rec.data_len); while (!write_record_to_block(dcr->block, &rec)) { Dmsg2(850, "!write_record_to_block data_len=%d rem=%d\n", rec.data_len, rec.remainder); if (!write_block_to_device(dcr)) { Dmsg2(90, "Got write_block_to_dev error on device %s. %s\n", dev->print_name(), dev->bstrerror()); ok = false; break; } } if (!ok) { Dmsg0(400, "Not OK\n"); break; } jcr->JobBytes += rec.data_len; /* increment bytes this job */ Dmsg4(850, "write_record FI=%s SessId=%d Strm=%s len=%d\n", FI_to_ascii(buf1, rec.FileIndex), rec.VolSessionId, stream_to_ascii(buf2, rec.Stream, rec.FileIndex), rec.data_len); send_attrs_to_dir(jcr, &rec); Dmsg0(650, "Enter bnet_get\n"); } Dmsg1(650, "End read loop with FD. Stat=%d\n", n); if (fd->is_error()) { if (!jcr->is_job_canceled()) { Dmsg1(350, "Network read error from FD. ERR=%s\n", fd->bstrerror()); Jmsg1(jcr, M_FATAL, 0, _("Network error reading from FD. ERR=%s\n"), fd->bstrerror()); possible_incomplete_job(jcr, last_file_index); } ok = false; break; } } /* Create Job status for end of session label */ jcr->setJobStatus(ok?JS_Terminated:JS_ErrorTerminated); if (ok) { /* Terminate connection with FD */ fd->fsend(OK_append); do_fd_commands(jcr); /* finish dialog with FD */ } else { fd->fsend("3999 Failed append\n"); } /* * Don't use time_t for job_elapsed as time_t can be 32 or 64 bits, * and the subsequent Jmsg() editing will break */ int32_t job_elapsed = time(NULL) - jcr->run_time; if (job_elapsed <= 0) { job_elapsed = 1; } Jmsg(dcr->jcr, M_INFO, 0, _("Job write elapsed time = %02d:%02d:%02d, Transfer rate = %s Bytes/second\n"), job_elapsed / 3600, job_elapsed % 3600 / 60, job_elapsed % 60, edit_uint64_with_suffix(jcr->JobBytes / job_elapsed, ec)); Dmsg1(200, "Write EOS label JobStatus=%c\n", jcr->JobStatus); /* * Check if we can still write. This may not be the case * if we are at the end of the tape or we got a fatal I/O error. */ if (ok || dev->can_write()) { if (!write_session_label(dcr, EOS_LABEL)) { /* Print only if ok and not cancelled to avoid spurious messages */ if (ok && !jcr->is_job_canceled()) { Jmsg1(jcr, M_FATAL, 0, _("Error writing end session label. ERR=%s\n"), dev->bstrerror()); possible_incomplete_job(jcr, last_file_index); } jcr->setJobStatus(JS_ErrorTerminated); ok = false; } if (dev->VolCatInfo.VolCatName[0] == 0) { Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n")); Dmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n")); } Dmsg0(90, "back from write_end_session_label()\n"); /* Flush out final partial block of this session */ if (!write_block_to_device(dcr)) { /* Print only if ok and not cancelled to avoid spurious messages */ if (ok && !jcr->is_job_canceled()) { Jmsg2(jcr, M_FATAL, 0, _("Fatal append error on device %s: ERR=%s\n"), dev->print_name(), dev->bstrerror()); Dmsg0(100, _("Set ok=FALSE after write_block_to_device.\n")); possible_incomplete_job(jcr, last_file_index); } jcr->setJobStatus(JS_ErrorTerminated); ok = false; } } if (!ok && !jcr->is_JobStatus(JS_Incomplete)) { discard_data_spool(dcr); } else { /* Note: if commit is OK, the device will remain blocked */ commit_data_spool(dcr); } if (ok) { ok = dvd_close_job(dcr); /* do DVD cleanup if any */ } /* * Release the device -- and send final Vol info to DIR * and unlock it. */ release_device(dcr); if ((!ok || jcr->is_job_canceled()) && !jcr->is_JobStatus(JS_Incomplete)) { discard_attribute_spool(jcr); } else { commit_attribute_spool(jcr); } dir_send_job_status(jcr); /* update director */ Dmsg1(100, "return from do_append_data() ok=%d\n", ok); return ok; }
/* * Append Data sent from Client (FD/SD) * */ bool do_append_data(JCR *jcr) { int32_t n; int32_t file_index, stream, last_file_index; uint64_t stream_len; BSOCK *fd = jcr->file_bsock; bool ok = true; DEV_RECORD rec; char buf1[100], buf2[100]; DCR *dcr = jcr->dcr; DEVICE *dev; char ec[50]; POOLMEM *eblock = NULL; POOL_MEM errmsg(PM_EMSG); if (!dcr) { pm_strcpy(jcr->errmsg, _("DCR is NULL!!!\n")); Jmsg0(jcr, M_FATAL, 0, jcr->errmsg); return false; } dev = dcr->dev; if (!dev) { pm_strcpy(jcr->errmsg, _("DEVICE is NULL!!!\n")); Jmsg0(jcr, M_FATAL, 0, jcr->errmsg); return false; } Dmsg1(100, "Start append data. res=%d\n", dev->num_reserved()); memset(&rec, 0, sizeof(rec)); if (!fd->set_buffer_size(dcr->device->max_network_buffer_size, BNET_SETBUF_WRITE)) { jcr->setJobStatus(JS_ErrorTerminated); pm_strcpy(jcr->errmsg, _("Unable to set network buffer size.\n")); Jmsg0(jcr, M_FATAL, 0, jcr->errmsg); return false; } if (!acquire_device_for_append(dcr)) { jcr->setJobStatus(JS_ErrorTerminated); return false; } jcr->sendJobStatus(JS_Running); //ASSERT(dev->VolCatInfo.VolCatName[0]); if (dev->VolCatInfo.VolCatName[0] == 0) { Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n")); } Dmsg1(50, "Begin append device=%s\n", dev->print_name()); begin_data_spool(dcr); begin_attribute_spool(jcr); Dmsg0(100, "Just after acquire_device_for_append\n"); //ASSERT(dev->VolCatInfo.VolCatName[0]); if (dev->VolCatInfo.VolCatName[0] == 0) { Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n")); } /* * Write Begin Session Record */ if (!write_session_label(dcr, SOS_LABEL)) { Jmsg1(jcr, M_FATAL, 0, _("Write session label failed. ERR=%s\n"), dev->bstrerror()); jcr->setJobStatus(JS_ErrorTerminated); ok = false; } //ASSERT(dev->VolCatInfo.VolCatName[0]); if (dev->VolCatInfo.VolCatName[0] == 0) { Pmsg0(000, _("NULL Volume name. This shouldn't happen!!!\n")); } /* Tell File daemon to send data */ if (!fd->fsend(OK_data)) { berrno be; Jmsg1(jcr, M_FATAL, 0, _("Network send error to FD. ERR=%s\n"), be.bstrerror(fd->b_errno)); ok = false; } /* * Get Data from File daemon, write to device. To clarify what is * going on here. We expect: * - A stream header * - Multiple records of data * - EOD record * * The Stream header is just used to synchronize things, and * none of the stream header is written to tape. * The Multiple records of data, contain first the Attributes, * then after another stream header, the file data, then * after another stream header, the MD5 data if any. * * So we get the (stream header, data, EOD) three time for each * file. 1. for the Attributes, 2. for the file data if any, * and 3. for the MD5 if any. */ dcr->VolFirstIndex = dcr->VolLastIndex = 0; jcr->run_time = time(NULL); /* start counting time for rates */ GetMsg *qfd; qfd = New(GetMsg(jcr, fd, NULL, GETMSG_MAX_MSG_SIZE)); qfd->start_read_sock(); for (last_file_index = 0; ok && !jcr->is_job_canceled(); ) { /* Read Stream header from the File daemon. * The stream header consists of the following: * file_index (sequential Bacula file index, base 1) * stream (Bacula number to distinguish parts of data) * stream_len (Expected length of this stream. This * will be the size backed up if the file does not * grow during the backup. */ n = qfd->bget_msg(NULL); if (n <= 0) { if (n == BNET_SIGNAL && qfd->msglen == BNET_EOD) { Dmsg0(200, "Got EOD on reading header.\n"); break; /* end of data */ } Jmsg3(jcr, M_FATAL, 0, _("Error reading data header from FD. n=%d msglen=%d ERR=%s\n"), n, qfd->msglen, fd->bstrerror()); // ASX TODO the fd->bstrerror() can be related to the wrong error, I should Queue the error too possible_incomplete_job(jcr, last_file_index); ok = false; break; } if (sscanf(qfd->msg, "%ld %ld %lld", &file_index, &stream, &stream_len) != 3) { // TODO ASX already done in bufmsg, should reuse the values char buf[256]; Jmsg1(jcr, M_FATAL, 0, _("Malformed data header from FD: %s\n"), asciidump(qfd->msg, qfd->msglen, buf, sizeof(buf))); ok = false; possible_incomplete_job(jcr, last_file_index); break; } Dmsg3(890, "<filed: Header FilInx=%d stream=%d stream_len=%lld\n", file_index, stream, stream_len); /* * We make sure the file_index is advancing sequentially. * An incomplete job can start the file_index at any number. * otherwise, it must start at 1. */ if (jcr->rerunning && file_index > 0 && last_file_index == 0) { goto fi_checked; } Dmsg2(400, "file_index=%d last_file_index=%d\n", file_index, last_file_index); if (file_index > 0 && (file_index == last_file_index || file_index == last_file_index + 1)) { goto fi_checked; } Jmsg2(jcr, M_FATAL, 0, _("FI=%d from FD not positive or last_FI=%d\n"), file_index, last_file_index); possible_incomplete_job(jcr, last_file_index); ok = false; break; fi_checked: if (file_index != last_file_index) { jcr->JobFiles = file_index; last_file_index = file_index; } /* Read data stream from the File daemon. * The data stream is just raw bytes */ while ((n=qfd->bget_msg(NULL)) > 0 && !jcr->is_job_canceled()) { rec.VolSessionId = jcr->VolSessionId; rec.VolSessionTime = jcr->VolSessionTime; rec.FileIndex = file_index; rec.Stream = stream; rec.StreamLen = stream_len; rec.maskedStream = stream & STREAMMASK_TYPE; /* strip high bits */ rec.data_len = qfd->msglen; rec.data = qfd->msg; /* use message buffer */ Dmsg4(850, "before writ_rec FI=%d SessId=%d Strm=%s len=%d\n", rec.FileIndex, rec.VolSessionId, stream_to_ascii(buf1, rec.Stream,rec.FileIndex), rec.data_len); ok = dcr->write_record(&rec); if (!ok) { Dmsg2(90, "Got write_block_to_dev error on device %s. %s\n", dcr->dev->print_name(), dcr->dev->bstrerror()); break; } jcr->JobBytes += rec.data_len; /* increment bytes this job */ jcr->JobBytes += qfd->bmsg->jobbytes; // if the block as been downloaded, count it Dmsg4(850, "write_record FI=%s SessId=%d Strm=%s len=%d\n", FI_to_ascii(buf1, rec.FileIndex), rec.VolSessionId, stream_to_ascii(buf2, rec.Stream, rec.FileIndex), rec.data_len); send_attrs_to_dir(jcr, &rec); Dmsg0(650, "Enter bnet_get\n"); } Dmsg2(650, "End read loop with FD. JobFiles=%d Stat=%d\n", jcr->JobFiles, n); if (fd->is_error()) { if (!jcr->is_job_canceled()) { Dmsg1(350, "Network read error from FD. ERR=%s\n", fd->bstrerror()); Jmsg1(jcr, M_FATAL, 0, _("Network error reading from FD. ERR=%s\n"), fd->bstrerror()); possible_incomplete_job(jcr, last_file_index); } ok = false; break; } } qfd->wait_read_sock(); free_GetMsg(qfd); if (eblock != NULL) { free_pool_memory(eblock); } /* Create Job status for end of session label */ jcr->setJobStatus(ok?JS_Terminated:JS_ErrorTerminated); if (ok) { /* Terminate connection with Client */ fd->fsend(OK_append); do_client_commands(jcr); /* finish dialog with Client */ } else { fd->fsend("3999 Failed append\n"); } Dmsg1(200, "Write EOS label JobStatus=%c\n", jcr->JobStatus); /* * Check if we can still write. This may not be the case * if we are at the end of the tape or we got a fatal I/O error. */ if (ok || dev->can_write()) { if (!write_session_label(dcr, EOS_LABEL)) { /* Print only if ok and not cancelled to avoid spurious messages */ if (ok && !jcr->is_job_canceled()) { Jmsg1(jcr, M_FATAL, 0, _("Error writing end session label. ERR=%s\n"), dev->bstrerror()); possible_incomplete_job(jcr, last_file_index); } jcr->setJobStatus(JS_ErrorTerminated); ok = false; } /* Flush out final partial block of this session */ if (!dcr->write_final_block_to_device()) { /* Print only if ok and not cancelled to avoid spurious messages */ if (ok && !jcr->is_job_canceled()) { Jmsg2(jcr, M_FATAL, 0, _("Fatal append error on device %s: ERR=%s\n"), dev->print_name(), dev->bstrerror()); Dmsg0(100, _("Set ok=FALSE after write_final_block_to_device.\n")); possible_incomplete_job(jcr, last_file_index); } jcr->setJobStatus(JS_ErrorTerminated); ok = false; } } flush_jobmedia_queue(jcr); if (!ok && !jcr->is_JobStatus(JS_Incomplete)) { discard_data_spool(dcr); } else { /* Note: if commit is OK, the device will remain blocked */ commit_data_spool(dcr); } /* * Don't use time_t for job_elapsed as time_t can be 32 or 64 bits, * and the subsequent Jmsg() editing will break */ int32_t job_elapsed = time(NULL) - jcr->run_time; if (job_elapsed <= 0) { job_elapsed = 1; } Jmsg(dcr->jcr, M_INFO, 0, _("Elapsed time=%02d:%02d:%02d, Transfer rate=%s Bytes/second\n"), job_elapsed / 3600, job_elapsed % 3600 / 60, job_elapsed % 60, edit_uint64_with_suffix(jcr->JobBytes / job_elapsed, ec)); /* * Release the device -- and send final Vol info to DIR * and unlock it. */ release_device(dcr); if ((!ok || jcr->is_job_canceled()) && !jcr->is_JobStatus(JS_Incomplete)) { discard_attribute_spool(jcr); } else { commit_attribute_spool(jcr); } jcr->sendJobStatus(); /* update director */ Dmsg1(100, "return from do_append_data() ok=%d\n", ok); return ok; }
/* * Called here for each record from read_records() * This function is used when we do a internal clone of a Job e.g. * this SD is both the reading and writing SD. * * Returns: true if OK * false if error */ static bool clone_record_internally(DCR *dcr, DEV_RECORD *rec) { JCR *jcr = dcr->jcr; DEVICE *dev = jcr->dcr->dev; char buf1[100], buf2[100]; #ifdef xxx Dmsg5(000, "on entry JobId=%d FI=%s SessId=%d Strm=%s len=%d\n", jcr->JobId, FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId, stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len); #endif /* * If label and not for us, discard it */ if (rec->FileIndex < 0 && rec->match_stat <= 0) { return true; } /* * We want to write SOS_LABEL and EOS_LABEL discard all others */ switch (rec->FileIndex) { case PRE_LABEL: case VOL_LABEL: case EOT_LABEL: case EOM_LABEL: return true; /* don't write vol labels */ } // if (jcr->is_JobType(JT_BACKUP)) { /* * For normal migration jobs, FileIndex values are sequential because * we are dealing with one job. However, for Vbackup (consolidation), * we will be getting records from multiple jobs and writing them back * out, so we need to ensure that the output FileIndex is sequential. * We do so by detecting a FileIndex change and incrementing the * JobFiles, which we then use as the output FileIndex. */ if (rec->FileIndex >= 0) { /* * If something changed, increment FileIndex */ if (rec->VolSessionId != rec->last_VolSessionId || rec->VolSessionTime != rec->last_VolSessionTime || rec->FileIndex != rec->last_FileIndex) { jcr->JobFiles++; rec->last_VolSessionId = rec->VolSessionId; rec->last_VolSessionTime = rec->VolSessionTime; rec->last_FileIndex = rec->FileIndex; } rec->FileIndex = jcr->JobFiles; /* set sequential output FileIndex */ } // } /* * Modify record SessionId and SessionTime to correspond to output. */ rec->VolSessionId = jcr->VolSessionId; rec->VolSessionTime = jcr->VolSessionTime; Dmsg5(200, "before write JobId=%d FI=%s SessId=%d Strm=%s len=%d\n", jcr->JobId, FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId, stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len); while (!write_record_to_block(jcr->dcr, rec)) { Dmsg4(200, "!write_record_to_block blkpos=%u:%u len=%d rem=%d\n", dev->file, dev->block_num, rec->data_len, rec->remainder); if (!jcr->dcr->write_block_to_device()) { Dmsg2(90, "Got write_block_to_dev error on device %s. %s\n", dev->print_name(), dev->bstrerror()); Jmsg2(jcr, M_FATAL, 0, _("Fatal append error on device %s: ERR=%s\n"), dev->print_name(), dev->bstrerror()); return false; } Dmsg2(200, "===== Wrote block new pos %u:%u\n", dev->file, dev->block_num); } /* * Restore packet */ rec->VolSessionId = rec->last_VolSessionId; rec->VolSessionTime = rec->last_VolSessionTime; if (rec->FileIndex < 0) { return true; /* don't send LABELs to Dir */ } jcr->JobBytes += rec->data_len; /* increment bytes of this job */ Dmsg5(500, "wrote_record JobId=%d FI=%s SessId=%d Strm=%s len=%d\n", jcr->JobId, FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId, stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len); send_attrs_to_dir(jcr, rec); return true; }