static inline int ndmp_wait_for_job_termination(JCR *jcr)
{
jcr->setJobStatus(JS_Running);
/*
* Force cancel in SD if failing, but not for Incomplete jobs
* so that we let the SD despool.
*/
Dmsg4(100, "cancel=%d FDJS=%d JS=%d SDJS=%d\n",
jcr->is_canceled(), jcr->FDJobStatus,
jcr->JobStatus, jcr->SDJobStatus);
if (jcr->is_canceled() || (!jcr->res.job->RescheduleIncompleteJobs)) {
Dmsg3(100, "FDJS=%d JS=%d SDJS=%d\n",
jcr->FDJobStatus, jcr->JobStatus, jcr->SDJobStatus);
cancel_storage_daemon_job(jcr);
}
/*
* Note, the SD stores in jcr->JobFiles/ReadBytes/JobBytes/JobErrors
*/
wait_for_storage_daemon_termination(jcr);
jcr->FDJobStatus = JS_Terminated;
if (jcr->JobStatus != JS_Terminated) {
return jcr->JobStatus;
}
if (jcr->FDJobStatus != JS_Terminated) {
return jcr->FDJobStatus;
}
return jcr->SDJobStatus;
}
/*
* Update File Attributes in the catalog with data sent by the Storage daemon.
*/
void catalog_update(JCR *jcr, BSOCK *bs)
{
if (!jcr->res.pool->catalog_files) {
return; /* user disabled cataloging */
}
if (jcr->is_job_canceled()) {
goto bail_out;
}
if (!jcr->db) {
POOLMEM *omsg = get_memory(bs->msglen+1);
pm_strcpy(omsg, bs->msg);
bs->fsend(_("1994 Invalid Catalog Update: %s"), omsg);
Jmsg1(jcr, M_FATAL, 0, _("Invalid Catalog Update; DB not open: %s"), omsg);
free_memory(omsg);
goto bail_out;
}
update_attribute(jcr, bs->msg, bs->msglen);
bail_out:
if (jcr->is_job_canceled()) {
cancel_storage_daemon_job(jcr);
}
}
/*
* Cleanup a NDMP restore session.
*/
void ndmp_restore_cleanup(JCR *jcr, int TermCode)
{
char term_code[100];
const char *term_msg;
int msg_type = M_INFO;
Dmsg0(20, "In ndmp_restore_cleanup\n");
update_job_end(jcr, TermCode);
if (jcr->unlink_bsr && jcr->RestoreBootstrap) {
secure_erase(jcr, jcr->RestoreBootstrap);
jcr->unlink_bsr = false;
}
if (job_canceled(jcr)) {
cancel_storage_daemon_job(jcr);
}
switch (TermCode) {
case JS_Terminated:
if (jcr->ExpectedFiles > jcr->jr.JobFiles) {
term_msg = _("Restore OK -- warning file count mismatch");
} else {
term_msg = _("Restore OK");
}
break;
case JS_Warnings:
term_msg = _("Restore OK -- with warnings");
break;
case JS_FatalError:
case JS_ErrorTerminated:
term_msg = _("*** Restore Error ***");
msg_type = M_ERROR; /* Generate error message */
if (jcr->store_bsock) {
jcr->store_bsock->signal(BNET_TERMINATE);
if (jcr->SD_msg_chan_started) {
pthread_cancel(jcr->SD_msg_chan);
}
}
break;
case JS_Canceled:
term_msg = _("Restore Canceled");
if (jcr->store_bsock) {
jcr->store_bsock->signal(BNET_TERMINATE);
if (jcr->SD_msg_chan_started) {
pthread_cancel(jcr->SD_msg_chan);
}
}
break;
default:
term_msg = term_code;
sprintf(term_code, _("Inappropriate term code: %c\n"), TermCode);
break;
}
generate_restore_summary(jcr, msg_type, term_msg);
Dmsg0(20, "Leaving ndmp_restore_cleanup\n");
}
/*
* Cancel a job -- typically called by the UA (Console program), but may also
* be called by the job watchdog.
*
* Returns: true if cancel appears to be successful
* false on failure. Message sent to ua->jcr.
*/
bool cancel_job(UAContext *ua, JCR *jcr)
{
char ed1[50];
int32_t old_status = jcr->JobStatus;
jcr->setJobStatus(JS_Canceled);
switch (old_status) {
case JS_Created:
case JS_WaitJobRes:
case JS_WaitClientRes:
case JS_WaitStoreRes:
case JS_WaitPriority:
case JS_WaitMaxJobs:
case JS_WaitStartTime:
ua->info_msg(_("JobId %s, Job %s marked to be canceled.\n"),
edit_uint64(jcr->JobId, ed1), jcr->Job);
jobq_remove(&job_queue, jcr); /* attempt to remove it from queue */
break;
default:
/*
* Cancel File daemon
*/
if (jcr->file_bsock) {
if (!cancel_file_daemon_job(ua, jcr)) {
return false;
}
}
/*
* Cancel Storage daemon
*/
if (jcr->store_bsock) {
if (!cancel_storage_daemon_job(ua, jcr)) {
return false;
}
}
break;
}
run_scripts(jcr, jcr->res.job->RunScripts, "AfterJob");
return true;
}
/*
* Cancel a running job on a storage daemon. System invoked
* non interactive version this builds a ua context and calls
* the interactive one with the silent flag set.
*/
void cancel_storage_daemon_job(JCR *jcr)
{
UAContext *ua;
JCR *control_jcr;
if (jcr->sd_canceled) {
return; /* cancel only once */
}
ua = new_ua_context(jcr);
control_jcr = new_control_jcr("*JobCancel*", JT_SYSTEM);
ua->jcr = control_jcr;
if (jcr->store_bsock) {
if (!cancel_storage_daemon_job(ua, jcr, true)) {
goto bail_out;
}
}
bail_out:
free_jcr(control_jcr);
free_ua_context(ua);
}
/*
* Update File Attributes in the catalog with data read from
* the storage daemon spool file. We receive the filename and
* we try to read it.
*/
bool despool_attributes_from_file(JCR *jcr, const char *file)
{
bool retval = false;
int32_t pktsiz;
size_t nbytes;
ssize_t size = 0;
int32_t msglen; /* message length */
FILE *spool_fd = NULL;
POOLMEM *msg = get_pool_memory(PM_MESSAGE);
Dmsg0(100, "Begin despool_attributes_from_file\n");
if (jcr->is_job_canceled() || !jcr->res.pool->catalog_files || !jcr->db) {
goto bail_out; /* user disabled cataloging */
}
spool_fd = fopen(file, "rb");
if (!spool_fd) {
Dmsg0(100, "cancel despool_attributes_from_file\n");
/* send an error message */
goto bail_out;
}
#if defined(HAVE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED)
posix_fadvise(fileno(spool_fd), 0, 0, POSIX_FADV_WILLNEED);
#endif
while (fread((char *)&pktsiz, 1, sizeof(int32_t), spool_fd) == sizeof(int32_t)) {
size += sizeof(int32_t);
msglen = ntohl(pktsiz);
if (msglen > 0) {
if (msglen > (int32_t) sizeof_pool_memory(msg)) {
msg = realloc_pool_memory(msg, msglen + 1);
}
nbytes = fread(msg, 1, msglen, spool_fd);
if (nbytes != (size_t) msglen) {
berrno be;
Dmsg2(400, "nbytes=%d msglen=%d\n", nbytes, msglen);
Qmsg1(jcr, M_FATAL, 0, _("fread attr spool error. ERR=%s\n"),
be.bstrerror());
goto bail_out;
}
size += nbytes;
}
if (!jcr->is_job_canceled()) {
update_attribute(jcr, msg, msglen);
if (jcr->is_job_canceled()) {
goto bail_out;
}
}
}
if (ferror(spool_fd)) {
berrno be;
Qmsg1(jcr, M_FATAL, 0, _("fread attr spool error. ERR=%s\n"),
be.bstrerror());
goto bail_out;
}
retval = true;
bail_out:
if (spool_fd) {
fclose(spool_fd);
}
if (jcr->is_job_canceled()) {
cancel_storage_daemon_job(jcr);
}
free_pool_memory(msg);
Dmsg1(100, "End despool_attributes_from_file retval=%i\n", retval);
return retval;
}
/*
* Release resources allocated during backup.
*
*/
void verify_cleanup(JCR *jcr, int TermCode)
{
char sdt[50], edt[50];
char ec1[30], ec2[30];
char term_code[100], fd_term_msg[100], sd_term_msg[100];
const char *term_msg;
int msg_type;
const char *Name;
// Dmsg1(100, "Enter verify_cleanup() TermCod=%d\n", TermCode);
Dmsg3(900, "JobLevel=%c Expected=%u JobFiles=%u\n", jcr->getJobLevel(),
jcr->ExpectedFiles, jcr->JobFiles);
if ((jcr->getJobLevel() == L_VERIFY_VOLUME_TO_CATALOG || jcr->getJobLevel() == L_VERIFY_DATA) &&
jcr->ExpectedFiles != jcr->JobFiles) {
TermCode = JS_ErrorTerminated;
}
update_job_end(jcr, TermCode);
if (job_canceled(jcr)) {
cancel_storage_daemon_job(jcr);
}
if (jcr->unlink_bsr && jcr->RestoreBootstrap) {
unlink(jcr->RestoreBootstrap);
jcr->unlink_bsr = false;
}
msg_type = M_INFO; /* by default INFO message */
switch (TermCode) {
case JS_Terminated:
if (jcr->JobErrors || jcr->SDErrors) {
term_msg = _("Verify OK -- with warnings");
} else {
term_msg = _("Verify OK");
}
break;
case JS_FatalError:
case JS_ErrorTerminated:
term_msg = _("*** Verify Error ***");
msg_type = M_ERROR; /* Generate error message */
break;
case JS_Error:
term_msg = _("Verify warnings");
break;
case JS_Canceled:
term_msg = _("Verify Canceled");
break;
case JS_Differences:
term_msg = _("Verify Differences");
break;
default:
term_msg = term_code;
bsnprintf(term_code, sizeof(term_code),
_("Inappropriate term code: %d %c\n"), TermCode, TermCode);
break;
}
bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime);
bstrftimes(edt, sizeof(edt), jcr->jr.EndTime);
if (jcr->verify_job) {
Name = jcr->verify_job->hdr.name;
} else {
Name = "";
}
jobstatus_to_ascii(jcr->FDJobStatus, fd_term_msg, sizeof(fd_term_msg));
if (jcr->getJobLevel() == L_VERIFY_VOLUME_TO_CATALOG || jcr->getJobLevel() == L_VERIFY_DATA) {
jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
Jmsg(jcr, msg_type, 0, _("%s %s %s (%s):\n"
" Build OS: %s %s %s\n"
" JobId: %d\n"
" Job: %s\n"
" FileSet: %s\n"
" Verify Level: %s\n"
" Client: %s\n"
" Verify JobId: %d\n"
" Verify Job: %s\n"
" Start time: %s\n"
" End time: %s\n"
" Files Expected: %s\n"
" Files Examined: %s\n"
" Non-fatal FD errors: %d\n"
" SD Errors: %d\n"
" FD termination status: %s\n"
" SD termination status: %s\n"
" Termination: %s\n\n"),
BACULA, my_name, VERSION, LSMDATE,
HOST_OS, DISTNAME, DISTVER,
jcr->jr.JobId,
jcr->jr.Job,
jcr->fileset->hdr.name,
level_to_str(jcr->getJobLevel()),
jcr->client->hdr.name,
jcr->previous_jr.JobId,
Name,
sdt,
edt,
edit_uint64_with_commas(jcr->ExpectedFiles, ec1),
edit_uint64_with_commas(jcr->JobFiles, ec2),
jcr->JobErrors,
jcr->SDErrors,
fd_term_msg,
sd_term_msg,
term_msg);
} else {
Jmsg(jcr, msg_type, 0, _("%s %s %s (%s):\n"
" Build: %s %s %s\n"
" JobId: %d\n"
" Job: %s\n"
" FileSet: %s\n"
" Verify Level: %s\n"
" Client: %s\n"
" Verify JobId: %d\n"
" Verify Job: %s\n"
" Start time: %s\n"
" End time: %s\n"
" Files Examined: %s\n"
" Non-fatal FD errors: %d\n"
" FD termination status: %s\n"
" Termination: %s\n\n"),
BACULA, my_name, VERSION, LSMDATE,
HOST_OS, DISTNAME, DISTVER,
jcr->jr.JobId,
jcr->jr.Job,
jcr->fileset->hdr.name,
level_to_str(jcr->getJobLevel()),
jcr->client->name(),
jcr->previous_jr.JobId,
Name,
sdt,
edt,
edit_uint64_with_commas(jcr->JobFiles, ec1),
jcr->JobErrors,
fd_term_msg,
term_msg);
}
Dmsg0(100, "Leave verify_cleanup()\n");
}
/*
* Here we wait for the File daemon to signal termination,
* then we wait for the Storage daemon. When both
* are done, we return the job status.
* Also used by restore.c
*/
int wait_for_job_termination(JCR *jcr, int timeout)
{
int32_t n = 0;
BSOCK *fd = jcr->file_bsock;
bool fd_ok = false;
uint32_t JobFiles, JobErrors;
uint32_t JobWarnings = 0;
uint64_t ReadBytes = 0;
uint64_t JobBytes = 0;
int VSS = 0;
int Encrypt = 0;
btimer_t *tid=NULL;
set_jcr_job_status(jcr, JS_Running);
if (fd) {
if (timeout) {
tid = start_bsock_timer(fd, timeout); /* TODO: New timeout directive??? */
}
/* Wait for Client to terminate */
while ((n = bget_dirmsg(fd)) >= 0) {
if (!fd_ok &&
(sscanf(fd->msg, EndJob, &jcr->FDJobStatus, &JobFiles,
&ReadBytes, &JobBytes, &JobErrors, &VSS, &Encrypt) == 7 ||
sscanf(fd->msg, OldEndJob, &jcr->FDJobStatus, &JobFiles,
&ReadBytes, &JobBytes, &JobErrors) == 5)) {
fd_ok = true;
set_jcr_job_status(jcr, jcr->FDJobStatus);
Dmsg1(100, "FDStatus=%c\n", (char)jcr->JobStatus);
} else {
Jmsg(jcr, M_WARNING, 0, _("Unexpected Client Job message: %s\n"),
fd->msg);
}
if (job_canceled(jcr)) {
break;
}
}
if (tid) {
stop_bsock_timer(tid);
}
if (is_bnet_error(fd)) {
Jmsg(jcr, M_FATAL, 0, _("Network error with FD during %s: ERR=%s\n"),
job_type_to_str(jcr->get_JobType()), fd->bstrerror());
}
fd->signal(BNET_TERMINATE); /* tell Client we are terminating */
}
/* Force cancel in SD if failing */
if (job_canceled(jcr) || !fd_ok) {
cancel_storage_daemon_job(jcr);
}
/* Note, the SD stores in jcr->JobFiles/ReadBytes/JobBytes/JobErrors */
wait_for_storage_daemon_termination(jcr);
/* Return values from FD */
if (fd_ok) {
jcr->JobFiles = JobFiles;
jcr->JobErrors += JobErrors; /* Keep total errors */
jcr->ReadBytes = ReadBytes;
jcr->JobBytes = JobBytes;
jcr->JobWarnings = JobWarnings;
jcr->VSS = VSS;
jcr->Encrypt = Encrypt;
} else {
Jmsg(jcr, M_FATAL, 0, _("No Job status returned from FD.\n"));
}
// Dmsg4(100, "fd_ok=%d FDJS=%d JS=%d SDJS=%d\n", fd_ok, jcr->FDJobStatus,
// jcr->JobStatus, jcr->SDJobStatus);
/* Return the first error status we find Dir, FD, or SD */
if (!fd_ok || is_bnet_error(fd)) { /* if fd not set, that use !fd_ok */
jcr->FDJobStatus = JS_ErrorTerminated;
}
if (jcr->JobStatus != JS_Terminated) {
return jcr->JobStatus;
}
if (jcr->FDJobStatus != JS_Terminated) {
return jcr->FDJobStatus;
}
return jcr->SDJobStatus;
}
