/*
* Director requests us to start a job
* Basic tasks done here:
* - We pickup the JobId to be run from the Director.
* - We pickup the device, media, and pool from the Director
* - Wait for a connection from the File Daemon (FD)
* - Accept commands from the FD (i.e. run the job)
* - Return when the connection is terminated or
* there is an error.
*/
bool job_cmd(JCR *jcr)
{
int32_t JobId;
char sd_auth_key[200];
char spool_size[30];
char seed[100];
BSOCK *dir = jcr->dir_bsock;
POOL_MEM job_name, client_name, job, fileset_name, fileset_md5;
int32_t JobType, level, spool_attributes, no_attributes, spool_data;
int32_t write_part_after_job, PreferMountedVols;
int32_t rerunning;
int32_t is_client;
int stat;
JCR *ojcr;
/*
* Get JobId and permissions from Director
*/
Dmsg1(100, "<dird: %s", dir->msg);
bstrncpy(spool_size, "0", sizeof(spool_size));
stat = sscanf(dir->msg, jobcmd, &JobId, job.c_str(), job_name.c_str(),
client_name.c_str(),
&JobType, &level, fileset_name.c_str(), &no_attributes,
&spool_attributes, fileset_md5.c_str(), &spool_data,
&write_part_after_job, &PreferMountedVols, spool_size,
&rerunning, &jcr->VolSessionId, &jcr->VolSessionTime,
&is_client, &sd_auth_key);
if (stat != 19) {
pm_strcpy(jcr->errmsg, dir->msg);
dir->fsend(BAD_job, stat, jcr->errmsg);
Dmsg1(100, ">dird: %s", dir->msg);
jcr->setJobStatus(JS_ErrorTerminated);
return false;
}
jcr->rerunning = rerunning;
jcr->sd_client = is_client;
if (is_client) {
jcr->sd_auth_key = bstrdup(sd_auth_key);
}
Dmsg3(100, "rerunning=%d VolSesId=%d VolSesTime=%d\n", jcr->rerunning,
jcr->VolSessionId, jcr->VolSessionTime);
/*
* Since this job could be rescheduled, we
* check to see if we have it already. If so
* free the old jcr and use the new one.
*/
ojcr = get_jcr_by_full_name(job.c_str());
if (ojcr && !ojcr->authenticated) {
Dmsg2(100, "Found ojcr=0x%x Job %s\n", (unsigned)(intptr_t)ojcr, job.c_str());
free_jcr(ojcr);
}
jcr->JobId = JobId;
Dmsg2(800, "Start JobId=%d %p\n", JobId, jcr);
set_jcr_in_tsd(jcr);
/*
* If job rescheduled because previous was incomplete,
* the Resched flag is set and VolSessionId and VolSessionTime
* are given to us (same as restarted job).
*/
if (!jcr->rerunning) {
jcr->VolSessionId = newVolSessionId();
jcr->VolSessionTime = VolSessionTime;
}
bstrncpy(jcr->Job, job, sizeof(jcr->Job));
unbash_spaces(job_name);
jcr->job_name = get_pool_memory(PM_NAME);
pm_strcpy(jcr->job_name, job_name);
unbash_spaces(client_name);
jcr->client_name = get_pool_memory(PM_NAME);
pm_strcpy(jcr->client_name, client_name);
unbash_spaces(fileset_name);
jcr->fileset_name = get_pool_memory(PM_NAME);
pm_strcpy(jcr->fileset_name, fileset_name);
jcr->setJobType(JobType);
jcr->setJobLevel(level);
jcr->no_attributes = no_attributes;
jcr->spool_attributes = spool_attributes;
jcr->spool_data = spool_data;
jcr->spool_size = str_to_int64(spool_size);
jcr->write_part_after_job = write_part_after_job;
jcr->fileset_md5 = get_pool_memory(PM_NAME);
pm_strcpy(jcr->fileset_md5, fileset_md5);
jcr->PreferMountedVols = PreferMountedVols;
jcr->authenticated = false;
/*
* Pass back an authorization key for the File daemon
*/
if (jcr->sd_client) {
bstrncpy(sd_auth_key, "xxx", 3);
} else {
bsnprintf(seed, sizeof(seed), "%p%d", jcr, JobId);
make_session_key(sd_auth_key, seed, 1);
}
dir->fsend(OKjob, jcr->VolSessionId, jcr->VolSessionTime, sd_auth_key);
Dmsg2(150, ">dird jid=%u: %s", (uint32_t)jcr->JobId, dir->msg);
/* If not client, set key, otherwise it is already set */
if (!jcr->sd_client) {
jcr->sd_auth_key = bstrdup(sd_auth_key);
memset(sd_auth_key, 0, sizeof(sd_auth_key));
}
new_plugins(jcr); /* instantiate the plugins */
generate_daemon_event(jcr, "JobStart");
generate_plugin_event(jcr, bsdEventJobStart, (void *)"JobStart");
return true;
}
/*
* Initialize the static structure to zeros, then
* apply all the default values.
*/
void CONFIG::init_resource(int type, RES_ITEM *items, int pass)
{
memset(m_res_all, 0, m_res_all_size);
((URES *)m_res_all)->hdr.rcode = type;
((URES *)m_res_all)->hdr.refcnt = 1;
/*
* Set defaults in each item. We only set defaults in pass 1.
*/
if (pass == 1) {
int i;
for (i = 0; items[i].name; i++) {
Dmsg3(900, "Item=%s def=%s defval=%s\n", items[i].name,
(items[i].flags & CFG_ITEM_DEFAULT) ? "yes" : "no",
(items[i].default_value) ? items[i].default_value : "None");
/*
* Sanity check.
*
* Items with a default value but without the CFG_ITEM_DEFAULT flag set
* are most of the time an indication of a programmers error.
*/
if (items[i].default_value != NULL && !(items[i].flags & CFG_ITEM_DEFAULT)) {
Pmsg1(000, _("Found config item %s which has default value but no CFG_ITEM_DEFAULT flag set\n"),
items[i].name);
items[i].flags |= CFG_ITEM_DEFAULT;
}
if (items[i].flags & CFG_ITEM_DEFAULT && items[i].default_value != NULL) {
/*
* First try to handle the generic types.
*/
switch (items[i].type) {
case CFG_TYPE_BIT:
if (bstrcasecmp(items[i].default_value, "on")) {
*(items[i].ui32value) |= items[i].code;
} else if (bstrcasecmp(items[i].default_value, "off")) {
*(items[i].ui32value) &= ~(items[i].code);
}
break;
case CFG_TYPE_BOOL:
if (bstrcasecmp(items[i].default_value, "yes") ||
bstrcasecmp(items[i].default_value, "true")) {
*(items[i].boolvalue) = true;
} else if (bstrcasecmp(items[i].default_value, "no") ||
bstrcasecmp(items[i].default_value, "false")) {
*(items[i].boolvalue) = false;
}
break;
case CFG_TYPE_PINT32:
case CFG_TYPE_INT32:
case CFG_TYPE_SIZE32:
*(items[i].ui32value) = str_to_int32(items[i].default_value);
break;
case CFG_TYPE_INT64:
*(items[i].i64value) = str_to_int64(items[i].default_value);
break;
case CFG_TYPE_SIZE64:
*(items[i].ui64value) = str_to_uint64(items[i].default_value);
break;
case CFG_TYPE_SPEED:
*(items[i].ui64value) = str_to_uint64(items[i].default_value);
break;
case CFG_TYPE_TIME:
*(items[i].utimevalue) = str_to_int64(items[i].default_value);
break;
case CFG_TYPE_STRNAME:
case CFG_TYPE_STR:
*items[i].value = bstrdup(items[i].default_value);
break;
case CFG_TYPE_DIR: {
POOL_MEM pathname(PM_FNAME);
pm_strcpy(pathname, items[i].default_value);
if (*pathname.c_str() != '|') {
int size;
/*
* Make sure we have enough room
*/
size = pathname.size() + 1024;
pathname.check_size(size);
do_shell_expansion(pathname.c_str(), pathname.size());
}
*items[i].value = bstrdup(pathname.c_str());
break;
}
case CFG_TYPE_ADDRESSES:
init_default_addresses(items[i].dlistvalue, items[i].default_value);
break;
default:
/*
* None of the generic types fired if there is a registered callback call that now.
*/
if (m_init_res) {
m_init_res(&items[i]);
}
break;
}
}
/*
* If this triggers, take a look at lib/parse_conf.h
*/
if (i >= MAX_RES_ITEMS) {
Emsg1(M_ERROR_TERM, 0, _("Too many items in %s resource\n"), m_resources[type - m_r_first]);
}
}
}
}
bool run_cmd(JCR *jcr)
{
struct timeval tv;
struct timezone tz;
struct timespec timeout;
int errstat = 0;
Dsm_check(200);
Dmsg1(200, "Run_cmd: %s\n", jcr->dir_bsock->msg);
/* If we do not need the FD, we are doing a virtual backup. */
if (jcr->no_client_used()) {
do_vbackup(jcr);
return false;
}
jcr->sendJobStatus(JS_WaitFD); /* wait for FD to connect */
Dmsg2(050, "sd_calls_client=%d sd_client=%d\n", jcr->sd_calls_client, jcr->sd_client);
if (jcr->sd_calls_client) {
if (!read_client_hello(jcr)) {
return false;
}
/*
* Authenticate the File daemon
*/
Dmsg0(050, "=== Authenticate FD\n");
if (jcr->authenticated || !authenticate_filed(jcr, jcr->file_bsock, jcr->FDVersion)) {
Dmsg1(050, "Authentication failed Job %s\n", jcr->Job);
Jmsg(jcr, M_FATAL, 0, _("Unable to authenticate File daemon\n"));
} else {
jcr->authenticated = true;
}
} else if (!jcr->sd_client) {
/* We wait to receive connection from Client */
gettimeofday(&tv, &tz);
timeout.tv_nsec = tv.tv_usec * 1000;
timeout.tv_sec = tv.tv_sec + me->client_wait;
Dmsg3(050, "%s waiting %d sec for FD to contact SD key=%s\n",
jcr->Job, (int)(timeout.tv_sec-time(NULL)), jcr->sd_auth_key);
Dmsg3(800, "=== Block Job=%s jid=%d %p\n", jcr->Job, jcr->JobId, jcr);
/*
* Wait for the File daemon to contact us to start the Job,
* when he does, we will be released, unless the 30 minutes
* expires.
*/
P(mutex);
while ( !jcr->authenticated && !job_canceled(jcr) ) {
errstat = pthread_cond_timedwait(&jcr->job_start_wait, &mutex, &timeout);
if (errstat == ETIMEDOUT || errstat == EINVAL || errstat == EPERM) {
break;
}
Dmsg1(800, "=== Auth cond errstat=%d\n", errstat);
}
Dmsg4(050, "=== Auth=%d jid=%d canceled=%d errstat=%d\n",
jcr->JobId, jcr->authenticated, job_canceled(jcr), errstat);
V(mutex);
Dmsg2(800, "Auth fail or cancel for jid=%d %p\n", jcr->JobId, jcr);
}
memset(jcr->sd_auth_key, 0, strlen(jcr->sd_auth_key));
if (jcr->authenticated && !job_canceled(jcr)) {
Dmsg2(800, "Running jid=%d %p\n", jcr->JobId, jcr);
run_job(jcr); /* Run the job */
}
Dmsg2(800, "Done jid=%d %p\n", jcr->JobId, jcr);
return false;
}
/*
* Authenticate File daemon connection
*/
int authenticate_file_daemon(JCR *jcr)
{
BSOCK *fd = jcr->file_bsock;
CLIENT *client = jcr->client;
char dirname[MAX_NAME_LENGTH];
int tls_local_need = BNET_TLS_NONE;
int tls_remote_need = BNET_TLS_NONE;
int compatible = true;
bool auth_success = false;
/*
* Send my name to the File daemon then do authentication
*/
bstrncpy(dirname, director->name(), sizeof(dirname));
bash_spaces(dirname);
/* Timeout Hello after 1 min */
btimer_t *tid = start_bsock_timer(fd, AUTH_TIMEOUT);
if (!fd->fsend(hello, "", dirname, DIR_VERSION)) {
stop_bsock_timer(tid);
Jmsg(jcr, M_FATAL, 0, _("Error sending Hello to File daemon at \"%s:%d\". ERR=%s\n"),
fd->host(), fd->port(), fd->bstrerror());
Dmsg3(50, _("Error sending Hello to File daemon at \"%s:%d\". ERR=%s\n"),
fd->host(), fd->port(), fd->bstrerror());
return 0;
}
Dmsg1(dbglvl, "Sent: %s", fd->msg);
/* TLS Requirement */
if (client->tls_enable) {
if (client->tls_require) {
tls_local_need = BNET_TLS_REQUIRED;
} else {
tls_local_need = BNET_TLS_OK;
}
}
if (client->tls_authenticate) {
tls_local_need = BNET_TLS_REQUIRED;
}
auth_success = cram_md5_respond(fd, client->password, &tls_remote_need, &compatible);
if (auth_success) {
auth_success = cram_md5_challenge(fd, client->password, tls_local_need, compatible);
if (!auth_success) {
Dmsg1(dbglvl, "cram_auth failed for %s\n", fd->who());
}
} else {
Dmsg1(dbglvl, "cram_get_auth failed for %s\n", fd->who());
}
if (!auth_success) {
stop_bsock_timer(tid);
Dmsg0(dbglvl, _("Director and File daemon passwords or names not the same.\n"));
Jmsg(jcr, M_FATAL, 0,
_("Unable to authenticate with File daemon at \"%s:%d\". Possible causes:\n"
"Passwords or names not the same or\n"
"Maximum Concurrent Jobs exceeded on the FD or\n"
"FD networking messed up (restart daemon).\n"
"For help, please see: " MANUAL_AUTH_URL "\n"),
fd->host(), fd->port());
return 0;
}
/* Verify that the remote host is willing to meet our TLS requirements */
if (tls_remote_need < tls_local_need && tls_local_need != BNET_TLS_OK && tls_remote_need != BNET_TLS_OK) {
stop_bsock_timer(tid);
Jmsg(jcr, M_FATAL, 0, _("Authorization problem: FD \"%s:%s\" did not advertise required TLS support.\n"),
fd->who(), fd->host());
return 0;
}
/* Verify that we are willing to meet the remote host's requirements */
if (tls_remote_need > tls_local_need && tls_local_need != BNET_TLS_OK && tls_remote_need != BNET_TLS_OK) {
stop_bsock_timer(tid);
Jmsg(jcr, M_FATAL, 0, _("Authorization problem: FD at \"%s:%d\" requires TLS.\n"),
fd->host(), fd->port());
return 0;
}
/* Is TLS Enabled? */
if (tls_local_need >= BNET_TLS_OK && tls_remote_need >= BNET_TLS_OK) {
/* Engage TLS! Full Speed Ahead! */
if (!bnet_tls_client(client->tls_ctx, fd, client->tls_allowed_cns)) {
stop_bsock_timer(tid);
Jmsg(jcr, M_FATAL, 0, _("TLS negotiation failed with FD at \"%s:%d\".\n"),
fd->host(), fd->port());
return 0;
}
if (client->tls_authenticate) { /* tls authentication only? */
fd->free_tls(); /* yes, shutdown tls */
}
}
Dmsg1(116, ">filed: %s", fd->msg);
if (fd->recv() <= 0) {
stop_bsock_timer(tid);
Dmsg1(dbglvl, _("Bad response from File daemon to Hello command: ERR=%s\n"),
fd->bstrerror());
Jmsg(jcr, M_FATAL, 0, _("Bad response from File daemon at \"%s:%d\" to Hello command: ERR=%s\n"),
fd->host(), fd->port(), fd->bstrerror());
return 0;
}
Dmsg1(110, "<filed: %s", fd->msg);
stop_bsock_timer(tid);
jcr->FDVersion = 0;
if (strncmp(fd->msg, FDOKhello, sizeof(FDOKhello)) != 0 &&
sscanf(fd->msg, FDOKnewHello, &jcr->FDVersion) != 1) {
Dmsg0(dbglvl, _("File daemon rejected Hello command\n"));
Jmsg(jcr, M_FATAL, 0, _("File daemon at \"%s:%d\" rejected Hello command\n"),
fd->host(), fd->port());
return 0;
}
return 1;
}
bool CONFIG::parse_config()
{
LEX *lc = NULL;
int token, i, pass;
int res_type = 0;
enum parse_state state = p_none;
RES_ITEM *items = NULL;
int level = 0;
static bool first = true;
int errstat;
const char *cf = m_cf;
LEX_ERROR_HANDLER *scan_error = m_scan_error;
LEX_WARNING_HANDLER *scan_warning = m_scan_warning;
int err_type = m_err_type;
if (first && (errstat = rwl_init(&m_res_lock)) != 0) {
berrno be;
Jmsg1(NULL, M_ABORT, 0, _("Unable to initialize resource lock. ERR=%s\n"),
be.bstrerror(errstat));
}
first = false;
char *full_path = (char *)alloca(MAX_PATH + 1);
if (!find_config_file(cf, full_path, MAX_PATH +1)) {
Jmsg0(NULL, M_ABORT, 0, _("Config filename too long.\n"));
}
cf = full_path;
/*
* Make two passes. The first builds the name symbol table,
* and the second picks up the items.
*/
Dmsg0(900, "Enter parse_config()\n");
for (pass = 1; pass <= 2; pass++) {
Dmsg1(900, "parse_config pass %d\n", pass);
if ((lc = lex_open_file(lc, cf, scan_error, scan_warning)) == NULL) {
berrno be;
/*
* We must create a lex packet to print the error
*/
lc = (LEX *)malloc(sizeof(LEX));
memset(lc, 0, sizeof(LEX));
if (scan_error) {
lc->scan_error = scan_error;
} else {
lex_set_default_error_handler(lc);
}
if (scan_warning) {
lc->scan_warning = scan_warning;
} else {
lex_set_default_warning_handler(lc);
}
lex_set_error_handler_error_type(lc, err_type) ;
scan_err2(lc, _("Cannot open config file \"%s\": %s\n"),
cf, be.bstrerror());
free(lc);
return 0;
}
lex_set_error_handler_error_type(lc, err_type) ;
while ((token=lex_get_token(lc, T_ALL)) != T_EOF) {
Dmsg3(900, "parse state=%d pass=%d got token=%s\n", state, pass,
lex_tok_to_str(token));
switch (state) {
case p_none:
if (token == T_EOL) {
break;
} else if (token == T_UTF8_BOM) {
/*
* We can assume the file is UTF-8 as we have seen a UTF-8 BOM
*/
break;
} else if (token == T_UTF16_BOM) {
scan_err0(lc, _("Currently we cannot handle UTF-16 source files. "
"Please convert the conf file to UTF-8\n"));
goto bail_out;
} else if (token != T_IDENTIFIER) {
scan_err1(lc, _("Expected a Resource name identifier, got: %s"), lc->str);
goto bail_out;
}
for (i = 0; m_resources[i].name; i++) {
if (bstrcasecmp(m_resources[i].name, lc->str)) {
items = m_resources[i].items;
if (!items) {
break;
}
state = p_resource;
res_type = m_resources[i].rcode;
init_resource(res_type, items, pass);
break;
}
}
if (state == p_none) {
scan_err1(lc, _("expected resource name, got: %s"), lc->str);
goto bail_out;
}
break;
case p_resource:
switch (token) {
case T_BOB:
level++;
break;
case T_IDENTIFIER:
if (level != 1) {
scan_err1(lc, _("not in resource definition: %s"), lc->str);
goto bail_out;
}
for (i = 0; items[i].name; i++) {
if (bstrcasecmp(items[i].name, lc->str)) {
/*
* If the CFG_ITEM_NO_EQUALS flag is set we do NOT
* scan for = after the keyword
*/
if (!(items[i].flags & CFG_ITEM_NO_EQUALS)) {
token = lex_get_token(lc, T_SKIP_EOL);
Dmsg1 (900, "in T_IDENT got token=%s\n", lex_tok_to_str(token));
if (token != T_EQUALS) {
scan_err1(lc, _("expected an equals, got: %s"), lc->str);
goto bail_out;
}
}
/*
* See if we are processing a deprecated keyword if so warn the user about it.
*/
if (items[i].flags & CFG_ITEM_DEPRECATED) {
scan_warn2(lc, _("using deprecated keyword %s on line %d"), items[i].name, lc->line_no);
/*
* As we only want to warn we continue parsing the config. So no goto bail_out here.
*/
}
Dmsg1(800, "calling handler for %s\n", items[i].name);
/*
* Call item handler
*/
if (!store_resource(items[i].type, lc, &items[i], i, pass)) {
/*
* None of the generic types fired if there is a registered callback call that now.
*/
if (m_store_res) {
m_store_res(lc, &items[i], i, pass);
}
}
i = -1;
break;
}
}
if (i >= 0) {
Dmsg2(900, "level=%d id=%s\n", level, lc->str);
Dmsg1(900, "Keyword = %s\n", lc->str);
scan_err1(lc, _("Keyword \"%s\" not permitted in this resource.\n"
"Perhaps you left the trailing brace off of the previous resource."), lc->str);
goto bail_out;
}
break;
case T_EOB:
level--;
state = p_none;
Dmsg0(900, "T_EOB => define new resource\n");
if (((URES *)m_res_all)->hdr.name == NULL) {
scan_err0(lc, _("Name not specified for resource"));
goto bail_out;
}
save_resource(res_type, items, pass); /* save resource */
break;
case T_EOL:
break;
default:
scan_err2(lc, _("unexpected token %d %s in resource definition"),
token, lex_tok_to_str(token));
goto bail_out;
}
break;
default:
scan_err1(lc, _("Unknown parser state %d\n"), state);
goto bail_out;
}
}
if (state != p_none) {
scan_err0(lc, _("End of conf file reached with unclosed resource."));
goto bail_out;
}
if (debug_level >= 900 && pass == 2) {
int i;
for (i = m_r_first; i <= m_r_last; i++) {
dump_resource(i, m_res_head[i-m_r_first], prtmsg, NULL);
}
}
lc = lex_close_file(lc);
}
Dmsg0(900, "Leave parse_config()\n");
return 1;
bail_out:
if (lc) {
lc = lex_close_file(lc);
}
return 0;
}
/*
* Scan for right hand side of Include options (keyword=option) is
* converted into one or two characters. Verifyopts=xxxx is Vxxxx:
* Whatever is found is concatenated to the opts string.
* This code is also used inside an Options resource.
*/
static void scan_include_options(LEX *lc, int keyword, char *opts, int optlen)
{
int i;
char option[3];
int lcopts = lc->options;
option[0] = 0; /* default option = none */
option[2] = 0; /* terminate options */
lc->options |= LOPT_STRING; /* force string */
lex_get_token(lc, T_STRING); /* expect at least one option */
if (keyword == INC_KW_VERIFY) { /* special case */
/* ***FIXME**** ensure these are in permitted set */
bstrncat(opts, "V", optlen); /* indicate Verify */
bstrncat(opts, lc->str, optlen);
bstrncat(opts, ":", optlen); /* terminate it */
Dmsg3(900, "Catopts=%s option=%s optlen=%d\n", opts, option,optlen);
} else if (keyword == INC_KW_ACCURATE) { /* special case */
/* ***FIXME**** ensure these are in permitted set */
bstrncat(opts, "C", optlen); /* indicate Accurate */
bstrncat(opts, lc->str, optlen);
bstrncat(opts, ":", optlen); /* terminate it */
Dmsg3(900, "Catopts=%s option=%s optlen=%d\n", opts, option,optlen);
} else if (keyword == INC_KW_BASEJOB) { /* special case */
/* ***FIXME**** ensure these are in permitted set */
bstrncat(opts, "J", optlen); /* indicate BaseJob */
bstrncat(opts, lc->str, optlen);
bstrncat(opts, ":", optlen); /* terminate it */
Dmsg3(900, "Catopts=%s option=%s optlen=%d\n", opts, option,optlen);
} else if (keyword == INC_KW_STRIPPATH) { /* another special case */
if (!is_an_integer(lc->str)) {
scan_err1(lc, _("Expected a strip path positive integer, got:%s:"), lc->str);
}
bstrncat(opts, "P", optlen); /* indicate strip path */
bstrncat(opts, lc->str, optlen);
bstrncat(opts, ":", optlen); /* terminate it */
Dmsg3(900, "Catopts=%s option=%s optlen=%d\n", opts, option,optlen);
/*
* Standard keyword options for Include/Exclude
*/
} else {
for (i=0; FS_options[i].name; i++) {
if (FS_options[i].keyword == keyword && strcasecmp(lc->str, FS_options[i].name) == 0) {
/* NOTE! maximum 2 letters here or increase option[3] */
option[0] = FS_options[i].option[0];
option[1] = FS_options[i].option[1];
i = 0;
break;
}
}
if (i != 0) {
scan_err1(lc, _("Expected a FileSet option keyword, got:%s:"), lc->str);
} else { /* add option */
bstrncat(opts, option, optlen);
Dmsg3(900, "Catopts=%s option=%s optlen=%d\n", opts, option,optlen);
}
}
lc->options = lcopts;
/* If option terminated by comma, eat it */
if (lc->ch == ',') {
lex_get_token(lc, T_ALL); /* yes, eat comma */
}
}
/*
* Items needed:
* mr.PoolId must be set
* mr.StorageId should also be set
* mr.ScratchPoolId could be set (used if create==true)
* jcr->wstore
* jcr->db
* jcr->pool
* MEDIA_DBR mr with PoolId set
* create -- whether or not to create a new volume
*/
int find_next_volume_for_append(JCR *jcr, MEDIA_DBR *mr, int index,
bool create, bool prune)
{
int retry = 0;
bool ok;
bool InChanger;
STORE *store = jcr->wstore;
bstrncpy(mr->MediaType, store->media_type, sizeof(mr->MediaType));
Dmsg3(100, "find_next_vol_for_append: JobId=%u PoolId=%d, MediaType=%s\n",
(uint32_t)jcr->JobId, (int)mr->PoolId, mr->MediaType);
/*
* If we are using an Autochanger, restrict Volume
* search to the Autochanger on the first pass
*/
InChanger = store->autochanger;
/*
* Find the Next Volume for Append
*/
db_lock(jcr->db);
for ( ;; ) {
bstrncpy(mr->VolStatus, "Append", sizeof(mr->VolStatus)); /* want only appendable volumes */
/*
* 1. Look for volume with "Append" status.
*/
ok = db_find_next_volume(jcr, jcr->db, index, InChanger, mr);
if (!ok) {
Dmsg4(150, "after find_next_vol ok=%d index=%d InChanger=%d Vstat=%s\n",
ok, index, InChanger, mr->VolStatus);
/*
* 2. Try finding a recycled volume
*/
ok = find_recycled_volume(jcr, InChanger, mr);
Dmsg2(150, "find_recycled_volume ok=%d FW=%d\n", ok, mr->FirstWritten);
if (!ok) {
/*
* 3. Try recycling any purged volume
*/
ok = recycle_oldest_purged_volume(jcr, InChanger, mr);
if (!ok) {
/*
* 4. Try pruning Volumes
*/
if (prune) {
Dmsg0(150, "Call prune_volumes\n");
prune_volumes(jcr, InChanger, mr);
}
ok = recycle_oldest_purged_volume(jcr, InChanger, mr);
if (!ok && create) {
Dmsg4(150, "after prune volumes_vol ok=%d index=%d InChanger=%d Vstat=%s\n",
ok, index, InChanger, mr->VolStatus);
/*
* 5. Try pulling a volume from the Scratch pool
*/
ok = get_scratch_volume(jcr, InChanger, mr);
Dmsg4(150, "after get scratch volume ok=%d index=%d InChanger=%d Vstat=%s\n",
ok, index, InChanger, mr->VolStatus);
}
/*
* If we are using an Autochanger and have not found
* a volume, retry looking for any volume.
*/
if (!ok && InChanger) {
InChanger = false;
continue; /* retry again accepting any volume */
}
}
}
if (!ok && create) {
/*
* 6. Try "creating" a new Volume
*/
ok = newVolume(jcr, mr);
}
/*
* Look at more drastic ways to find an Appendable Volume
*/
if (!ok && (jcr->pool->purge_oldest_volume ||
jcr->pool->recycle_oldest_volume)) {
Dmsg2(200, "No next volume found. PurgeOldest=%d\n RecyleOldest=%d",
jcr->pool->purge_oldest_volume, jcr->pool->recycle_oldest_volume);
/* Find oldest volume to recycle */
ok = db_find_next_volume(jcr, jcr->db, -1, InChanger, mr);
Dmsg1(200, "Find oldest=%d Volume\n", ok);
if (ok && prune) {
UAContext *ua;
Dmsg0(200, "Try purge Volume.\n");
/*
* 7. Try to purging oldest volume only if not UA calling us.
*/
ua = new_ua_context(jcr);
if (jcr->pool->purge_oldest_volume && create) {
Jmsg(jcr, M_INFO, 0, _("Purging oldest volume \"%s\"\n"), mr->VolumeName);
ok = purge_jobs_from_volume(ua, mr);
/*
* 8. or try recycling the oldest volume
*/
} else if (jcr->pool->recycle_oldest_volume) {
Jmsg(jcr, M_INFO, 0, _("Pruning oldest volume \"%s\"\n"), mr->VolumeName);
ok = prune_volume(ua, mr);
}
free_ua_context(ua);
if (ok) {
ok = recycle_volume(jcr, mr);
Dmsg1(400, "Recycle after purge oldest=%d\n", ok);
}
}
}
}
Dmsg2(100, "VolJobs=%d FirstWritten=%d\n", mr->VolJobs, mr->FirstWritten);
if (ok) {
/* If we can use the volume, check if it is expired */
if (has_volume_expired(jcr, mr)) {
if (retry++ < 200) { /* sanity check */
continue; /* try again from the top */
} else {
Jmsg(jcr, M_ERROR, 0, _(
"We seem to be looping trying to find the next volume. I give up.\n"));
}
}
}
break;
} /* end for loop */
db_unlock(jcr->db);
Dmsg1(150, "return ok=%d find_next_vol\n", ok);
return ok;
}
static inline DEVICE *m_init_dev(JCR *jcr, DEVRES *device, bool new_init)
{
struct stat statp;
int errstat;
DCR *dcr = NULL;
DEVICE *dev = NULL;
uint32_t max_bs;
Dmsg1(400, "max_block_size in device res is %u\n", device->max_block_size);
/*
* If no device type specified, try to guess
*/
if (!device->dev_type) {
/*
* Check that device is available
*/
if (stat(device->device_name, &statp) < 0) {
berrno be;
Jmsg2(jcr, M_ERROR, 0, _("Unable to stat device %s: ERR=%s\n"),
device->device_name, be.bstrerror());
return NULL;
}
if (S_ISDIR(statp.st_mode)) {
device->dev_type = B_FILE_DEV;
} else if (S_ISCHR(statp.st_mode)) {
device->dev_type = B_TAPE_DEV;
} else if (S_ISFIFO(statp.st_mode)) {
device->dev_type = B_FIFO_DEV;
} else if (!bit_is_set(CAP_REQMOUNT, device->cap_bits)) {
Jmsg2(jcr, M_ERROR, 0,
_("%s is an unknown device type. Must be tape or directory, st_mode=%x\n"),
device->device_name, statp.st_mode);
return NULL;
}
}
/*
* See what type of device is wanted.
*/
switch (device->dev_type) {
/*
* When using dynamic loading use the init_backend_dev() function
* for any type of device not being of the type file.
*/
#ifndef HAVE_DYNAMIC_SD_BACKENDS
#ifdef HAVE_GFAPI
case B_GFAPI_DEV:
dev = New(gfapi_device);
break;
#endif
#ifdef HAVE_OBJECTSTORE
case B_OBJECT_STORE_DEV:
dev = New(object_store_device);
break;
#endif
#ifdef HAVE_RADOS
case B_RADOS_DEV:
dev = New(rados_device);
break;
#endif
#ifdef HAVE_CEPHFS
case B_CEPHFS_DEV:
dev = New(cephfs_device);
break;
#endif
#ifdef HAVE_ELASTO
case B_ELASTO_DEV:
dev = New(elasto_device);
break;
#endif
#ifdef HAVE_WIN32
case B_TAPE_DEV:
dev = New(win32_tape_device);
break;
case B_FIFO_DEV:
dev = New(win32_fifo_device);
break;
#else
case B_TAPE_DEV:
dev = New(unix_tape_device);
break;
case B_FIFO_DEV:
dev = New(unix_fifo_device);
break;
#endif
#endif /* HAVE_DYNAMIC_SD_BACKENDS */
#ifdef HAVE_WIN32
case B_FILE_DEV:
dev = New(win32_file_device);
break;
#else
case B_FILE_DEV:
dev = New(unix_file_device);
break;
#endif
default:
#ifdef HAVE_DYNAMIC_SD_BACKENDS
dev = init_backend_dev(jcr, device->dev_type);
#endif
break;
}
if (!dev) {
Jmsg2(jcr, M_ERROR, 0, _("%s has an unknown device type %d\n"),
device->device_name, device->dev_type);
return NULL;
}
dev->clear_slot(); /* unknown */
/*
* Copy user supplied device parameters from Resource
*/
dev->dev_name = get_memory(strlen(device->device_name) + 1);
pm_strcpy(dev->dev_name, device->device_name);
if (device->device_options) {
dev->dev_options = get_memory(strlen(device->device_options) + 1);
pm_strcpy(dev->dev_options, device->device_options);
}
dev->prt_name = get_memory(strlen(device->device_name) + strlen(device->name()) + 20);
/*
* We edit "Resource-name" (physical-name)
*/
Mmsg(dev->prt_name, "\"%s\" (%s)", device->name(), device->device_name);
Dmsg1(400, "Allocate dev=%s\n", dev->print_name());
copy_bits(CAP_MAX, device->cap_bits, dev->capabilities);
/*
* current block sizes
*/
dev->min_block_size = device->min_block_size;
dev->max_block_size = device->max_block_size;
dev->max_volume_size = device->max_volume_size;
dev->max_file_size = device->max_file_size;
dev->max_concurrent_jobs = device->max_concurrent_jobs;
dev->volume_capacity = device->volume_capacity;
dev->max_rewind_wait = device->max_rewind_wait;
dev->max_open_wait = device->max_open_wait;
dev->max_open_vols = device->max_open_vols;
dev->vol_poll_interval = device->vol_poll_interval;
dev->max_spool_size = device->max_spool_size;
dev->drive_index = device->drive_index;
dev->autoselect = device->autoselect;
dev->norewindonclose = device->norewindonclose;
dev->dev_type = device->dev_type;
dev->device = device;
/*
* Sanity check
*/
if (dev->vol_poll_interval && dev->vol_poll_interval < 60) {
dev->vol_poll_interval = 60;
}
device->dev = dev;
if (dev->is_fifo()) {
dev->set_cap(CAP_STREAM); /* set stream device */
}
/*
* If the device requires mount :
* - Check that the mount point is available
* - Check that (un)mount commands are defined
*/
if (dev->is_file() && dev->requires_mount()) {
if (!device->mount_point || stat(device->mount_point, &statp) < 0) {
berrno be;
dev->dev_errno = errno;
Jmsg2(jcr, M_ERROR_TERM, 0, _("Unable to stat mount point %s: ERR=%s\n"),
device->mount_point, be.bstrerror());
}
if (!device->mount_command || !device->unmount_command) {
Jmsg0(jcr, M_ERROR_TERM, 0, _("Mount and unmount commands must defined for a device which requires mount.\n"));
}
}
/*
* Sanity check
*/
if (dev->max_block_size == 0) {
max_bs = DEFAULT_BLOCK_SIZE;
} else {
max_bs = dev->max_block_size;
}
if (dev->min_block_size > max_bs) {
Jmsg(jcr, M_ERROR_TERM, 0, _("Min block size > max on device %s\n"), dev->print_name());
}
if (dev->max_block_size > MAX_BLOCK_LENGTH) {
Jmsg3(jcr, M_ERROR, 0, _("Block size %u on device %s is too large, using default %u\n"),
dev->max_block_size, dev->print_name(), DEFAULT_BLOCK_SIZE);
dev->max_block_size = 0;
}
if (dev->max_block_size % TAPE_BSIZE != 0) {
Jmsg3(jcr, M_WARNING, 0, _("Max block size %u not multiple of device %s block size=%d.\n"),
dev->max_block_size, dev->print_name(), TAPE_BSIZE);
}
if (dev->max_volume_size != 0 && dev->max_volume_size < (dev->max_block_size << 4)) {
Jmsg(jcr, M_ERROR_TERM, 0, _("Max Vol Size < 8 * Max Block Size for device %s\n"), dev->print_name());
}
dev->errmsg = get_pool_memory(PM_EMSG);
*dev->errmsg = 0;
if ((errstat = dev->init_mutex()) != 0) {
berrno be;
dev->dev_errno = errstat;
Mmsg1(dev->errmsg, _("Unable to init mutex: ERR=%s\n"), be.bstrerror(errstat));
Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg);
}
if ((errstat = pthread_cond_init(&dev->wait, NULL)) != 0) {
berrno be;
dev->dev_errno = errstat;
Mmsg1(dev->errmsg, _("Unable to init cond variable: ERR=%s\n"), be.bstrerror(errstat));
Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg);
}
if ((errstat = pthread_cond_init(&dev->wait_next_vol, NULL)) != 0) {
berrno be;
dev->dev_errno = errstat;
Mmsg1(dev->errmsg, _("Unable to init cond variable: ERR=%s\n"), be.bstrerror(errstat));
Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg);
}
if ((errstat = pthread_mutex_init(&dev->spool_mutex, NULL)) != 0) {
berrno be;
dev->dev_errno = errstat;
Mmsg1(dev->errmsg, _("Unable to init spool mutex: ERR=%s\n"), be.bstrerror(errstat));
Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg);
}
if ((errstat = dev->init_acquire_mutex()) != 0) {
berrno be;
dev->dev_errno = errstat;
Mmsg1(dev->errmsg, _("Unable to init acquire mutex: ERR=%s\n"), be.bstrerror(errstat));
Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg);
}
if ((errstat = dev->init_read_acquire_mutex()) != 0) {
berrno be;
dev->dev_errno = errstat;
Mmsg1(dev->errmsg, _("Unable to init read acquire mutex: ERR=%s\n"), be.bstrerror(errstat));
Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg);
}
dev->set_mutex_priorities();
#ifdef xxx
if ((errstat = rwl_init(&dev->lock)) != 0) {
berrno be;
dev->dev_errno = errstat;
Mmsg1(dev->errmsg, _("Unable to init mutex: ERR=%s\n"), be.bstrerror(errstat));
Jmsg0(jcr, M_ERROR_TERM, 0, dev->errmsg);
}
#endif
dev->clear_opened();
dev->attached_dcrs = New(dlist(dcr, &dcr->dev_link));
Dmsg2(100, "init_dev: tape=%d dev_name=%s\n", dev->is_tape(), dev->dev_name);
dev->initiated = true;
Dmsg3(100, "dev=%s dev_max_bs=%u max_bs=%u\n", dev->dev_name, dev->device->max_block_size, dev->max_block_size);
return dev;
}
/*
* Set the block size of the device.
* If the volume block size is zero, we set the max block size to what is
* configured in the device resource i.e. dev->device->max_block_size.
*
* If dev->device->max_block_size is zero, do nothing and leave dev->max_block_size as it is.
*/
void DEVICE::set_blocksizes(DCR *dcr) {
DEVICE* dev = this;
JCR* jcr = dcr->jcr;
uint32_t max_bs;
Dmsg4(100, "Device %s has dev->device->max_block_size of %u and dev->max_block_size of %u, dcr->VolMaxBlocksize is %u\n",
dev->print_name(), dev->device->max_block_size, dev->max_block_size, dcr->VolMaxBlocksize);
if (dcr->VolMaxBlocksize == 0 && dev->device->max_block_size != 0) {
Dmsg2(100, "setting dev->max_block_size to dev->device->max_block_size=%u "
"on device %s because dcr->VolMaxBlocksize is 0\n",
dev->device->max_block_size, dev->print_name());
dev->min_block_size = dev->device->min_block_size;
dev->max_block_size = dev->device->max_block_size;
} else if (dcr->VolMaxBlocksize != 0) {
dev->min_block_size = dcr->VolMinBlocksize;
dev->max_block_size = dcr->VolMaxBlocksize;
}
/*
* Sanity check
*/
if (dev->max_block_size == 0) {
max_bs = DEFAULT_BLOCK_SIZE;
} else {
max_bs = dev->max_block_size;
}
if (dev->min_block_size > max_bs) {
Jmsg(jcr, M_ERROR_TERM, 0, _("Min block size > max on device %s\n"), dev->print_name());
}
if (dev->max_block_size > MAX_BLOCK_LENGTH) {
Jmsg3(jcr, M_ERROR, 0, _("Block size %u on device %s is too large, using default %u\n"),
dev->max_block_size, dev->print_name(), DEFAULT_BLOCK_SIZE);
dev->max_block_size = 0;
}
if (dev->max_block_size % TAPE_BSIZE != 0) {
Jmsg3(jcr, M_WARNING, 0, _("Max block size %u not multiple of device %s block size=%d.\n"),
dev->max_block_size, dev->print_name(), TAPE_BSIZE);
}
if (dev->max_volume_size != 0 && dev->max_volume_size < (dev->max_block_size << 4)) {
Jmsg(jcr, M_ERROR_TERM, 0, _("Max Vol Size < 8 * Max Block Size for device %s\n"), dev->print_name());
}
Dmsg3(100, "set minblocksize to %d, maxblocksize to %d on device %s\n",
dev->min_block_size, dev->max_block_size, dev->print_name());
/*
* If blocklen is not dev->max_block_size create a new block with the right size.
* (as header is always dev->label_block_size which is preset with DEFAULT_BLOCK_SIZE)
*/
if (dcr->block) {
if (dcr->block->buf_len != dev->max_block_size) {
Dmsg2(100, "created new block of buf_len: %u on device %s\n",
dev->max_block_size, dev->print_name());
free_block(dcr->block);
dcr->block = new_block(dev);
Dmsg2(100, "created new block of buf_len: %u on device %s, freeing block\n",
dcr->block->buf_len, dev->print_name());
}
}
}
/*
* Close both pipes and free resources
*
* Returns: 0 on success
* berrno on failure
*/
int close_bpipe(BPIPE *bpipe)
{
int chldstatus = 0;
int status = 0;
int wait_option;
int remaining_wait;
pid_t wpid = 0;
/* Close pipes */
if (bpipe->rfd) {
fclose(bpipe->rfd);
bpipe->rfd = NULL;
}
if (bpipe->wfd) {
fclose(bpipe->wfd);
bpipe->wfd = NULL;
}
if (bpipe->wait == 0) {
wait_option = 0; /* wait indefinitely */
} else {
wait_option = WNOHANG; /* don't hang */
}
remaining_wait = bpipe->wait;
/* wait for worker child to exit */
for ( ;; ) {
Dmsg2(800, "Wait for %d opt=%d\n", bpipe->worker_pid, wait_option);
do {
wpid = waitpid(bpipe->worker_pid, &chldstatus, wait_option);
} while (wpid == -1 && (errno == EINTR || errno == EAGAIN));
if (wpid == bpipe->worker_pid || wpid == -1) {
berrno be;
status = errno;
Dmsg3(800, "Got break wpid=%d status=%d ERR=%s\n", wpid, chldstatus,
wpid==-1?be.bstrerror():"none");
break;
}
Dmsg3(800, "Got wpid=%d status=%d ERR=%s\n", wpid, chldstatus,
wpid==-1?strerror(errno):"none");
if (remaining_wait > 0) {
bmicrosleep(1, 0); /* wait one second */
remaining_wait--;
} else {
status = ETIME; /* set error status */
wpid = -1;
break; /* don't wait any longer */
}
}
if (wpid > 0) {
if (WIFEXITED(chldstatus)) { /* process exit()ed */
status = WEXITSTATUS(chldstatus);
if (status != 0) {
Dmsg1(800, "Non-zero status %d returned from child.\n", status);
status |= b_errno_exit; /* exit status returned */
}
Dmsg1(800, "child status=%d\n", status & ~b_errno_exit);
} else if (WIFSIGNALED(chldstatus)) { /* process died */
#ifndef HAVE_WIN32
status = WTERMSIG(chldstatus);
#else
status = 1; /* fake child status */
#endif
Dmsg1(800, "Child died from signal %d\n", status);
status |= b_errno_signal; /* exit signal returned */
}
}
if (bpipe->timer_id) {
stop_child_timer(bpipe->timer_id);
}
free(bpipe);
Dmsg2(800, "returning status=%d,%d\n", status & ~(b_errno_exit|b_errno_signal), status);
return status;
}
/*
* Run an external program. Optionally wait a specified number
* of seconds. Program killed if wait exceeded (it is done by the
* watchdog, as fgets is a blocking function).
*
* If the watchdog kills the program, fgets returns, and ferror is set
* to 1 (=>SUCCESS), so we check if the watchdog killed the program.
*
* Return the full output from the program (not only the first line).
*
* Contrary to my normal calling conventions, this program
*
* Returns: 0 on success
* non-zero on error == berrno status
*
*/
int run_program_full_output(char *prog, int wait, POOLMEM *&results)
{
BPIPE *bpipe;
int stat1, stat2;
char *mode;
POOLMEM* tmp;
char *buf;
const int bufsize = 32000;
Dsm_check(200);
tmp = get_pool_memory(PM_MESSAGE);
buf = (char *)malloc(bufsize+1);
results[0] = 0;
mode = (char *)"r";
bpipe = open_bpipe(prog, wait, mode);
if (!bpipe) {
stat1 = ENOENT;
goto bail_out;
}
Dsm_check(200);
tmp[0] = 0;
while (1) {
buf[0] = 0;
fgets(buf, bufsize, bpipe->rfd);
buf[bufsize] = 0;
pm_strcat(tmp, buf);
if (feof(bpipe->rfd)) {
stat1 = 0;
Dmsg1(900, "Run program fgets stat=%d\n", stat1);
break;
} else {
stat1 = ferror(bpipe->rfd);
}
if (stat1 < 0) {
berrno be;
Dmsg2(200, "Run program fgets stat=%d ERR=%s\n", stat1, be.bstrerror());
break;
} else if (stat1 != 0) {
Dmsg1(900, "Run program fgets stat=%d\n", stat1);
if (bpipe->timer_id && bpipe->timer_id->killed) {
Dmsg1(250, "Run program saw fgets killed=%d\n", bpipe->timer_id->killed);
break;
}
}
}
/*
* We always check whether the timer killed the program. We would see
* an eof even when it does so we just have to trust the killed flag
* and set the timer values to avoid edge cases where the program ends
* just as the timer kills it.
*/
if (bpipe->timer_id && bpipe->timer_id->killed) {
Dmsg1(150, "Run program fgets killed=%d\n", bpipe->timer_id->killed);
pm_strcpy(tmp, _("Program killed by BAREOS (timeout)\n"));
stat1 = ETIME;
}
pm_strcpy(results, tmp);
Dmsg3(1900, "resadr=0x%x reslen=%d res=%s\n", results, strlen(results), results);
stat2 = close_bpipe(bpipe);
stat1 = stat2 != 0 ? stat2 : stat1;
Dmsg1(900, "Run program returning %d\n", stat1);
bail_out:
free_pool_memory(tmp);
free(buf);
return stat1;
}
/*
* BSR + EOF => begin of EOF + EIO
* BSR + BSR + EOF => last block
* current_block = -1
*/
int vtape::bsr(int count)
{
ASSERT(online);
ASSERT(current_file >= 0);
ASSERT(count == 1);
ASSERT(fd >= 0);
check_eof();
if (!count) {
return 0;
}
int ret=0;
int last_f=0;
int last_b=0;
boffset_t last=-1, last2=-1;
boffset_t orig = lseek(fd, 0, SEEK_CUR);
int orig_f = current_file;
int orig_b = current_block;
Dmsg4(dbglevel, "bsr(%i) cur_blk=%i orig=%lli cur_FM=%lli\n",
count, current_block, orig, cur_FM);
/* begin of tape, do nothing */
if (atBOT) {
errno = EIO;
return -1;
}
/* at EOF 0:-1 BOT=0 EOD=0 EOF=0 ERR: Input/output error */
if (atEOF) {
lseek(fd, cur_FM, SEEK_SET);
atEOF = false;
if (current_file > 0) {
current_file--;
}
current_block=-1;
errno = EIO;
return -1;
}
/*
* First, go to cur/last_FM and read all blocks to find the good one
*/
if (cur_FM == orig) { /* already just before EOF */
lseek(fd, last_FM, SEEK_SET);
} else {
lseek(fd, cur_FM, SEEK_SET);
}
ret = read_fm(VT_READ_EOF);
do {
if (!atEOF) {
last2 = last; /* keep track of the 2 last blocs position */
last = lseek(fd, 0, SEEK_CUR);
last_f = current_file;
last_b = current_block;
Dmsg6(dbglevel, "EOF=%i last2=%lli last=%lli < orig=%lli %i:%i\n",
atEOF, last2, last, orig, current_file, current_block);
}
ret = fsr(1);
} while ((lseek(fd, 0, SEEK_CUR) < orig) && (ret == 0));
if (last2 > 0 && atEOF) { /* we take the previous position */
lseek(fd, last2, SEEK_SET);
current_file = last_f;
current_block = last_b - 1;
Dmsg3(dbglevel, "1 set offset2=%lli %i:%i\n",
last, current_file, current_block);
} else if (last > 0) {
lseek(fd, last, SEEK_SET);
current_file = last_f;
current_block = last_b;
Dmsg3(dbglevel, "2 set offset=%lli %i:%i\n",
last, current_file, current_block);
} else {
lseek(fd, orig, SEEK_SET);
current_file = orig_f;
current_block = orig_b;
return -1;
}
Dmsg2(dbglevel, "bsr %i:%i\n", current_file, current_block);
errno=0;
atEOT = atEOF = atEOD = false;
atBOT = (lseek(fd, 0, SEEK_CUR) - (sizeof(uint32_t)+2*sizeof(boffset_t))) == 0;
if (orig_b == -1) {
current_block = orig_b;
}
return 0;
}
/*
* Send data read from an already open file descriptor.
*
* We return 1 on sucess and 0 on errors.
*
* ***FIXME***
* We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
* reading.
* Currently this is not a problem as the only other stream, resource forks,
* are not handled as sparse files.
*/
static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
DIGEST *signing_digest)
{
BSOCK *sd = jcr->store_bsock;
uint64_t fileAddr = 0; /* file address */
char *rbuf, *wbuf;
int32_t rsize = jcr->buf_size; /* read buffer size */
POOLMEM *msgsave;
CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
const uint8_t *cipher_input;
uint32_t cipher_input_len;
uint32_t cipher_block_size;
uint32_t encrypted_len;
#ifdef FD_NO_SEND_TEST
return 1;
#endif
msgsave = sd->msg;
rbuf = sd->msg; /* read buffer */
wbuf = sd->msg; /* write buffer */
cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
#ifdef HAVE_LIBZ
uLong compress_len = 0;
uLong max_compress_len = 0;
const Bytef *cbuf = NULL;
int zstat;
if (ff_pkt->flags & FO_GZIP) {
if (ff_pkt->flags & FO_SPARSE) {
cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
} else {
cbuf = (Bytef *)jcr->compress_buf;
max_compress_len = jcr->compress_buf_size; /* set max length */
}
wbuf = jcr->compress_buf; /* compressed output here */
cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
/*
* Only change zlib parameters if there is no pending operation.
* This should never happen as deflatereset is called after each
* deflate.
*/
if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
/* set gzip compression level - must be done per file */
if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
set_jcr_job_status(jcr, JS_ErrorTerminated);
goto err;
}
}
}
#else
const uint32_t max_compress_len = 0;
#endif
if (ff_pkt->flags & FO_ENCRYPT) {
if (ff_pkt->flags & FO_SPARSE) {
Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
goto err;
}
/* Allocate the cipher context */
if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true,
&cipher_block_size)) == NULL) {
/* Shouldn't happen! */
Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
goto err;
}
/*
* Grow the crypto buffer, if necessary.
* crypto_cipher_update() will buffer up to (cipher_block_size - 1).
* We grow crypto_buf to the maximum number of blocks that
* could be returned for the given read buffer size.
* (Using the larger of either rsize or max_compress_len)
*/
jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf,
(MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
cipher_block_size - 1) / cipher_block_size * cipher_block_size);
wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */
}
/*
* Send Data header to Storage daemon
* <file-index> <stream> <info>
*/
if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
sd->bstrerror());
goto err;
}
Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
/*
* Make space at beginning of buffer for fileAddr because this
* same buffer will be used for writing if compression is off.
*/
if (ff_pkt->flags & FO_SPARSE) {
rbuf += SPARSE_FADDR_SIZE;
rsize -= SPARSE_FADDR_SIZE;
#ifdef HAVE_FREEBSD_OS
/*
* To read FreeBSD partitions, the read size must be
* a multiple of 512.
*/
rsize = (rsize/512) * 512;
#endif
}
/* a RAW device read on win32 only works if the buffer is a multiple of 512 */
#ifdef HAVE_WIN32
if (S_ISBLK(ff_pkt->statp.st_mode))
rsize = (rsize/512) * 512;
#endif
/*
* Read the file data
*/
while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
/* Check for sparse blocks */
if (ff_pkt->flags & FO_SPARSE) {
ser_declare;
bool allZeros = false;
if ((sd->msglen == rsize &&
fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size) ||
((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
(uint64_t)ff_pkt->statp.st_size == 0)) {
allZeros = is_buf_zero(rbuf, rsize);
}
if (!allZeros) {
/* Put file address as first data in buffer */
ser_begin(wbuf, SPARSE_FADDR_SIZE);
ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
}
fileAddr += sd->msglen; /* update file address */
/* Skip block of all zeros */
if (allZeros) {
continue; /* skip block of zeros */
}
}
jcr->ReadBytes += sd->msglen; /* count bytes read */
/* Uncompressed cipher input length */
cipher_input_len = sd->msglen;
/* Update checksum if requested */
if (digest) {
crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
}
/* Update signing digest if requested */
if (signing_digest) {
crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
}
#ifdef HAVE_LIBZ
/* Do compression if turned on */
if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
set_jcr_job_status(jcr, JS_ErrorTerminated);
goto err;
}
compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
/* reset zlib stream to be able to begin from scratch again */
if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
set_jcr_job_status(jcr, JS_ErrorTerminated);
goto err;
}
Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
sd->msglen);
sd->msglen = compress_len; /* set compressed length */
cipher_input_len = compress_len;
}
#endif
/*
* Note, here we prepend the current record length to the beginning
* of the encrypted data. This is because both sparse and compression
* restore handling want records returned to them with exactly the
* same number of bytes that were processed in the backup handling.
* That is, both are block filters rather than a stream. When doing
* compression, the compression routines may buffer data, so that for
* any one record compressed, when it is decompressed the same size
* will not be obtained. Of course, the buffered data eventually comes
* out in subsequent crypto_cipher_update() calls or at least
* when crypto_cipher_finalize() is called. Unfortunately, this
* "feature" of encryption enormously complicates the restore code.
*/
if (ff_pkt->flags & FO_ENCRYPT) {
uint32_t initial_len = 0;
ser_declare;
if (ff_pkt->flags & FO_SPARSE) {
cipher_input_len += SPARSE_FADDR_SIZE;
}
/* Encrypt the length of the input block */
uint8_t packet_len[sizeof(uint32_t)];
ser_begin(packet_len, sizeof(uint32_t));
ser_uint32(cipher_input_len); /* store data len in begin of buffer */
Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
(uint8_t *)jcr->crypto.crypto_buf, &initial_len)) {
/* Encryption failed. Shouldn't happen. */
Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
goto err;
}
/* Encrypt the input block */
if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
(uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) {
if ((initial_len + encrypted_len) == 0) {
/* No full block of data available, read more data */
continue;
}
Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
sd->msglen);
sd->msglen = initial_len + encrypted_len; /* set encrypted length */
} else {
/* Encryption failed. Shouldn't happen. */
Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
goto err;
}
}
/* Send the buffer to the Storage daemon */
if (ff_pkt->flags & FO_SPARSE) {
sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
}
sd->msg = wbuf; /* set correct write buffer */
if (!sd->send()) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
sd->bstrerror());
goto err;
}
Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
/* #endif */
jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
sd->msg = msgsave; /* restore read buffer */
} /* end while read file data */
if (sd->msglen < 0) { /* error */
berrno be;
Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
if (jcr->JobErrors++ > 1000) { /* insanity check */
Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
}
} else if (ff_pkt->flags & FO_ENCRYPT) {
/*
* For encryption, we must call finalize to push out any
* buffered data.
*/
if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf,
&encrypted_len)) {
/* Padding failed. Shouldn't happen. */
Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
goto err;
}
/* Note, on SSL pre-0.9.7, there is always some output */
if (encrypted_len > 0) {
sd->msglen = encrypted_len; /* set encrypted length */
sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
if (!sd->send()) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
sd->bstrerror());
goto err;
}
Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
sd->msg = msgsave; /* restore bnet buffer */
}
}
if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
sd->bstrerror());
goto err;
}
/* Free the cipher context */
if (cipher_ctx) {
crypto_cipher_free(cipher_ctx);
}
return 1;
err:
/* Free the cipher context */
if (cipher_ctx) {
crypto_cipher_free(cipher_ctx);
}
sd->msg = msgsave; /* restore bnet buffer */
sd->msglen = 0;
return 0;
}
bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
{
BSOCK *sd = jcr->store_bsock;
char attribs[MAXSTRING];
char attribsEx[MAXSTRING];
int attr_stream;
int stat;
#ifdef FD_NO_SEND_TEST
return true;
#endif
Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname);
/* Find what data stream we will use, then encode the attributes */
if ((data_stream = select_data_stream(ff_pkt)) == STREAM_NONE) {
/* This should not happen */
Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
return false;
}
encode_stat(attribs, &ff_pkt->statp, ff_pkt->LinkFI, data_stream);
/* Now possibly extend the attributes */
attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs, attribsEx);
jcr->lock();
jcr->JobFiles++; /* increment number of files sent */
ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
pm_strcpy(jcr->last_fname, ff_pkt->fname);
jcr->unlock();
/*
* Send Attributes header to Storage daemon
* <file-index> <stream> <info>
*/
if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
sd->bstrerror());
return false;
}
Dmsg1(300, ">stored: attrhdr %s\n", sd->msg);
/*
* Send file attributes to Storage daemon
* File_index
* File type
* Filename (full path)
* Encoded attributes
* Link name (if type==FT_LNK or FT_LNKSAVED)
* Encoded extended-attributes (for Win32)
*
* For a directory, link is the same as fname, but with trailing
* slash. For a linked file, link is the link.
*/
if (ff_pkt->type != FT_DELETED) { /* already stripped */
strip_path(ff_pkt);
}
if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link);
stat = sd->fsend("%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles,
ff_pkt->type, ff_pkt->fname, 0, attribs, 0, ff_pkt->link, 0,
attribsEx, 0);
} else if (ff_pkt->type == FT_DIREND || ff_pkt->type == FT_REPARSE) {
/* Here link is the canonical filename (i.e. with trailing slash) */
stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
ff_pkt->type, ff_pkt->link, 0, attribs, 0, 0, attribsEx, 0);
} else {
stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles,
ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0);
}
if (ff_pkt->type != FT_DELETED) {
unstrip_path(ff_pkt);
}
Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
if (!stat) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
sd->bstrerror());
return false;
}
sd->signal(BNET_EOD); /* indicate end of attributes data */
return true;
}
/*
* Rewind the device.
*
* Returns: true on success
* false on failure
*/
bool generic_tape_device::rewind(DCR *dcr)
{
struct mtop mt_com;
unsigned int i;
bool first = true;
Dmsg3(400, "rewind res=%d fd=%d %s\n", num_reserved(), m_fd, prt_name);
state &= ~(ST_EOT | ST_EOF | ST_WEOT); /* Remove EOF/EOT flags */
block_num = file = 0;
file_size = 0;
file_addr = 0;
if (m_fd < 0) {
return false;
}
mt_com.mt_op = MTREW;
mt_com.mt_count = 1;
/*
* If we get an I/O error on rewind, it is probably because
* the drive is actually busy. We loop for (about 5 minutes)
* retrying every 5 seconds.
*/
for (i = max_rewind_wait; ; i -= 5) {
if (d_ioctl(m_fd, MTIOCTOP, (char *)&mt_com) < 0) {
berrno be;
clrerror(mt_com.mt_op);
if (i == max_rewind_wait) {
Dmsg1(200, "Rewind error, %s. retrying ...\n", be.bstrerror());
}
/*
* This is a gross hack, because if the user has the
* device mounted (i.e. open), then uses mtx to load
* a tape, the current open file descriptor is invalid.
* So, we close the drive and re-open it.
*/
if (first && dcr) {
int oo_mode = open_mode;
d_close(m_fd);
clear_opened();
open(dcr, oo_mode);
if (m_fd < 0) {
return false;
}
first = false;
continue;
}
#ifdef HAVE_SUN_OS
if (dev_errno == EIO) {
Mmsg1(errmsg, _("No tape loaded or drive offline on %s.\n"), prt_name);
return false;
}
#else
if (dev_errno == EIO && i > 0) {
Dmsg0(200, "Sleeping 5 seconds.\n");
bmicrosleep(5, 0);
continue;
}
#endif
Mmsg2(errmsg, _("Rewind error on %s. ERR=%s.\n"), prt_name, be.bstrerror());
return false;
}
break;
}
return true;
}
void b_free_jcr(const char *file, int line, JCR *jcr)
{
struct s_last_job *je;
Dmsg3(dbglvl, "Enter free_jcr jid=%u from %s:%d\n", jcr->JobId, file, line);
#else
void free_jcr(JCR *jcr)
{
struct s_last_job *je;
Dmsg3(dbglvl, "Enter free_jcr jid=%u use_count=%d Job=%s\n",
jcr->JobId, jcr->use_count(), jcr->Job);
#endif
lock_jcr_chain();
jcr->dec_use_count(); /* decrement use count */
if (jcr->use_count() < 0) {
Jmsg2(jcr, M_ERROR, 0, _("JCR use_count=%d JobId=%d\n"),
jcr->use_count(), jcr->JobId);
}
if (jcr->JobId > 0) {
Dmsg3(dbglvl, "Dec free_jcr jid=%u use_count=%d Job=%s\n",
jcr->JobId, jcr->use_count(), jcr->Job);
}
if (jcr->use_count() > 0) { /* if in use */
unlock_jcr_chain();
return;
}
if (jcr->JobId > 0) {
Dmsg3(dbglvl, "remove jcr jid=%u use_count=%d Job=%s\n",
jcr->JobId, jcr->use_count(), jcr->Job);
}
remove_jcr(jcr); /* remove Jcr from chain */
unlock_jcr_chain();
dequeue_messages(jcr);
job_end_pop(jcr); /* pop and call hooked routines */
Dmsg1(dbglvl, "End job=%d\n", jcr->JobId);
/* Keep some statistics */
switch (jcr->get_JobType()) {
case JT_BACKUP:
case JT_VERIFY:
case JT_RESTORE:
case JT_MIGRATE:
case JT_COPY:
case JT_ADMIN:
/* Keep list of last jobs, but not Console where JobId==0 */
if (jcr->JobId > 0) {
lock_last_jobs_list();
num_jobs_run++;
je = (struct s_last_job *)malloc(sizeof(struct s_last_job));
memset(je, 0, sizeof(struct s_last_job)); /* zero in case unset fields */
je->Errors = jcr->JobErrors;
je->JobType = jcr->get_JobType();
je->JobId = jcr->JobId;
je->VolSessionId = jcr->VolSessionId;
je->VolSessionTime = jcr->VolSessionTime;
bstrncpy(je->Job, jcr->Job, sizeof(je->Job));
je->JobFiles = jcr->JobFiles;
je->JobBytes = jcr->JobBytes;
je->JobStatus = jcr->JobStatus;
je->JobLevel = jcr->get_JobLevel();
je->start_time = jcr->start_time;
je->end_time = time(NULL);
if (!last_jobs) {
init_last_jobs_list();
}
last_jobs->append(je);
if (last_jobs->size() > max_last_jobs) {
je = (struct s_last_job *)last_jobs->first();
last_jobs->remove(je);
free(je);
}
unlock_last_jobs_list();
}
break;
default:
break;
}
if (jcr->daemon_free_jcr) {
jcr->daemon_free_jcr(jcr); /* call daemon free routine */
}
free_common_jcr(jcr);
close_msg(NULL); /* flush any daemon messages */
garbage_collect_memory_pool();
Dmsg0(dbglvl, "Exit free_jcr\n");
}
/*
* Remove jcr from thread specific data, but
* but make sure it is us who are attached.
*/
void remove_jcr_from_tsd(JCR *jcr)
{
JCR *tjcr = get_jcr_from_tsd();
if (tjcr == jcr) {
set_jcr_in_tsd(INVALID_JCR);
}
}
/*
* Called here from "core" expand code to look up a variable
*/
static var_rc_t lookup_var(var_t *ctx,
void *my_ctx,
const char *var_ptr,
int var_len,
int var_inc,
int var_index,
const char **val_ptr,
int *val_len,
int *val_size)
{
POOL_MEM buf(PM_NAME);
char *val, *p, *v;
var_rc_t status;
int count;
/*
* Note, if val_size > 0 and val_ptr!=NULL, the core code will free() it
*/
if ((status = lookup_built_in_var(ctx, my_ctx, var_ptr, var_len, var_index,
val_ptr, val_len, val_size)) == VAR_OK) {
return VAR_OK;
}
if ((status = lookup_counter_var(ctx, my_ctx, var_ptr, var_len, var_inc, var_index,
val_ptr, val_len, val_size)) == VAR_OK) {
return VAR_OK;
}
/*
* Look in environment
*/
buf.check_size(var_len + 1);
pm_memcpy(buf, var_ptr, var_len);
(buf.c_str())[var_len] = 0;
Dmsg1(100, "Var=%s\n", buf.c_str());
if ((val = getenv(buf.c_str())) == NULL) {
return VAR_ERR_UNDEFINED_VARIABLE;
}
/*
* He wants to index the "array"
*/
count = 1;
/*
* Find the size of the "array" each element is separated by a |
*/
for (p = val; *p; p++) {
if (*p == '|') {
count++;
}
}
Dmsg3(100, "For %s, reqest index=%d have=%d\n", buf.c_str(), var_index, count);
/*
* -1 => return size of array
*/
if (var_index == -1) {
int len;
if (count == 1) { /* if not array */
len = strlen(val); /* return length of string */
} else {
len = count; /* else return # array items */
}
*val_len = Mmsg(buf, "%d", len);
*val_ptr = bstrdup(buf.c_str());
*val_size = 0; /* don't try to free val_ptr */
return VAR_OK;
}
if (var_index < -1 || var_index > --count) {
// return VAR_ERR_SUBMATCH_OUT_OF_RANGE;
return VAR_ERR_UNDEFINED_VARIABLE;
}
/*
* Now find the particular item (var_index) he wants
*/
count = 0;
for (p = val; *p; ) {
if (*p == '|') {
if (count < var_index) {
val = ++p;
count++;
continue;
}
break;
}
p++;
}
buf.check_size(p - val);
Dmsg2(100, "val=%s len=%d\n", val, p - val);
/*
* Make a copy of item, and pass it back
*/
v = (char *)malloc(p-val+1);
memcpy(v, val, p-val);
v[p-val] = 0;
*val_ptr = v;
*val_len = p-val;
*val_size = p-val+1;
Dmsg1(100, "v=%s\n", v);
return VAR_OK;
}
/*
* Try to limit the bandwidth of a network connection
*/
void BSOCK::control_bwlimit(int bytes)
{
btime_t now, temp;
int64_t usec_sleep;
/*
* If nothing written or read nothing todo.
*/
if (bytes == 0) {
return;
}
/*
* See if this is the first time we enter here.
*/
now = get_current_btime();
if (m_last_tick == 0) {
m_nb_bytes = bytes;
m_last_tick = now;
return;
}
/*
* Calculate the number of microseconds since the last check.
*/
temp = now - m_last_tick;
/*
* Less than 0.1ms since the last call, see the next time
*/
if (temp < 100) {
m_nb_bytes += bytes;
return;
}
/*
* Keep track of how many bytes are written in this timeslice.
*/
m_nb_bytes += bytes;
m_last_tick = now;
if (debug_level >= 400) {
Dmsg3(400, "control_bwlimit: now = %lld, since = %lld, nb_bytes = %d\n", now, temp, m_nb_bytes);
}
/*
* Take care of clock problems (>10s)
*/
if (temp > 10000000) {
return;
}
/*
* Remove what was authorised to be written in temp usecs.
*/
m_nb_bytes -= (int64_t)(temp * ((double)m_bwlimit / 1000000.0));
if (m_nb_bytes < 0) {
/*
* If more was authorized then used but bursting is not enabled
* reset the counter as these bytes cannot be used later on when
* we are exceeding our bandwidth.
*/
if (!m_use_bursting) {
m_nb_bytes = 0;
}
return;
}
/*
* What exceed should be converted in sleep time
*/
usec_sleep = (int64_t)(m_nb_bytes /((double)m_bwlimit / 1000000.0));
if (usec_sleep > 100) {
if (debug_level >= 400) {
Dmsg1(400, "control_bwlimit: sleeping for %lld usecs\n", usec_sleep);
}
/*
* Sleep the right number of usecs.
*/
while (1) {
bmicrosleep(0, usec_sleep);
now = get_current_btime();
/*
* See if we slept enough or that bmicrosleep() returned early.
*/
if ((now - m_last_tick) < usec_sleep) {
usec_sleep -= (now - m_last_tick);
continue;
} else {
m_last_tick = now;
break;
}
}
/*
* Subtract the number of bytes we could have sent during the sleep
* time given the bandwidth limit set. We only do this when we are
* allowed to burst e.g. use unused bytes from previous timeslices
* to get an overall bandwidth limiting which may sometimes be below
* the bandwidth and sometimes above it but the average will be near
* the set bandwidth.
*/
if (m_use_bursting) {
m_nb_bytes -= (int64_t)(usec_sleep * ((double)m_bwlimit / 1000000.0));
} else {
m_nb_bytes = 0;
}
}
}
/*
* Check if any time limits or use limits have expired
* if so, set the VolStatus appropriately.
*/
bool has_volume_expired(JCR *jcr, MEDIA_DBR *mr)
{
bool expired = false;
/*
* Check limits and expirations if "Append" and it has been used
* i.e. mr->VolJobs > 0
*
*/
if (strcmp(mr->VolStatus, "Append") == 0 && mr->VolJobs > 0) {
/* First handle Max Volume Bytes */
if ((mr->MaxVolBytes > 0 && mr->VolBytes >= mr->MaxVolBytes)) {
Jmsg(jcr, M_INFO, 0, _("Max Volume bytes exceeded. "
"Marking Volume \"%s\" as Full.\n"), mr->VolumeName);
bstrncpy(mr->VolStatus, "Full", sizeof(mr->VolStatus));
expired = true;
/* Now see if Volume should only be used once */
} else if (mr->VolBytes > 0 && jcr->pool->use_volume_once) {
Jmsg(jcr, M_INFO, 0, _("Volume used once. "
"Marking Volume \"%s\" as Used.\n"), mr->VolumeName);
bstrncpy(mr->VolStatus, "Used", sizeof(mr->VolStatus));
expired = true;
/* Now see if Max Jobs written to volume */
} else if (mr->MaxVolJobs > 0 && mr->MaxVolJobs <= mr->VolJobs) {
Jmsg(jcr, M_INFO, 0, _("Max Volume jobs exceeded. "
"Marking Volume \"%s\" as Used.\n"), mr->VolumeName);
Dmsg3(100, "MaxVolJobs=%d JobId=%d Vol=%s\n", mr->MaxVolJobs,
(uint32_t)jcr->JobId, mr->VolumeName);
bstrncpy(mr->VolStatus, "Used", sizeof(mr->VolStatus));
expired = true;
/* Now see if Max Files written to volume */
} else if (mr->MaxVolFiles > 0 && mr->MaxVolFiles <= mr->VolFiles) {
Jmsg(jcr, M_INFO, 0, _("Max Volume files exceeded. "
"Marking Volume \"%s\" as Used.\n"), mr->VolumeName);
bstrncpy(mr->VolStatus, "Used", sizeof(mr->VolStatus));
expired = true;
/* Finally, check Use duration expiration */
} else if (mr->VolUseDuration > 0) {
utime_t now = time(NULL);
/* See if Vol Use has expired */
if (mr->VolUseDuration <= (now - mr->FirstWritten)) {
Jmsg(jcr, M_INFO, 0, _("Max configured use duration exceeded. "
"Marking Volume \"%s\" as Used.\n"), mr->VolumeName);
bstrncpy(mr->VolStatus, "Used", sizeof(mr->VolStatus));
expired = true;
}
}
}
if (expired) {
/* Need to update media */
Dmsg1(150, "Vol=%s has expired update media record\n", mr->VolumeName);
if (!db_update_media_record(jcr, jcr->db, mr)) {
Jmsg(jcr, M_ERROR, 0, _("Catalog error updating volume \"%s\". ERR=%s"),
mr->VolumeName, db_strerror(jcr->db));
}
}
Dmsg2(150, "Vol=%s expired=%d\n", mr->VolumeName, expired);
return expired;
}
bool run_cmd(JCR *jcr)
{
struct timeval tv;
struct timezone tz;
struct timespec timeout;
int errstat = 0;
BSOCK *cl;
int fd_version = 0;
int sd_version = 0;
char job_name[500];
int i;
int stat;
Dsm_check(200);
Dmsg1(200, "Run_cmd: %s\n", jcr->dir_bsock->msg);
/* If we do not need the FD, we are doing a virtual backup. */
if (jcr->no_client_used()) {
do_vbackup(jcr);
return false;
}
jcr->sendJobStatus(JS_WaitFD); /* wait for FD to connect */
Dmsg2(050, "sd_calls_client=%d sd_client=%d\n", jcr->sd_calls_client, jcr->sd_client);
if (jcr->sd_calls_client) {
/* We connected to Client, so finish work */
cl = jcr->file_bsock;
if (!cl) {
Jmsg0(jcr, M_FATAL, 0, _("Client socket not open. Could not connect to Client.\n"));
Dmsg0(050, "Client socket not open. Could not connect to Client.\n");
return false;
}
/* Get response to Hello command sent earlier */
Dmsg0(050, "Read Hello command from Client\n");
for (i=0; i<60; i++) {
stat = cl->recv();
if (stat <= 0) {
bmicrosleep(1, 0);
} else {
break;
}
}
if (stat <= 0) {
berrno be;
Jmsg1(jcr, M_FATAL, 0, _("Recv request to Client failed. ERR=%s\n"),
be.bstrerror());
Dmsg1(050, _("Recv request to Client failed. ERR=%s\n"), be.bstrerror());
return false;
}
Dmsg1(050, "Got from FD: %s\n", cl->msg);
if (sscanf(cl->msg, "Hello Bacula SD: Start Job %127s %d %d", job_name, &fd_version, &sd_version) != 3) {
Jmsg1(jcr, M_FATAL, 0, _("Bad Hello from Client: %s.\n"), cl->msg);
Dmsg1(050, _("Bad Hello from Client: %s.\n"), cl->msg);
return false;
}
unbash_spaces(job_name);
jcr->FDVersion = fd_version;
jcr->SDVersion = sd_version;
Dmsg1(050, "FDVersion=%d\n", fd_version);
/*
* Authenticate the File daemon
*/
Dmsg0(050, "=== Authenticate FD\n");
if (jcr->authenticated || !authenticate_filed(jcr)) {
Dmsg1(050, "Authentication failed Job %s\n", jcr->Job);
Jmsg(jcr, M_FATAL, 0, _("Unable to authenticate File daemon\n"));
} else {
jcr->authenticated = true;
}
} else if (!jcr->sd_client) {
/* We wait to receive connection from Client */
gettimeofday(&tv, &tz);
timeout.tv_nsec = tv.tv_usec * 1000;
timeout.tv_sec = tv.tv_sec + me->client_wait;
Dmsg3(050, "%s waiting %d sec for FD to contact SD key=%s\n",
jcr->Job, (int)(timeout.tv_sec-time(NULL)), jcr->sd_auth_key);
Dmsg3(800, "=== Block Job=%s jid=%d %p\n", jcr->Job, jcr->JobId, jcr);
/*
* Wait for the File daemon to contact us to start the Job,
* when he does, we will be released, unless the 30 minutes
* expires.
*/
P(mutex);
while ( !jcr->authenticated && !job_canceled(jcr) ) {
errstat = pthread_cond_timedwait(&jcr->job_start_wait, &mutex, &timeout);
if (errstat == ETIMEDOUT || errstat == EINVAL || errstat == EPERM) {
break;
}
Dmsg1(800, "=== Auth cond errstat=%d\n", errstat);
}
Dmsg4(050, "=== Auth=%d jid=%d canceled=%d errstat=%d\n",
jcr->JobId, jcr->authenticated, job_canceled(jcr), errstat);
V(mutex);
Dmsg2(800, "Auth fail or cancel for jid=%d %p\n", jcr->JobId, jcr);
}
memset(jcr->sd_auth_key, 0, strlen(jcr->sd_auth_key));
if (jcr->authenticated && !job_canceled(jcr)) {
Dmsg2(800, "Running jid=%d %p\n", jcr->JobId, jcr);
run_job(jcr); /* Run the job */
}
Dmsg2(800, "Done jid=%d %p\n", jcr->JobId, jcr);
return false;
}
bool do_job_run(JCR *jcr)
{
struct timeval tv;
struct timezone tz;
struct timespec timeout;
int errstat = 0;
jcr->sendJobStatus(JS_WaitFD); /* wait for FD to connect */
gettimeofday(&tv, &tz);
timeout.tv_nsec = tv.tv_usec * 1000;
timeout.tv_sec = tv.tv_sec + me->client_wait;
Dmsg3(50, "%s waiting %d sec for FD to contact SD key=%s\n",
jcr->Job, (int)(timeout.tv_sec-time(NULL)), jcr->sd_auth_key);
Dmsg2(800, "Wait FD for jid=%d %p\n", jcr->JobId, jcr);
/*
* Wait for the File daemon to contact us to start the Job,
* when he does, we will be released, unless the 30 minutes
* expires.
*/
P(mutex);
while (!jcr->authenticated && !job_canceled(jcr)) {
errstat = pthread_cond_timedwait(&jcr->job_start_wait, &mutex, &timeout);
if (errstat == ETIMEDOUT || errstat == EINVAL || errstat == EPERM) {
break;
}
Dmsg1(800, "=== Auth cond errstat=%d\n", errstat);
}
Dmsg3(50, "Auth=%d canceled=%d errstat=%d\n", jcr->authenticated,
job_canceled(jcr), errstat);
V(mutex);
Dmsg2(800, "Auth fail or cancel for jid=%d %p\n", jcr->JobId, jcr);
memset(jcr->sd_auth_key, 0, strlen(jcr->sd_auth_key));
switch (jcr->getJobProtocol()) {
case PT_NDMP:
if (jcr->authenticated && !job_canceled(jcr)) {
Dmsg2(800, "Running jid=%d %p\n", jcr->JobId, jcr);
/*
* Wait for the Job to finish. As we want exclusive access to
* things like the connection to the director we suspend this
* thread and let the actual NDMP connection wake us after it
* has performed the backup. E.g. instead of doing a busy wait
* we just hang on a conditional variable.
*/
Dmsg2(800, "Wait for end job jid=%d %p\n", jcr->JobId, jcr);
P(mutex);
pthread_cond_wait(&jcr->job_end_wait, &mutex);
V(mutex);
}
Dmsg2(800, "Done jid=%d %p\n", jcr->JobId, jcr);
/*
* For a NDMP backup we expect the protocol to send us either a nextrun cmd
* or a finish cmd to let us know they are finished.
*/
return true;
default:
/*
* Handle the file daemon session.
*/
if (jcr->authenticated && !job_canceled(jcr)) {
Dmsg2(800, "Running jid=%d %p\n", jcr->JobId, jcr);
run_job(jcr); /* Run the job */
}
Dmsg2(800, "Done jid=%d %p\n", jcr->JobId, jcr);
/*
* After a run cmd of a native backup we are done e.g.
* return false.
*/
return false;
}
}
/*
* Called here by find() for each file.
*
* Find the file, compute the MD5 or SHA1 and send it back to the Director
*/
static int verify_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
{
POOL_MEM attribs(PM_NAME),
attribsEx(PM_NAME);
int status;
BSOCK *dir;
if (job_canceled(jcr)) {
return 0;
}
dir = jcr->dir_bsock;
jcr->num_files_examined++; /* bump total file count */
switch (ff_pkt->type) {
case FT_LNKSAVED: /* Hard linked, file already saved */
Dmsg2(30, "FT_LNKSAVED saving: %s => %s\n", ff_pkt->fname, ff_pkt->link);
break;
case FT_REGE:
Dmsg1(30, "FT_REGE saving: %s\n", ff_pkt->fname);
break;
case FT_REG:
Dmsg1(30, "FT_REG saving: %s\n", ff_pkt->fname);
break;
case FT_LNK:
Dmsg2(30, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
break;
case FT_DIRBEGIN:
jcr->num_files_examined--; /* correct file count */
return 1; /* ignored */
case FT_REPARSE:
case FT_JUNCTION:
case FT_DIREND:
Dmsg1(30, "FT_DIR saving: %s\n", ff_pkt->fname);
break;
case FT_SPEC:
Dmsg1(30, "FT_SPEC saving: %s\n", ff_pkt->fname);
break;
case FT_RAW:
Dmsg1(30, "FT_RAW saving: %s\n", ff_pkt->fname);
break;
case FT_FIFO:
Dmsg1(30, "FT_FIFO saving: %s\n", ff_pkt->fname);
break;
case FT_NOACCESS: {
berrno be;
be.set_errno(ff_pkt->ff_errno);
Jmsg(jcr, M_NOTSAVED, 1, _(" Could not access %s: ERR=%s\n"), ff_pkt->fname, be.bstrerror());
jcr->JobErrors++;
return 1;
}
case FT_NOFOLLOW: {
berrno be;
be.set_errno(ff_pkt->ff_errno);
Jmsg(jcr, M_NOTSAVED, 1, _(" Could not follow link %s: ERR=%s\n"), ff_pkt->fname, be.bstrerror());
jcr->JobErrors++;
return 1;
}
case FT_NOSTAT: {
berrno be;
be.set_errno(ff_pkt->ff_errno);
Jmsg(jcr, M_NOTSAVED, 1, _(" Could not stat %s: ERR=%s\n"), ff_pkt->fname, be.bstrerror());
jcr->JobErrors++;
return 1;
}
case FT_DIRNOCHG:
case FT_NOCHG:
Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
return 1;
case FT_ISARCH:
Jmsg(jcr, M_SKIPPED, 1, _(" Archive file skipped: %s\n"), ff_pkt->fname);
return 1;
case FT_NORECURSE:
Jmsg(jcr, M_SKIPPED, 1, _(" Recursion turned off. Directory skipped: %s\n"), ff_pkt->fname);
ff_pkt->type = FT_DIREND; /* directory entry was backed up */
break;
case FT_NOFSCHG:
Jmsg(jcr, M_SKIPPED, 1, _(" File system change prohibited. Directory skipped: %s\n"), ff_pkt->fname);
return 1;
case FT_PLUGIN_CONFIG:
case FT_RESTORE_FIRST:
return 1; /* silently skip */
case FT_NOOPEN: {
berrno be;
be.set_errno(ff_pkt->ff_errno);
Jmsg(jcr, M_NOTSAVED, 1, _(" Could not open directory %s: ERR=%s\n"), ff_pkt->fname, be.bstrerror());
jcr->JobErrors++;
return 1;
}
default:
Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d: %s\n"), ff_pkt->type, ff_pkt->fname);
jcr->JobErrors++;
return 1;
}
/* Encode attributes and possibly extend them */
encode_stat(attribs.c_str(), &ff_pkt->statp, sizeof(ff_pkt->statp), ff_pkt->LinkFI, 0);
encode_attribsEx(jcr, attribsEx.c_str(), ff_pkt);
jcr->lock();
jcr->JobFiles++; /* increment number of files sent */
pm_strcpy(jcr->last_fname, ff_pkt->fname);
jcr->unlock();
/*
* Send file attributes to Director
* File_index
* Stream
* Verify Options
* Filename (full path)
* Encoded attributes
* Link name (if type==FT_LNK)
* For a directory, link is the same as fname, but with trailing
* slash. For a linked file, link is the link.
*/
/*
* Send file attributes to Director (note different format than for Storage)
*/
Dmsg2(400, "send ATTR inx=%d fname=%s\n", jcr->JobFiles, ff_pkt->fname);
if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
status = dir->fsend("%d %d %s %s%c%s%c%s%c", jcr->JobFiles,
STREAM_UNIX_ATTRIBUTES, ff_pkt->VerifyOpts, ff_pkt->fname,
0, attribs.c_str(), 0, ff_pkt->link, 0);
} else if (ff_pkt->type == FT_DIREND || ff_pkt->type == FT_REPARSE ||
ff_pkt->type == FT_JUNCTION) {
/*
* Here link is the canonical filename (i.e. with trailing slash)
*/
status = dir->fsend("%d %d %s %s%c%s%c%c", jcr->JobFiles,
STREAM_UNIX_ATTRIBUTES, ff_pkt->VerifyOpts, ff_pkt->link,
0, attribs.c_str(), 0, 0);
} else {
status = dir->fsend("%d %d %s %s%c%s%c%c", jcr->JobFiles,
STREAM_UNIX_ATTRIBUTES, ff_pkt->VerifyOpts, ff_pkt->fname,
0, attribs.c_str(), 0, 0);
}
Dmsg2(20, "filed>dir: attribs len=%d: msg=%s\n", dir->msglen, dir->msg);
if (!status) {
Jmsg(jcr, M_FATAL, 0, _("Network error in send to Director: ERR=%s\n"), bnet_strerror(dir));
return 0;
}
if (ff_pkt->type != FT_LNKSAVED &&
(S_ISREG(ff_pkt->statp.st_mode) &&
ff_pkt->flags & (FO_MD5 | FO_SHA1 | FO_SHA256 | FO_SHA512))) {
int digest_stream = STREAM_NONE;
DIGEST *digest = NULL;
char *digest_buf = NULL;
const char *digest_name = NULL;
if (calculate_file_chksum(jcr, ff_pkt, &digest, &digest_stream, &digest_buf, &digest_name)) {
/*
* Did digest initialization fail?
*/
if (digest_stream != STREAM_NONE && digest == NULL) {
Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"), stream_to_ascii(digest_stream));
} else if (digest && digest_buf) {
Dmsg3(400, "send inx=%d %s=%s\n", jcr->JobFiles, digest_name, digest_buf);
dir->fsend("%d %d %s *%s-%d*", jcr->JobFiles, digest_stream, digest_buf, digest_name, jcr->JobFiles);
Dmsg3(20, "filed>dir: %s len=%d: msg=%s\n", digest_name, dir->msglen, dir->msg);
}
}
/*
* Cleanup.
*/
if (digest_buf) {
free(digest_buf);
}
if (digest) {
crypto_digest_free(digest);
}
}
return 1;
}
