/* Release all freed pooled memory */
void close_memory_pool()
{
struct abufhead *buf, *next;
int count = 0;
uint64_t bytes = 0;
char ed1[50];
sm_check(__FILE__, __LINE__, false);
P(mutex);
for (int i=1; i<=PM_MAX; i++) {
buf = pool_ctl[i].free_buf;
while (buf) {
next = buf->next;
count++;
bytes += sizeof_pool_memory((char *)buf);
free((char *)buf);
buf = next;
}
pool_ctl[i].free_buf = NULL;
}
Dmsg2(DT_MEMORY|001, "Freed mem_pool count=%d size=%s\n", count, edit_uint64_with_commas(bytes, ed1));
if (chk_dbglvl(DT_MEMORY|1)) {
print_memory_pool_stats();
}
V(mutex);
}
void bmsg(UAContext *ua, const char *fmt, va_list arg_ptr)
{
BSOCK *bs = ua->UA_sock;
int maxlen, len;
POOLMEM *msg = NULL;
if (bs) {
msg = bs->msg;
}
if (!msg) {
msg = get_memory(5000);
}
maxlen = sizeof_pool_memory(msg) - 1;
if (maxlen < 4999) {
msg = realloc_pool_memory(msg, 5000);
maxlen = 4999;
}
len = bvsnprintf(msg, maxlen, fmt, arg_ptr);
if (len < 0 || len >= maxlen) {
pm_strcpy(msg, _("Message too long to display.\n"));
len = strlen(msg);
}
if (bs) {
bs->msg = msg;
bs->msglen = len;
bs->send();
} else { /* No UA, send to Job */
Jmsg(ua->jcr, M_INFO, 0, "%s", msg);
free_pool_memory(msg);
}
}
/*
* Format and send a message
* Returns: false on error
* true on success
*/
bool BSOCK::fsend(const char *fmt, ...)
{
va_list arg_ptr;
int maxlen;
if (errors || is_terminated()) {
return false;
}
/* This probably won't work, but we vsnprintf, then if we
* get a negative length or a length greater than our buffer
* (depending on which library is used), the printf was truncated, so
* get a bigger buffer and try again.
*/
for (;;) {
maxlen = sizeof_pool_memory(msg) - 1;
va_start(arg_ptr, fmt);
msglen = bvsnprintf(msg, maxlen, fmt, arg_ptr);
va_end(arg_ptr);
if (msglen >= 0 && msglen < (maxlen - 5)) {
break;
}
msg = realloc_pool_memory(msg, maxlen + maxlen / 2);
}
return send();
}
/*
* Save current working directory.
* Returns: true if OK
* false if failed
*/
bool saveCWD::save(JCR *jcr)
{
release(); /* clean up */
if (!fchdir_failed) {
m_fd = open(".", O_RDONLY);
if (m_fd < 0) {
berrno be;
Jmsg1(jcr, M_ERROR, 0, _("Cannot open current directory: ERR=%s\n"), be.bstrerror());
m_saved = false;
return false;
}
}
if (fchdir_failed) {
POOLMEM *buf = get_memory(5000);
m_cwd = (POOLMEM *)getcwd(buf, sizeof_pool_memory(buf));
if (m_cwd == NULL) {
berrno be;
Jmsg1(jcr, M_ERROR, 0, _("Cannot get current directory: ERR=%s\n"), be.bstrerror());
free_pool_memory(buf);
m_saved = false;
return false;
}
}
m_saved = true;
return true;
}
void bmsg(UAContext *ua, const char *fmt, va_list arg_ptr)
{
BSOCK *bs = ua->UA_sock;
int maxlen, len;
POOLMEM *msg = NULL;
va_list ap;
if (bs) {
msg = bs->msg;
}
if (!msg) {
msg = get_pool_memory(PM_EMSG);
}
again:
maxlen = sizeof_pool_memory(msg) - 1;
va_copy(ap, arg_ptr);
len = bvsnprintf(msg, maxlen, fmt, ap);
va_end(ap);
if (len < 0 || len >= maxlen) {
msg = realloc_pool_memory(msg, maxlen + maxlen/2);
goto again;
}
if (bs) {
bs->msg = msg;
bs->msglen = len;
bs->send();
} else { /* No UA, send to Job */
Jmsg(ua->jcr, M_INFO, 0, "%s", msg);
free_pool_memory(msg);
}
}
POOLMEM *sm_check_pool_memory_size(const char *fname, int lineno, POOLMEM *obuf, int32_t size)
{
ASSERT(obuf);
if (size <= sizeof_pool_memory(obuf)) {
return obuf;
}
return realloc_pool_memory(obuf, size);
}
POOLMEM *check_pool_memory_size(POOLMEM *obuf, int32_t size)
{
ASSERT(obuf);
if (size <= sizeof_pool_memory(obuf)) {
return obuf;
}
return realloc_pool_memory(obuf, size);
}
/*
* Terminate the signing digest and send it to the Storage daemon
*/
static inline bool terminate_signing_digest(b_save_ctx &bsctx)
{
uint32_t size = 0;
bool retval = false;
SIGNATURE *signature = NULL;
BSOCK *sd = bsctx.jcr->store_bsock;
if ((signature = crypto_sign_new(bsctx.jcr)) == NULL) {
Jmsg(bsctx.jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
goto bail_out;
}
if (!crypto_sign_add_signer(signature, bsctx.signing_digest, bsctx.jcr->crypto.pki_keypair)) {
Jmsg(bsctx.jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
goto bail_out;
}
/*
* Get signature size
*/
if (!crypto_sign_encode(signature, NULL, &size)) {
Jmsg(bsctx.jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
goto bail_out;
}
/*
* Grow the bsock buffer to fit our message if necessary
*/
if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
sd->msg = realloc_pool_memory(sd->msg, size);
}
/*
* Send our header
*/
sd->fsend("%ld %ld 0", bsctx.jcr->JobFiles, STREAM_SIGNED_DIGEST);
Dmsg1(300, "filed>stored:header %s", sd->msg);
/*
* Encode signature data
*/
if (!crypto_sign_encode(signature, (uint8_t *)sd->msg, &size)) {
Jmsg(bsctx.jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
goto bail_out;
}
sd->msglen = size;
sd->send();
sd->signal(BNET_EOD); /* end of checksum */
retval = true;
bail_out:
if (signature) {
crypto_sign_free(signature);
}
return retval;
}
/*
* BAREOS's implementation of fgets(). The difference is that it handles
* being interrupted by a signal (e.g. a SIGCHLD) and it has a
* different calling sequence which implements input lines of
* up to a million characters.
*/
char *bfgets(POOLMEM *&s, FILE *fd)
{
int ch;
int soft_max;
int i = 0;
s[0] = 0;
soft_max = sizeof_pool_memory(s) - 10;
for ( ;; ) {
do {
errno = 0;
ch = fgetc(fd);
} while (ch == EOF && ferror(fd) && (errno == EINTR || errno == EAGAIN));
if (ch == EOF) {
if (i == 0) {
return NULL;
} else {
return s;
}
}
if (i > soft_max) {
/* Insanity check */
if (soft_max > 1000000) {
return s;
}
s = check_pool_memory_size(s, soft_max+10000);
soft_max = sizeof_pool_memory(s) - 10;
}
s[i++] = ch;
s[i] = 0;
if (ch == '\r') { /* Support for Mac/Windows file format */
ch = fgetc(fd);
if (ch != '\n') { /* Mac (\r only) */
(void)ungetc(ch, fd); /* Push next character back to fd */
}
s[i-1] = '\n';
break;
}
if (ch == '\n') {
break;
}
}
return s;
}
/*
* Add a character to the current string
*/
static void add_str(LEX *lf, int ch)
{
/*
* The default config string is sized to 256 bytes.
* If we need longer config strings its increased with 256 bytes each time.
*/
if ((lf->str_len + 3) >= lf->str_max_len) {
lf->str = check_pool_memory_size(lf->str, lf->str_max_len + 256);
lf->str_max_len = sizeof_pool_memory(lf->str);
}
lf->str[lf->str_len++] = ch;
lf->str[lf->str_len] = 0;
}
/*
* Despool spooled attributes
*/
bool BSOCK::despool(void update_attr_spool_size(ssize_t size), ssize_t tsize)
{
int32_t pktsiz;
size_t nbytes;
ssize_t last = 0, size = 0;
int count = 0;
JCR *jcr = get_jcr();
if (lseek(m_spool_fd, 0, SEEK_SET) == -1) {
Qmsg(jcr, M_FATAL, 0, _("attr spool I/O error.\n"));
return false;
}
#if defined(HAVE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED)
posix_fadvise(m_spool_fd, 0, 0, POSIX_FADV_WILLNEED);
#endif
while ((nbytes = read(m_spool_fd, (char *)&pktsiz, sizeof(int32_t))) == sizeof(int32_t)) {
size += sizeof(int32_t);
msglen = ntohl(pktsiz);
if (msglen > 0) {
if (msglen > (int32_t)sizeof_pool_memory(msg)) {
msg = realloc_pool_memory(msg, msglen + 1);
}
nbytes = read(m_spool_fd, msg, msglen);
if (nbytes != (size_t)msglen) {
berrno be;
Dmsg2(400, "nbytes=%d msglen=%d\n", nbytes, msglen);
Qmsg1(get_jcr(), M_FATAL, 0, _("read attr spool error. ERR=%s\n"), be.bstrerror());
update_attr_spool_size(tsize - last);
return false;
}
size += nbytes;
if ((++count & 0x3F) == 0) {
update_attr_spool_size(size - last);
last = size;
}
}
send();
if (jcr && job_canceled(jcr)) {
return false;
}
}
update_attr_spool_size(tsize - last);
return true;
}
/*
* Add lex structure for an included config file.
*/
static inline LEX *lex_add(LEX *lf,
const char *filename,
FILE *fd,
BPIPE *bpipe,
LEX_ERROR_HANDLER *scan_error,
LEX_WARNING_HANDLER *scan_warning)
{
LEX *nf;
Dmsg1(100, "open config file: %s\n", filename);
nf = (LEX *)malloc(sizeof(LEX));
if (lf) {
memcpy(nf, lf, sizeof(LEX));
memset(lf, 0, sizeof(LEX));
lf->next = nf; /* if have lf, push it behind new one */
lf->options = nf->options; /* preserve user options */
/*
* preserve err_type to prevent bareos exiting on 'reload'
* if config is invalid. Fixes bug #877
*/
lf->err_type = nf->err_type;
} else {
lf = nf; /* start new packet */
memset(lf, 0, sizeof(LEX));
lex_set_error_handler_error_type(lf, M_ERROR_TERM);
}
if (scan_error) {
lf->scan_error = scan_error;
} else {
lex_set_default_error_handler(lf);
}
if (scan_warning) {
lf->scan_warning = scan_warning;
} else {
lex_set_default_warning_handler(lf);
}
lf->fd = fd;
lf->bpipe = bpipe;
lf->fname = bstrdup(filename);
lf->line = get_memory(1024);
lf->str = get_memory(256);
lf->str_max_len = sizeof_pool_memory(lf->str);
lf->state = lex_none;
lf->ch = L_EOL;
return lf;
}
/*
* Get next input command from terminal.
*
* Returns: 1 if got input
* -1 if EOF or error
*/
int get_cmd(FILE *input, const char *prompt, BSOCK *sock, int sec)
{
int len;
if (!stop) {
if (output == stdout || teeout) {
sendit(prompt);
}
}
again:
len = sizeof_pool_memory(sock->msg) - 1;
if (stop) {
sleep(1);
goto again;
}
#ifdef HAVE_CONIO
if (bisatty(fileno(input))) {
input_line(sock->msg, len);
goto ok_out;
}
#endif
if (input == stdin) {
if (win32_cgets(sock->msg, len) == NULL) {
return -1;
}
} else {
if (fgets(sock->msg, len, input) == NULL) {
return -1;
}
}
if (usrbrk()) {
clrbrk();
}
strip_trailing_junk(sock->msg);
sock->msglen = strlen(sock->msg);
return 1;
}
/*
* Get next input command from terminal.
*
* Returns: 1 if got input
* 0 if timeout
* -1 if EOF or error
*/
int get_cmd(FILE *input, const char *prompt, BSOCK *sock, int sec)
{
int len;
if (!stop) {
if (output == stdout || teeout) {
sendit(prompt);
}
}
again:
switch (wait_for_readable_fd(fileno(input), sec, true)) {
case 0:
return 0; /* timeout */
case -1:
return -1; /* error */
default:
len = sizeof_pool_memory(sock->msg) - 1;
if (stop) {
sleep(1);
goto again;
}
#ifdef HAVE_CONIO
if (bisatty(fileno(input))) {
input_line(sock->msg, len);
break;
}
#endif
if (fgets(sock->msg, len, input) == NULL) {
return -1;
}
break;
}
if (usrbrk()) {
clrbrk();
}
strip_trailing_junk(sock->msg);
sock->msglen = strlen(sock->msg);
return 1;
}
/*
* Store a directory name at specified address. Note, we do
* shell expansion except if the string begins with a vertical
* bar (i.e. it will likely be passed to the shell later).
*/
static void store_dir(LEX *lc, RES_ITEM *item, int index, int pass)
{
URES *res_all = (URES *)my_config->m_res_all;
lex_get_token(lc, T_STRING);
if (pass == 1) {
/*
* If a default was set free it first.
*/
if (*(item->value)) {
free(*(item->value));
}
if (lc->str[0] != '|') {
do_shell_expansion(lc->str, sizeof_pool_memory(lc->str));
}
*(item->value) = bstrdup(lc->str);
}
scan_to_eol(lc);
set_bit(index, res_all->hdr.item_present);
}
/*
* Run an external program. Optionally wait a specified number
* of seconds. Program killed if wait exceeded. Optionally
* return the output from the program (normally a single line).
*
* If the watchdog kills the program, fgets returns, and ferror is set
* to 1 (=>SUCCESS), so we check if the watchdog killed the program.
*
* Contrary to my normal calling conventions, this program
*
* Returns: 0 on success
* non-zero on error == berrno status
*/
int run_program(char *prog, int wait, POOLMEM *&results)
{
BPIPE *bpipe;
int stat1, stat2;
char *mode;
mode = (char *)"r";
bpipe = open_bpipe(prog, wait, mode);
if (!bpipe) {
return ENOENT;
}
results[0] = 0;
int len = sizeof_pool_memory(results) - 1;
fgets(results, len, bpipe->rfd);
results[len] = 0;
if (feof(bpipe->rfd)) {
stat1 = 0;
} else {
stat1 = ferror(bpipe->rfd);
}
if (stat1 < 0) {
berrno be;
Dmsg2(150, "Run program fgets stat=%d ERR=%s\n", stat1, be.bstrerror(errno));
} else if (stat1 != 0) {
Dmsg1(150, "Run program fgets stat=%d\n", stat1);
if (bpipe->timer_id) {
Dmsg1(150, "Run program fgets killed=%d\n", bpipe->timer_id->killed);
/* NB: I'm not sure it is really useful for run_program. Without the
* following lines run_program would not detect if the program was killed
* by the watchdog. */
if (bpipe->timer_id->killed) {
stat1 = ETIME;
pm_strcpy(results, _("Program killed by Bacula (timeout)\n"));
}
}
}
stat2 = close_bpipe(bpipe);
stat1 = stat2 != 0 ? stat2 : stat1;
Dmsg1(150, "Run program returning %d\n", stat1);
return stat1;
}
/*
* Terminate any digest and send it to Storage daemon
*/
static inline bool terminate_digest(b_save_ctx &bsctx)
{
uint32_t size;
bool retval = false;
BSOCK *sd = bsctx.jcr->store_bsock;
sd->fsend("%ld %d 0", bsctx.jcr->JobFiles, bsctx.digest_stream);
Dmsg1(300, "filed>stored:header %s", sd->msg);
size = CRYPTO_DIGEST_MAX_SIZE;
/*
* Grow the bsock buffer to fit our message if necessary
*/
if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
sd->msg = realloc_pool_memory(sd->msg, size);
}
if (!crypto_digest_finalize(bsctx.digest, (uint8_t *)sd->msg, &size)) {
Jmsg(bsctx.jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
goto bail_out;
}
/*
* Keep the checksum if this file is a hardlink
*/
if (bsctx.ff_pkt->linked) {
ff_pkt_set_link_digest(bsctx.ff_pkt, bsctx.digest_stream, sd->msg, size);
}
sd->msglen = size;
sd->send();
sd->signal(BNET_EOD); /* end of checksum */
retval = true;
bail_out:
return retval;
}
/*
* Store a directory name at specified address in an alist.
* Note, we do shell expansion except if the string begins
* with a vertical bar (i.e. it will likely be passed to the
* shell later).
*/
static void store_alist_dir(LEX *lc, RES_ITEM *item, int index, int pass)
{
alist *list;
URES *res_all = (URES *)my_config->m_res_all;
if (pass == 2) {
if (*(item->value) == NULL) {
list = New(alist(10, owned_by_alist));
} else {
list = (alist *)(*(item->value));
}
lex_get_token(lc, T_STRING); /* scan next item */
Dmsg4(900, "Append %s to alist %p size=%d %s\n",
lc->str, list, list->size(), item->name);
if (lc->str[0] != '|') {
do_shell_expansion(lc->str, sizeof_pool_memory(lc->str));
}
list->append(bstrdup(lc->str));
*(item->value) = (char *)list;
}
scan_to_eol(lc);
set_bit(index, res_all->hdr.item_present);
}
/*
* Get the next file to backup.
*/
static bRC get_next_file_to_backup(bpContext *ctx)
{
int status;
struct save_pkt sp;
struct dirent *entry;
plugin_ctx *p_ctx = (plugin_ctx *)ctx->pContext;
/*
* See if we just saved the directory then we are done processing this directory.
*/
switch (p_ctx->type) {
case FT_DIREND:
/*
* See if there is anything on the dir stack to pop off and continue reading that directory.
*/
if (!p_ctx->dir_stack->empty()) {
const char *cwd;
struct dir_stack_entry *new_entry;
/*
* Change the GLFS cwd back one dir.
*/
status = ceph_chdir(p_ctx->cmount, "..");
if (status < 0) {
berrno be;
Jmsg(ctx, M_ERROR, "ceph_chdir(%s) failed: %s\n", "..", be.bstrerror(-status));
return bRC_Error;
}
/*
* Save where we are in the tree.
*/
cwd = ceph_getcwd(p_ctx->cmount);
pm_strcpy(p_ctx->cwd, cwd);
/*
* Pop the previous directory handle and continue processing that.
*/
new_entry = (struct dir_stack_entry *)p_ctx->dir_stack->pop();
memcpy(&p_ctx->statp, &new_entry->statp, sizeof(p_ctx->statp));
p_ctx->cdir = new_entry->cdir;
free(new_entry);
} else {
return bRC_OK;
}
break;
default:
break;
}
if (!p_ctx->cdir) {
return bRC_Error;
}
/*
* Loop until we know what file is next or when we are done.
*/
while (1) {
int stmask = 0;
memset(&p_ctx->statp, 0, sizeof(p_ctx->statp));
memset(&p_ctx->de, 0, sizeof(p_ctx->de));
status = ceph_readdirplus_r(p_ctx->cmount, p_ctx->cdir, &p_ctx->de, &p_ctx->statp, &stmask);
/*
* No more entries in this directory ?
*/
if (status == 0) {
status = ceph_stat(p_ctx->cmount, p_ctx->cwd, &p_ctx->statp);
if (status < 0) {
berrno be;
Jmsg(ctx, M_ERROR, "ceph_stat(%s) failed: %s\n", p_ctx->cwd, be.bstrerror(-status));
return bRC_Error;
}
status = ceph_closedir(p_ctx->cmount, p_ctx->cdir);
if (status < 0) {
berrno be;
Jmsg(ctx, M_ERROR, "ceph_closedir(%s) failed: %s\n", p_ctx->cwd, be.bstrerror(-status));
return bRC_Error;
}
p_ctx->cdir = NULL;
p_ctx->type = FT_DIREND;
pm_strcpy(p_ctx->next_filename, p_ctx->cwd);
Dmsg(ctx, dbglvl, "cephfs-fd: next file to backup %s\n", p_ctx->next_filename);
return bRC_More;
}
entry = &p_ctx->de;
/*
* Skip `.', `..', and excluded file names.
*/
if (entry->d_name[0] == '\0' ||
(entry->d_name[0] == '.' && (entry->d_name[1] == '\0' ||
(entry->d_name[1] == '.' && entry->d_name[2] == '\0')))) {
continue;
}
Mmsg(p_ctx->next_filename, "%s/%s", p_ctx->cwd, entry->d_name);
/*
* Determine the FileType.
*/
switch (p_ctx->statp.st_mode & S_IFMT) {
case S_IFREG:
p_ctx->type = FT_REG;
break;
case S_IFLNK:
p_ctx->type = FT_LNK;
status = ceph_readlink(p_ctx->cmount, p_ctx->next_filename,
p_ctx->link_target, sizeof_pool_memory(p_ctx->link_target));
if (status < 0) {
berrno be;
Jmsg(ctx, M_ERROR, "ceph_readlink(%s) failed: %s\n", p_ctx->next_filename, be.bstrerror(-status));
p_ctx->type = FT_NOFOLLOW;
}
p_ctx->link_target[status] = '\0';
break;
case S_IFDIR:
p_ctx->type = FT_DIRBEGIN;
break;
case S_IFCHR:
case S_IFBLK:
case S_IFIFO:
#ifdef S_IFSOCK
case S_IFSOCK:
#endif
p_ctx->type = FT_SPEC;
break;
default:
Jmsg(ctx, M_FATAL, "Unknown filetype encountered %ld for %s\n",
p_ctx->statp.st_mode & S_IFMT, p_ctx->next_filename);
return bRC_Error;
}
/*
* See if we accept this file under the currently loaded fileset.
*/
memset(&sp, 0, sizeof(sp));
sp.pkt_size = sizeof(sp);
sp.pkt_end = sizeof(sp);
sp.fname = p_ctx->next_filename;
sp.type = p_ctx->type;
memcpy(&sp.statp, &p_ctx->statp, sizeof(sp.statp));
if (bfuncs->AcceptFile(ctx, &sp) == bRC_Skip) {
Dmsg(ctx, dbglvl, "cephfs-fd: file %s skipped due to current fileset settings\n", p_ctx->next_filename);
continue;
}
/*
* If we made it here we have the next file to backup.
*/
break;
}
Dmsg(ctx, dbglvl, "cephfs-fd: next file to backup %s\n", p_ctx->next_filename);
return bRC_More;
}
/*
* Verify attributes of the requested files on the Volume
*
*/
void do_verify_volume(JCR *jcr)
{
BSOCK *sd, *dir;
POOLMEM *fname; /* original file name */
POOLMEM *lname; /* link name */
int32_t stream;
uint32_t size;
uint32_t VolSessionId, VolSessionTime, file_index;
uint32_t record_file_index;
char digest[BASE64_SIZE(CRYPTO_DIGEST_MAX_SIZE)];
int type, stat;
sd = jcr->store_bsock;
if (!sd) {
Jmsg(jcr, M_FATAL, 0, _("Storage command not issued before Verify.\n"));
jcr->setJobStatus(JS_FatalError);
return;
}
dir = jcr->dir_bsock;
jcr->setJobStatus(JS_Running);
LockRes();
CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
UnlockRes();
uint32_t buf_size;
if (client) {
buf_size = client->max_network_buffer_size;
} else {
buf_size = 0; /* use default */
}
if (!sd->set_buffer_size(buf_size, BNET_SETBUF_WRITE)) {
jcr->setJobStatus(JS_FatalError);
return;
}
jcr->buf_size = sd->msglen;
fname = get_pool_memory(PM_FNAME);
lname = get_pool_memory(PM_FNAME);
/*
* Get a record from the Storage daemon
*/
while (bget_msg(sd) >= 0 && !job_canceled(jcr)) {
/*
* First we expect a Stream Record Header
*/
if (sscanf(sd->msg, rec_header, &VolSessionId, &VolSessionTime, &file_index,
&stream, &size) != 5) {
Jmsg1(jcr, M_FATAL, 0, _("Record header scan error: %s\n"), sd->msg);
goto bail_out;
}
Dmsg3(30, "Got hdr: FilInx=%d Stream=%d size=%d.\n", file_index, stream, size);
/*
* Now we expect the Stream Data
*/
if (bget_msg(sd) < 0) {
Jmsg1(jcr, M_FATAL, 0, _("Data record error. ERR=%s\n"), sd->bstrerror());
goto bail_out;
}
if (size != ((uint32_t)sd->msglen)) {
Jmsg2(jcr, M_FATAL, 0, _("Actual data size %d not same as header %d\n"), sd->msglen, size);
goto bail_out;
}
Dmsg2(30, "Got stream data %s, len=%d\n", stream_to_ascii(stream), sd->msglen);
/* File Attributes stream */
switch (stream) {
case STREAM_UNIX_ATTRIBUTES:
case STREAM_UNIX_ATTRIBUTES_EX:
char *ap, *lp, *fp;
Dmsg0(400, "Stream=Unix Attributes.\n");
if ((int)sizeof_pool_memory(fname) < sd->msglen) {
fname = realloc_pool_memory(fname, sd->msglen + 1);
}
if ((int)sizeof_pool_memory(lname) < sd->msglen) {
lname = realloc_pool_memory(lname, sd->msglen + 1);
}
*fname = 0;
*lname = 0;
/*
* An Attributes record consists of:
* File_index
* Type (FT_types)
* Filename
* Attributes
* Link name (if file linked i.e. FT_LNK)
* Extended Attributes (if Win32)
*/
if (sscanf(sd->msg, "%d %d", &record_file_index, &type) != 2) {
Jmsg(jcr, M_FATAL, 0, _("Error scanning record header: %s\n"), sd->msg);
Dmsg0(0, "\nError scanning header\n");
goto bail_out;
}
Dmsg2(30, "Got Attr: FilInx=%d type=%d\n", record_file_index, type);
ap = sd->msg;
while (*ap++ != ' ') /* skip record file index */
;
while (*ap++ != ' ') /* skip type */
;
/* Save filename and position to attributes */
fp = fname;
while (*ap != 0) {
*fp++ = *ap++; /* copy filename to fname */
}
*fp = *ap++; /* terminate filename & point to attribs */
Dmsg2(100, "File=%s Attr=%s\n", fname, ap);
/* Skip to Link name */
if (type == FT_LNK || type == FT_LNKSAVED) {
lp = ap;
while (*lp++ != 0) {
;
}
pm_strcat(lname, lp); /* "save" link name */
} else {
*lname = 0;
}
jcr->lock();
jcr->JobFiles++;
jcr->num_files_examined++;
pm_strcpy(jcr->last_fname, fname); /* last file examined */
jcr->unlock();
/*
* Send file attributes to Director
* File_index
* Stream
* Verify Options
* Filename (full path)
* Encoded attributes
* Link name (if type==FT_LNK)
* For a directory, link is the same as fname, but with trailing
* slash. For a linked file, link is the link.
*/
/* Send file attributes to Director */
Dmsg2(200, "send ATTR inx=%d fname=%s\n", jcr->JobFiles, fname);
if (type == FT_LNK || type == FT_LNKSAVED) {
stat = dir->fsend("%d %d %s %s%c%s%c%s%c", jcr->JobFiles,
STREAM_UNIX_ATTRIBUTES, "pinsug5", fname,
0, ap, 0, lname, 0);
/* for a deleted record, we set fileindex=0 */
} else if (type == FT_DELETED) {
stat = dir->fsend("%d %d %s %s%c%s%c%c", 0,
STREAM_UNIX_ATTRIBUTES, "pinsug5", fname,
0, ap, 0, 0);
} else {
stat = dir->fsend("%d %d %s %s%c%s%c%c", jcr->JobFiles,
STREAM_UNIX_ATTRIBUTES, "pinsug5", fname,
0, ap, 0, 0);
}
Dmsg2(200, "bfiled>bdird: attribs len=%d: msg=%s\n", dir->msglen, dir->msg);
if (!stat) {
Jmsg(jcr, M_FATAL, 0, _("Network error in send to Director: ERR=%s\n"), dir->bstrerror());
goto bail_out;
}
break;
case STREAM_MD5_DIGEST:
bin_to_base64(digest, sizeof(digest), (char *)sd->msg, CRYPTO_DIGEST_MD5_SIZE, true);
Dmsg2(400, "send inx=%d MD5=%s\n", jcr->JobFiles, digest);
dir->fsend("%d %d %s *MD5-%d*", jcr->JobFiles, STREAM_MD5_DIGEST, digest,
jcr->JobFiles);
Dmsg2(20, "bfiled>bdird: MD5 len=%d: msg=%s\n", dir->msglen, dir->msg);
break;
case STREAM_SHA1_DIGEST:
bin_to_base64(digest, sizeof(digest), (char *)sd->msg, CRYPTO_DIGEST_SHA1_SIZE, true);
Dmsg2(400, "send inx=%d SHA1=%s\n", jcr->JobFiles, digest);
dir->fsend("%d %d %s *SHA1-%d*", jcr->JobFiles, STREAM_SHA1_DIGEST,
digest, jcr->JobFiles);
Dmsg2(20, "bfiled>bdird: SHA1 len=%d: msg=%s\n", dir->msglen, dir->msg);
break;
case STREAM_SHA256_DIGEST:
bin_to_base64(digest, sizeof(digest), (char *)sd->msg, CRYPTO_DIGEST_SHA256_SIZE, true);
Dmsg2(400, "send inx=%d SHA256=%s\n", jcr->JobFiles, digest);
dir->fsend("%d %d %s *SHA256-%d*", jcr->JobFiles, STREAM_SHA256_DIGEST,
digest, jcr->JobFiles);
Dmsg2(20, "bfiled>bdird: SHA256 len=%d: msg=%s\n", dir->msglen, dir->msg);
break;
case STREAM_SHA512_DIGEST:
bin_to_base64(digest, sizeof(digest), (char *)sd->msg, CRYPTO_DIGEST_SHA512_SIZE, true);
Dmsg2(400, "send inx=%d SHA512=%s\n", jcr->JobFiles, digest);
dir->fsend("%d %d %s *SHA512-%d*", jcr->JobFiles, STREAM_SHA512_DIGEST,
digest, jcr->JobFiles);
Dmsg2(20, "bfiled>bdird: SHA512 len=%d: msg=%s\n", dir->msglen, dir->msg);
break;
/*
* Restore stream object is counted, but not restored here
*/
case STREAM_RESTORE_OBJECT:
jcr->lock();
jcr->JobFiles++;
jcr->num_files_examined++;
jcr->unlock();
break;
/* Ignore everything else */
default:
break;
} /* end switch */
} /* end while bnet_get */
jcr->setJobStatus(JS_Terminated);
goto ok_out;
bail_out:
jcr->setJobStatus(JS_ErrorTerminated);
ok_out:
if (jcr->compress_buf) {
free(jcr->compress_buf);
jcr->compress_buf = NULL;
}
free_pool_memory(fname);
free_pool_memory(lname);
Dmsg2(050, "End Verify-Vol. Files=%d Bytes=%" lld "\n", jcr->JobFiles,
jcr->JobBytes);
}
/*
* NB! This routine locks the device, but if committing will
* not unlock it. If not committing, it will be unlocked.
*/
static bool despool_data(DCR *dcr, bool commit)
{
DEVICE *rdev;
DCR *rdcr;
bool ok = true;
DEV_BLOCK *block;
JCR *jcr = dcr->jcr;
int status;
char ec1[50];
BSOCK *dir = jcr->dir_bsock;
Dmsg0(100, "Despooling data\n");
if (jcr->dcr->job_spool_size == 0) {
Jmsg(jcr, M_WARNING, 0, _("Despooling zero bytes. Your disk is probably FULL!\n"));
}
/*
* Commit means that the job is done, so we commit, otherwise, we
* are despooling because of user spool size max or some error
* (e.g. filesystem full).
*/
if (commit) {
Jmsg(jcr, M_INFO, 0, _("Committing spooled data to Volume \"%s\". Despooling %s bytes ...\n"),
jcr->dcr->VolumeName,
edit_uint64_with_commas(jcr->dcr->job_spool_size, ec1));
jcr->setJobStatus(JS_DataCommitting);
} else {
Jmsg(jcr, M_INFO, 0, _("Writing spooled data to Volume. Despooling %s bytes ...\n"),
edit_uint64_with_commas(jcr->dcr->job_spool_size, ec1));
jcr->setJobStatus(JS_DataDespooling);
}
jcr->sendJobStatus(JS_DataDespooling);
dcr->despool_wait = true;
dcr->spooling = false;
/*
* We work with device blocked, but not locked so that other threads
* e.g. reservations can lock the device structure.
*/
dcr->dblock(BST_DESPOOLING);
dcr->despool_wait = false;
dcr->despooling = true;
/*
* This is really quite kludgy and should be fixed some time.
* We create a dev structure to read from the spool file
* in rdev and rdcr.
*/
rdev = (DEVICE *)malloc(sizeof(DEVICE));
memset(rdev, 0, sizeof(DEVICE));
rdev->dev_name = get_memory(strlen(spool_name)+1);
bstrncpy(rdev->dev_name, spool_name, sizeof_pool_memory(rdev->dev_name));
rdev->errmsg = get_pool_memory(PM_EMSG);
*rdev->errmsg = 0;
rdev->max_block_size = dcr->dev->max_block_size;
rdev->min_block_size = dcr->dev->min_block_size;
rdev->device = dcr->dev->device;
rdcr = dcr->get_new_spooling_dcr();
setup_new_dcr_device(jcr, rdcr, rdev, NULL);
rdcr->spool_fd = dcr->spool_fd;
block = dcr->block; /* save block */
dcr->block = rdcr->block; /* make read and write block the same */
Dmsg1(800, "read/write block size = %d\n", block->buf_len);
lseek(rdcr->spool_fd, 0, SEEK_SET); /* rewind */
#if defined(HAVE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED)
posix_fadvise(rdcr->spool_fd, 0, 0, POSIX_FADV_WILLNEED);
#endif
/* Add run time, to get current wait time */
int32_t despool_start = time(NULL) - jcr->run_time;
set_new_file_parameters(dcr);
while (ok) {
if (job_canceled(jcr)) {
ok = false;
break;
}
status = read_block_from_spool_file(rdcr);
if (status == RB_EOT) {
break;
} else if (status == RB_ERROR) {
ok = false;
break;
}
ok = dcr->write_block_to_device();
if (!ok) {
Jmsg2(jcr, M_FATAL, 0, _("Fatal append error on device %s: ERR=%s\n"),
dcr->dev->print_name(), dcr->dev->bstrerror());
Dmsg2(000, "Fatal append error on device %s: ERR=%s\n",
dcr->dev->print_name(), dcr->dev->bstrerror());
/* Force in case Incomplete set */
jcr->forceJobStatus(JS_FatalError);
}
Dmsg3(800, "Write block ok=%d FI=%d LI=%d\n", ok, block->FirstIndex, block->LastIndex);
}
/*
* If this Job is incomplete, we need to backup the FileIndex
* to the last correctly saved file so that the JobMedia
* LastIndex is correct.
*/
if (jcr->is_JobStatus(JS_Incomplete)) {
dcr->VolLastIndex = dir->get_FileIndex();
Dmsg1(100, "======= Set FI=%ld\n", dir->get_FileIndex());
}
if (!dcr->dir_create_jobmedia_record(false)) {
Jmsg2(jcr, M_FATAL, 0, _("Could not create JobMedia record for Volume=\"%s\" Job=%s\n"),
dcr->getVolCatName(), jcr->Job);
jcr->forceJobStatus(JS_FatalError); /* override any Incomplete */
}
/* Set new file/block parameters for current dcr */
set_new_file_parameters(dcr);
/*
* Subtracting run_time give us elapsed time - wait_time since
* we started despooling. Note, don't use time_t as it is 32 or 64
* bits depending on the OS and doesn't edit with %d
*/
int32_t despool_elapsed = time(NULL) - despool_start - jcr->run_time;
if (despool_elapsed <= 0) {
despool_elapsed = 1;
}
Jmsg(jcr, M_INFO, 0, _("Despooling elapsed time = %02d:%02d:%02d, Transfer rate = %s Bytes/second\n"),
despool_elapsed / 3600, despool_elapsed % 3600 / 60, despool_elapsed % 60,
edit_uint64_with_suffix(jcr->dcr->job_spool_size / despool_elapsed, ec1));
dcr->block = block; /* reset block */
lseek(rdcr->spool_fd, 0, SEEK_SET); /* rewind */
if (ftruncate(rdcr->spool_fd, 0) != 0) {
berrno be;
Jmsg(jcr, M_ERROR, 0, _("Ftruncate spool file failed: ERR=%s\n"), be.bstrerror());
/* Note, try continuing despite ftruncate problem */
}
P(mutex);
if (spool_stats.data_size < dcr->job_spool_size) {
spool_stats.data_size = 0;
} else {
spool_stats.data_size -= dcr->job_spool_size;
}
V(mutex);
P(dcr->dev->spool_mutex);
dcr->dev->spool_size -= dcr->job_spool_size;
dcr->job_spool_size = 0; /* zap size in input dcr */
V(dcr->dev->spool_mutex);
free_memory(rdev->dev_name);
free_pool_memory(rdev->errmsg);
/* Be careful to NULL the jcr and free rdev after free_dcr() */
rdcr->jcr = NULL;
rdcr->set_dev(NULL);
free_dcr(rdcr);
free(rdev);
dcr->spooling = true; /* turn on spooling again */
dcr->despooling = false;
/*
* Note, if committing we leave the device blocked. It will be removed in
* release_device();
*/
if (!commit) {
dcr->dev->dunblock();
}
jcr->sendJobStatus(JS_Running);
return ok;
}
static void read_and_process_input(FILE *input, BSOCK *UA_sock)
{
const char *prompt = "*";
bool at_prompt = false;
int tty_input = isatty(fileno(input));
int status;
btimer_t *tid = NULL;
while (1) {
if (at_prompt) { /* don't prompt multiple times */
prompt = "";
} else {
prompt = "*";
at_prompt = true;
}
if (tty_input) {
status = get_cmd(input, prompt, UA_sock, 30);
if (usrbrk() == 1) {
clrbrk();
}
if (usrbrk()) {
break;
}
} else {
/*
* Reading input from a file
*/
int len = sizeof_pool_memory(UA_sock->msg) - 1;
if (usrbrk()) {
break;
}
if (fgets(UA_sock->msg, len, input) == NULL) {
status = -1;
} else {
sendit(UA_sock->msg); /* echo to terminal */
strip_trailing_junk(UA_sock->msg);
UA_sock->msglen = strlen(UA_sock->msg);
status = 1;
}
}
if (status < 0) {
break; /* error or interrupt */
} else if (status == 0) { /* timeout */
if (bstrcmp(prompt, "*")) {
tid = start_bsock_timer(UA_sock, timeout);
UA_sock->fsend(".messages");
stop_bsock_timer(tid);
} else {
continue;
}
} else {
at_prompt = false;
/*
* @ => internal command for us
*/
if (UA_sock->msg[0] == '@') {
parse_args(UA_sock->msg, &args, &argc, argk, argv, MAX_CMD_ARGS);
if (!do_a_command(input, UA_sock)) {
break;
}
continue;
}
tid = start_bsock_timer(UA_sock, timeout);
if (!UA_sock->send()) { /* send command */
stop_bsock_timer(tid);
break; /* error */
}
stop_bsock_timer(tid);
}
if (bstrcmp(UA_sock->msg, ".quit") || bstrcmp(UA_sock->msg, ".exit")) {
break;
}
tid = start_bsock_timer(UA_sock, timeout);
while ((status = UA_sock->recv()) >= 0 ||
((status == BNET_SIGNAL) && (
(UA_sock->msglen != BNET_EOD) &&
(UA_sock->msglen != BNET_MAIN_PROMPT) &&
(UA_sock->msglen != BNET_SUB_PROMPT)))) {
if (status == BNET_SIGNAL) {
if (UA_sock->msglen == BNET_START_RTREE) {
file_selection = true;
} else if (UA_sock->msglen == BNET_END_RTREE) {
file_selection = false;
}
continue;
}
if (at_prompt) {
if (!stop) {
sendit("\n");
}
at_prompt = false;
}
/*
* Suppress output if running in background or user hit ctl-c
*/
if (!stop && !usrbrk()) {
if (UA_sock->msg) {
sendit(UA_sock->msg);
}
}
}
stop_bsock_timer(tid);
if (usrbrk() > 1) {
break;
} else {
clrbrk();
}
if (!stop) {
fflush(stdout);
}
if (is_bnet_stop(UA_sock)) {
break; /* error or term */
} else if (status == BNET_SIGNAL) {
if (UA_sock->msglen == BNET_SUB_PROMPT) {
at_prompt = true;
}
Dmsg1(100, "Got poll %s\n", bnet_sig_to_ascii(UA_sock));
}
}
}
static bRC getXattr(bpContext *ctx, xattr_pkt *xp)
{
char *bp;
bool skip_xattr;
int status, current_size;
int32_t xattr_value_length;
POOL_MEM xattr_value(PM_MESSAGE);
plugin_ctx *p_ctx = (plugin_ctx *)ctx->pContext;
if (!p_ctx) {
return bRC_Error;
}
/*
* See if we need to retrieve the xattr list.
*/
if (!p_ctx->processing_xattr) {
while (1) {
current_size = sizeof_pool_memory(p_ctx->xattr_list);
status = ceph_llistxattr(p_ctx->cmount, xp->fname, p_ctx->xattr_list, current_size);
if (status < 0) {
berrno be;
switch (status) {
#if defined(ENOTSUP) || defined(EOPNOTSUPP)
#if defined(ENOTSUP)
case ENOTSUP:
#endif
#if defined(EOPNOTSUPP) && EOPNOTSUPP != ENOTSUP
case EOPNOTSUPP:
#endif
return bRC_OK;
#endif
case -ERANGE:
/*
* Not enough room in buffer double its size and retry.
*/
p_ctx->xattr_list = check_pool_memory_size(p_ctx->xattr_list, current_size * 2);
continue;
default:
Jmsg(ctx, M_ERROR, "ceph_llistxattr(%s) failed: %s\n", xp->fname, be.bstrerror(-status));
return bRC_Error;
}
}
/*
* Retrieved the xattr list so break the loop.
*/
break;
}
p_ctx->next_xattr_name = p_ctx->xattr_list;
p_ctx->processing_xattr = true;
}
while (1) {
/*
* On some OSes you also get the acls in the extented attribute list.
* So we check if we are already backing up acls and if we do we
* don't store the extended attribute with the same info.
*/
skip_xattr = false;
if (bit_is_set(FO_ACL, p_ctx->flags)) {
for (int cnt = 0; xattr_acl_skiplist[cnt] != NULL; cnt++) {
if (bstrcmp(p_ctx->next_xattr_name, xattr_acl_skiplist[cnt])) {
skip_xattr = true;
break;
}
}
}
if (!skip_xattr) {
current_size = xattr_value.max_size();
xattr_value_length = ceph_lgetxattr(p_ctx->cmount, xp->fname, p_ctx->next_xattr_name,
xattr_value.c_str(), current_size);
if (xattr_value_length < 0) {
berrno be;
switch (xattr_value_length) {
#if defined(ENOATTR) || defined(ENODATA)
#if defined(ENOATTR)
case ENOATTR:
#endif
#if defined(ENODATA) && ENOATTR != ENODATA
case ENODATA:
#endif
skip_xattr = true;
break;
#endif
#if defined(ENOTSUP) || defined(EOPNOTSUPP)
#if defined(ENOTSUP)
case ENOTSUP:
#endif
#if defined(EOPNOTSUPP) && EOPNOTSUPP != ENOTSUP
case EOPNOTSUPP:
#endif
return bRC_OK;
#endif
case -ERANGE:
/*
* Not enough room in buffer double its size and retry.
*/
xattr_value.check_size(current_size * 2);
continue;
default:
Jmsg(ctx, M_ERROR, "ceph_lgetxattr(%s) failed: %s\n", xp->fname, be.bstrerror(-xattr_value_length));
return bRC_Error;
}
}
/*
* Retrieved the xattr so break the loop.
*/
break;
} else {
/*
* No data to retrieve so break the loop.
*/
break;
}
}
if (!skip_xattr) {
xp->name = bstrdup(p_ctx->next_xattr_name);
xp->name_length = strlen(xp->name) + 1;
xp->value = (char *)malloc(xattr_value_length);
memcpy(xp->value, xattr_value.c_str(), xattr_value_length);
xp->value_length = xattr_value_length;
}
/*
* See if there are more xattr to process.
*/
bp = strchr(p_ctx->next_xattr_name, '\0');
if (++bp != '\0') {
p_ctx->next_xattr_name = bp;
return bRC_More;
} else {
/*
* No more reset processing_xattr flag.
*/
p_ctx->processing_xattr = false;
return bRC_OK;
}
}
/*
* Update File Attributes in the catalog with data read from
* the storage daemon spool file. We receive the filename and
* we try to read it.
*/
bool despool_attributes_from_file(JCR *jcr, const char *file)
{
bool retval = false;
int32_t pktsiz;
size_t nbytes;
ssize_t size = 0;
int32_t msglen; /* message length */
FILE *spool_fd = NULL;
POOLMEM *msg = get_pool_memory(PM_MESSAGE);
Dmsg0(100, "Begin despool_attributes_from_file\n");
if (jcr->is_job_canceled() || !jcr->res.pool->catalog_files || !jcr->db) {
goto bail_out; /* user disabled cataloging */
}
spool_fd = fopen(file, "rb");
if (!spool_fd) {
Dmsg0(100, "cancel despool_attributes_from_file\n");
/* send an error message */
goto bail_out;
}
#if defined(HAVE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED)
posix_fadvise(fileno(spool_fd), 0, 0, POSIX_FADV_WILLNEED);
#endif
while (fread((char *)&pktsiz, 1, sizeof(int32_t), spool_fd) == sizeof(int32_t)) {
size += sizeof(int32_t);
msglen = ntohl(pktsiz);
if (msglen > 0) {
if (msglen > (int32_t) sizeof_pool_memory(msg)) {
msg = realloc_pool_memory(msg, msglen + 1);
}
nbytes = fread(msg, 1, msglen, spool_fd);
if (nbytes != (size_t) msglen) {
berrno be;
Dmsg2(400, "nbytes=%d msglen=%d\n", nbytes, msglen);
Qmsg1(jcr, M_FATAL, 0, _("fread attr spool error. ERR=%s\n"),
be.bstrerror());
goto bail_out;
}
size += nbytes;
}
if (!jcr->is_job_canceled()) {
update_attribute(jcr, msg, msglen);
if (jcr->is_job_canceled()) {
goto bail_out;
}
}
}
if (ferror(spool_fd)) {
berrno be;
Qmsg1(jcr, M_FATAL, 0, _("fread attr spool error. ERR=%s\n"),
be.bstrerror());
goto bail_out;
}
retval = true;
bail_out:
if (spool_fd) {
fclose(spool_fd);
}
if (jcr->is_job_canceled()) {
cancel_storage_daemon_job(jcr);
}
free_pool_memory(msg);
Dmsg1(100, "End despool_attributes_from_file retval=%i\n", retval);
return retval;
}
/*
* Called here by find() for each file included.
* This is a callback. The original is find_files() above.
*
* Send the file and its data to the Storage daemon.
*
* Returns: 1 if OK
* 0 if error
* -1 to ignore file/directory (not used here)
*/
int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
{
bool do_read = false;
int stat, data_stream;
int rtnstat = 0;
DIGEST *digest = NULL;
DIGEST *signing_digest = NULL;
int digest_stream = STREAM_NONE;
SIGNATURE *sig = NULL;
bool has_file_data = false;
// TODO landonf: Allow the user to specify the digest algorithm
#ifdef HAVE_SHA2
crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
#else
crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
#endif
BSOCK *sd = jcr->store_bsock;
if (job_canceled(jcr)) {
return 0;
}
jcr->num_files_examined++; /* bump total file count */
switch (ff_pkt->type) {
case FT_LNKSAVED: /* Hard linked, file already saved */
Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
break;
case FT_REGE:
Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
has_file_data = true;
break;
case FT_REG:
Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
has_file_data = true;
break;
case FT_LNK:
Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
break;
case FT_DIRBEGIN:
jcr->num_files_examined--; /* correct file count */
return 1; /* not used */
case FT_NORECURSE:
Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
ff_pkt->top_fname, ff_pkt->fname);
ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
break;
case FT_NOFSCHG:
/* Suppress message for /dev filesystems */
if (!is_in_fileset(ff_pkt)) {
Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
}
ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
break;
case FT_INVALIDFS:
Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
ff_pkt->top_fname, ff_pkt->fname);
ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
break;
case FT_INVALIDDT:
Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
ff_pkt->fname);
break;
case FT_REPARSE:
case FT_DIREND:
Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
break;
case FT_SPEC:
Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
if (S_ISSOCK(ff_pkt->statp.st_mode)) {
Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname);
return 1;
}
break;
case FT_RAW:
Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
has_file_data = true;
break;
case FT_FIFO:
Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
break;
case FT_NOACCESS: {
berrno be;
Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access \"%s\": ERR=%s\n"), ff_pkt->fname,
be.bstrerror(ff_pkt->ff_errno));
jcr->JobErrors++;
return 1;
}
case FT_NOFOLLOW: {
berrno be;
Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link \"%s\": ERR=%s\n"),
ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
jcr->JobErrors++;
return 1;
}
case FT_NOSTAT: {
berrno be;
Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat \"%s\": ERR=%s\n"), ff_pkt->fname,
be.bstrerror(ff_pkt->ff_errno));
jcr->JobErrors++;
return 1;
}
case FT_DIRNOCHG:
case FT_NOCHG:
Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
return 1;
case FT_ISARCH:
Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
return 1;
case FT_NOOPEN: {
berrno be;
Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory \"%s\": ERR=%s\n"),
ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
jcr->JobErrors++;
return 1;
}
default:
Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
ff_pkt->type, ff_pkt->fname);
jcr->JobErrors++;
return 1;
}
Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
/* Digests and encryption are only useful if there's file data */
if (has_file_data) {
/*
* Setup for digest handling. If this fails, the digest will be set to NULL
* and not used. Note, the digest (file hash) can be any one of the four
* algorithms below.
*
* The signing digest is a single algorithm depending on
* whether or not we have SHA2.
* ****FIXME**** the signing algoritm should really be
* determined a different way!!!!!! What happens if
* sha2 was available during backup but not restore?
*/
if (ff_pkt->flags & FO_MD5) {
digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
digest_stream = STREAM_MD5_DIGEST;
} else if (ff_pkt->flags & FO_SHA1) {
digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
digest_stream = STREAM_SHA1_DIGEST;
} else if (ff_pkt->flags & FO_SHA256) {
digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
digest_stream = STREAM_SHA256_DIGEST;
} else if (ff_pkt->flags & FO_SHA512) {
digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
digest_stream = STREAM_SHA512_DIGEST;
}
/* Did digest initialization fail? */
if (digest_stream != STREAM_NONE && digest == NULL) {
Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
stream_to_ascii(digest_stream));
}
/*
* Set up signature digest handling. If this fails, the signature digest will be set to
* NULL and not used.
*/
// TODO landonf: We should really only calculate the digest once, for both verification and signing.
if (jcr->crypto.pki_sign) {
signing_digest = crypto_digest_new(jcr, signing_algorithm);
/* Full-stop if a failure occurred initializing the signature digest */
if (signing_digest == NULL) {
Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
stream_to_ascii(signing_algorithm));
jcr->JobErrors++;
goto good_rtn;
}
}
/* Enable encryption */
if (jcr->crypto.pki_encrypt) {
ff_pkt->flags |= FO_ENCRYPT;
}
}
/* Initialize the file descriptor we use for data and other streams. */
binit(&ff_pkt->bfd);
if (ff_pkt->flags & FO_PORTABLE) {
set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
}
if (ff_pkt->cmd_plugin) {
if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) {
goto bail_out;
}
send_plugin_name(jcr, sd, true); /* signal start of plugin data */
}
/* Send attributes -- must be done after binit() */
if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
goto bail_out;
}
/* Set up the encryption context and send the session data to the SD */
if (has_file_data && jcr->crypto.pki_encrypt) {
if (!crypto_session_send(jcr, sd)) {
goto bail_out;
}
}
/*
* Open any file with data that we intend to save, then save it.
*
* Note, if is_win32_backup, we must open the Directory so that
* the BackupRead will save its permissions and ownership streams.
*/
if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
#ifdef HAVE_WIN32
do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
#else
do_read = ff_pkt->statp.st_size > 0;
#endif
} else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
ff_pkt->type == FT_REPARSE ||
(!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
do_read = true;
}
if (ff_pkt->cmd_plugin) {
do_read = true;
}
Dmsg1(400, "do_read=%d\n", do_read);
if (do_read) {
btimer_t *tid;
if (ff_pkt->type == FT_FIFO) {
tid = start_thread_timer(jcr, pthread_self(), 60);
} else {
tid = NULL;
}
int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
ff_pkt->bfd.reparse_point = ff_pkt->type == FT_REPARSE;
if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
ff_pkt->ff_errno = errno;
berrno be;
Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open \"%s\": ERR=%s.\n"), ff_pkt->fname,
be.bstrerror());
jcr->JobErrors++;
if (tid) {
stop_thread_timer(tid);
tid = NULL;
}
goto good_rtn;
}
if (tid) {
stop_thread_timer(tid);
tid = NULL;
}
stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
if (ff_pkt->flags & FO_CHKCHANGES) {
has_file_changed(jcr, ff_pkt);
}
bclose(&ff_pkt->bfd);
if (!stat) {
goto bail_out;
}
}
#ifdef HAVE_DARWIN_OS
/* Regular files can have resource forks and Finder Info */
if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
ff_pkt->flags & FO_HFSPLUS)) {
if (ff_pkt->hfsinfo.rsrclength > 0) {
int flags;
int rsrc_stream;
if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
ff_pkt->ff_errno = errno;
berrno be;
Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for \"%s\": ERR=%s.\n"),
ff_pkt->fname, be.bstrerror());
jcr->JobErrors++;
if (is_bopen(&ff_pkt->bfd)) {
bclose(&ff_pkt->bfd);
}
goto good_rtn;
}
flags = ff_pkt->flags;
ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
if (flags & FO_ENCRYPT) {
rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
} else {
rsrc_stream = STREAM_MACOS_FORK_DATA;
}
stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
ff_pkt->flags = flags;
bclose(&ff_pkt->bfd);
if (!stat) {
goto bail_out;
}
}
Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
pm_memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
sd->msglen = 32;
if (digest) {
crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
}
if (signing_digest) {
crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
}
sd->send();
sd->signal(BNET_EOD);
}
#endif
/*
* Save ACLs for anything not being a symlink and not being a plugin.
*/
if (!ff_pkt->cmd_plugin) {
if (ff_pkt->flags & FO_ACL && ff_pkt->type != FT_LNK) {
if (!build_acl_streams(jcr, ff_pkt))
goto bail_out;
}
}
/*
* Save Extended Attributes for all files not being a plugin.
*/
if (!ff_pkt->cmd_plugin) {
if (ff_pkt->flags & FO_XATTR) {
if (!build_xattr_streams(jcr, ff_pkt))
goto bail_out;
}
}
/* Terminate the signing digest and send it to the Storage daemon */
if (signing_digest) {
uint32_t size = 0;
if ((sig = crypto_sign_new(jcr)) == NULL) {
Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
goto bail_out;
}
if (!crypto_sign_add_signer(sig, signing_digest, jcr->crypto.pki_keypair)) {
Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
goto bail_out;
}
/* Get signature size */
if (!crypto_sign_encode(sig, NULL, &size)) {
Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
goto bail_out;
}
/* Grow the bsock buffer to fit our message if necessary */
if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
sd->msg = realloc_pool_memory(sd->msg, size);
}
/* Send our header */
sd->fsend("%ld %ld 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
/* Encode signature data */
if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) {
Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
goto bail_out;
}
sd->msglen = size;
sd->send();
sd->signal(BNET_EOD); /* end of checksum */
}
/* Terminate any digest and send it to Storage daemon */
if (digest) {
uint32_t size;
sd->fsend("%ld %d 0", jcr->JobFiles, digest_stream);
Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
size = CRYPTO_DIGEST_MAX_SIZE;
/* Grow the bsock buffer to fit our message if necessary */
if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
sd->msg = realloc_pool_memory(sd->msg, size);
}
if (!crypto_digest_finalize(digest, (uint8_t *)sd->msg, &size)) {
Jmsg(jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
goto bail_out;
}
sd->msglen = size;
sd->send();
sd->signal(BNET_EOD); /* end of checksum */
}
if (ff_pkt->cmd_plugin) {
send_plugin_name(jcr, sd, false); /* signal end of plugin data */
}
good_rtn:
rtnstat = 1; /* good return */
bail_out:
if (digest) {
crypto_digest_free(digest);
}
if (signing_digest) {
crypto_digest_free(signing_digest);
}
if (sig) {
crypto_sign_free(sig);
}
return rtnstat;
}