void bmsg(UAContext *ua, const char *fmt, va_list arg_ptr)
{
BSOCK *bs = ua->UA_sock;
int maxlen, len;
POOLMEM *msg = NULL;
if (bs) {
msg = bs->msg;
}
if (!msg) {
msg = get_memory(5000);
}
maxlen = sizeof_pool_memory(msg) - 1;
if (maxlen < 4999) {
msg = realloc_pool_memory(msg, 5000);
maxlen = 4999;
}
len = bvsnprintf(msg, maxlen, fmt, arg_ptr);
if (len < 0 || len >= maxlen) {
pm_strcpy(msg, _("Message too long to display.\n"));
len = strlen(msg);
}
if (bs) {
bs->msg = msg;
bs->msglen = len;
bs->send();
} else { /* No UA, send to Job */
Jmsg(ua->jcr, M_INFO, 0, "%s", msg);
free_pool_memory(msg);
}
}
/**
* Update File Attribute data
* We do the following:
* 1. expand the bsock buffer to be large enough
* 2. Write a "header" into the buffer with serialized data
* VolSessionId
* VolSeesionTime
* FileIndex
* Stream
* data length that follows
* start of raw byte data from the Device record.
* Note, this is primarily for Attribute data, but can
* also handle any device record. The Director must know
* the raw byte data format that is defined for each Stream.
* Now Restore Objects pass through here STREAM_RESTORE_OBJECT
*/
bool dir_update_file_attributes(DCR *dcr, DEV_RECORD *rec)
{
JCR *jcr = dcr->jcr;
BSOCK *dir = jcr->dir_bsock;
ser_declare;
#ifdef NO_ATTRIBUTES_TEST
return true;
#endif
dir->msg = check_pool_memory_size(dir->msg, sizeof(FileAttributes) +
MAX_NAME_LENGTH + sizeof(DEV_RECORD) + rec->data_len + 1);
dir->msglen = bsnprintf(dir->msg, sizeof(FileAttributes) +
MAX_NAME_LENGTH + 1, FileAttributes, jcr->Job);
ser_begin(dir->msg + dir->msglen, 0);
ser_uint32(rec->VolSessionId);
ser_uint32(rec->VolSessionTime);
ser_int32(rec->FileIndex);
ser_int32(rec->Stream);
ser_uint32(rec->data_len);
ser_bytes(rec->data, rec->data_len);
dir->msglen = ser_length(dir->msg);
Dmsg1(1800, ">dird %s\n", dir->msg); /* Attributes */
return dir->send();
}
void bmsg(UAContext *ua, const char *fmt, va_list arg_ptr)
{
BSOCK *bs = ua->UA_sock;
int maxlen, len;
POOLMEM *msg = NULL;
va_list ap;
if (bs) {
msg = bs->msg;
}
if (!msg) {
msg = get_pool_memory(PM_EMSG);
}
again:
maxlen = sizeof_pool_memory(msg) - 1;
va_copy(ap, arg_ptr);
len = bvsnprintf(msg, maxlen, fmt, ap);
va_end(ap);
if (len < 0 || len >= maxlen) {
msg = realloc_pool_memory(msg, maxlen + maxlen/2);
goto again;
}
if (bs) {
bs->msg = msg;
bs->msglen = len;
bs->send();
} else { /* No UA, send to Job */
Jmsg(ua->jcr, M_INFO, 0, "%s", msg);
free_pool_memory(msg);
}
}
/**
* Save OSX specific resource forks and finder info.
*/
static inline bool save_rsrc_and_finder(b_save_ctx &bsctx)
{
char flags[FOPTS_BYTES];
int rsrc_stream;
BSOCK *sd = bsctx.jcr->store_bsock;
bool retval = false;
if (bsctx.ff_pkt->hfsinfo.rsrclength > 0) {
if (bopen_rsrc(&bsctx.ff_pkt->bfd, bsctx.ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
bsctx.ff_pkt->ff_errno = errno;
berrno be;
Jmsg(bsctx.jcr, M_NOTSAVED, -1,
_(" Cannot open resource fork for \"%s\": ERR=%s.\n"),
bsctx.ff_pkt->fname, be.bstrerror());
bsctx.jcr->JobErrors++;
if (is_bopen(&bsctx.ff_pkt->bfd)) {
bclose(&bsctx.ff_pkt->bfd);
}
} else {
int status;
memcpy(flags, bsctx.ff_pkt->flags, sizeof(flags));
clear_bit(FO_COMPRESS, bsctx.ff_pkt->flags);
clear_bit(FO_SPARSE, bsctx.ff_pkt->flags);
clear_bit(FO_OFFSETS, bsctx.ff_pkt->flags);
rsrc_stream = bit_is_set(FO_ENCRYPT, flags) ? STREAM_ENCRYPTED_MACOS_FORK_DATA :
STREAM_MACOS_FORK_DATA;
status = send_data(bsctx.jcr, rsrc_stream, bsctx.ff_pkt,
bsctx.digest, bsctx.signing_digest);
memcpy(bsctx.ff_pkt->flags, flags, sizeof(flags));
bclose(&bsctx.ff_pkt->bfd);
if (!status) {
goto bail_out;
}
}
}
Dmsg1(300, "Saving Finder Info for \"%s\"\n", bsctx.ff_pkt->fname);
sd->fsend("%ld %d 0", bsctx.jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
Dmsg1(300, "filed>stored:header %s", sd->msg);
pm_memcpy(sd->msg, bsctx.ff_pkt->hfsinfo.fndrinfo, 32);
sd->msglen = 32;
if (bsctx.digest) {
crypto_digest_update(bsctx.digest, (uint8_t *)sd->msg, sd->msglen);
}
if (bsctx.signing_digest) {
crypto_digest_update(bsctx.signing_digest, (uint8_t *)sd->msg, sd->msglen);
}
sd->send();
sd->signal(BNET_EOD);
retval = true;
bail_out:
return retval;
}
/*
* Terminate the signing digest and send it to the Storage daemon
*/
static inline bool terminate_signing_digest(b_save_ctx &bsctx)
{
uint32_t size = 0;
bool retval = false;
SIGNATURE *signature = NULL;
BSOCK *sd = bsctx.jcr->store_bsock;
if ((signature = crypto_sign_new(bsctx.jcr)) == NULL) {
Jmsg(bsctx.jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
goto bail_out;
}
if (!crypto_sign_add_signer(signature, bsctx.signing_digest, bsctx.jcr->crypto.pki_keypair)) {
Jmsg(bsctx.jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
goto bail_out;
}
/*
* Get signature size
*/
if (!crypto_sign_encode(signature, NULL, &size)) {
Jmsg(bsctx.jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
goto bail_out;
}
/*
* Grow the bsock buffer to fit our message if necessary
*/
if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
sd->msg = realloc_pool_memory(sd->msg, size);
}
/*
* Send our header
*/
sd->fsend("%ld %ld 0", bsctx.jcr->JobFiles, STREAM_SIGNED_DIGEST);
Dmsg1(300, "filed>stored:header %s", sd->msg);
/*
* Encode signature data
*/
if (!crypto_sign_encode(signature, (uint8_t *)sd->msg, &size)) {
Jmsg(bsctx.jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
goto bail_out;
}
sd->msglen = size;
sd->send();
sd->signal(BNET_EOD); /* end of checksum */
retval = true;
bail_out:
if (signature) {
crypto_sign_free(signature);
}
return retval;
}
/*
* Called here for each record from read_records()
*
* Returns: true if OK
* false if error
*/
static bool record_cb(DCR *dcr, DEV_RECORD *rec)
{
JCR *jcr = dcr->jcr;
BSOCK *fd = jcr->file_bsock;
bool ok = true;
POOLMEM *save_msg;
char ec1[50], ec2[50];
if (rec->FileIndex < 0) {
return true;
}
Dmsg5(400, "Send to FD: SessId=%u SessTim=%u FI=%s Strm=%s, len=%d\n",
rec->VolSessionId, rec->VolSessionTime,
FI_to_ascii(ec1, rec->FileIndex),
stream_to_ascii(ec2, rec->Stream, rec->FileIndex),
rec->data_len);
/*
* Send record header to File daemon
*/
if (!fd->fsend(rec_header, rec->VolSessionId, rec->VolSessionTime,
rec->FileIndex, rec->Stream, rec->data_len)) {
Pmsg1(000, _(">filed: Error Hdr=%s"), fd->msg);
Jmsg1(jcr, M_FATAL, 0, _("Error sending to File daemon. ERR=%s\n"),
fd->bstrerror());
return false;
} else {
Dmsg1(400, ">filed: Hdr=%s", fd->msg);
}
/*
* Send data record to File daemon
*/
save_msg = fd->msg; /* save fd message pointer */
fd->msg = rec->data; /* pass data directly to the FD */
fd->msglen = rec->data_len;
Dmsg1(400, ">filed: send %d bytes data.\n", fd->msglen);
if (!fd->send()) {
Pmsg1(000, _("Error sending to FD. ERR=%s\n"), fd->bstrerror());
Jmsg1(jcr, M_FATAL, 0, _("Error sending to File daemon. ERR=%s\n"),
fd->bstrerror());
ok = false;
}
fd->msg = save_msg; /* restore fd message pointer */
return ok;
}
/*
* Terminate any digest and send it to Storage daemon
*/
static inline bool terminate_digest(b_save_ctx &bsctx)
{
uint32_t size;
bool retval = false;
BSOCK *sd = bsctx.jcr->store_bsock;
sd->fsend("%ld %d 0", bsctx.jcr->JobFiles, bsctx.digest_stream);
Dmsg1(300, "filed>stored:header %s", sd->msg);
size = CRYPTO_DIGEST_MAX_SIZE;
/*
* Grow the bsock buffer to fit our message if necessary
*/
if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
sd->msg = realloc_pool_memory(sd->msg, size);
}
if (!crypto_digest_finalize(bsctx.digest, (uint8_t *)sd->msg, &size)) {
Jmsg(bsctx.jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
goto bail_out;
}
/*
* Keep the checksum if this file is a hardlink
*/
if (bsctx.ff_pkt->linked) {
ff_pkt_set_link_digest(bsctx.ff_pkt, bsctx.digest_stream, sd->msg, size);
}
sd->msglen = size;
sd->send();
sd->signal(BNET_EOD); /* end of checksum */
retval = true;
bail_out:
return retval;
}
/*
* Handle the data just read and send it to the SD after doing any postprocessing needed.
*/
static inline bool send_data_to_sd(b_ctx *bctx)
{
BSOCK *sd = bctx->jcr->store_bsock;
bool need_more_data;
/*
* Check for sparse blocks
*/
if (bit_is_set(FO_SPARSE, bctx->ff_pkt->flags)) {
bool allZeros;
ser_declare;
allZeros = false;
if ((sd->msglen == bctx->rsize &&
(bctx->fileAddr + sd->msglen < (uint64_t)bctx->ff_pkt->statp.st_size)) ||
((bctx->ff_pkt->type == FT_RAW ||
bctx->ff_pkt->type == FT_FIFO) &&
((uint64_t)bctx->ff_pkt->statp.st_size == 0))) {
allZeros = is_buf_zero(bctx->rbuf, bctx->rsize);
}
if (!allZeros) {
/*
* Put file address as first data in buffer
*/
ser_begin(bctx->wbuf, OFFSET_FADDR_SIZE);
ser_uint64(bctx->fileAddr); /* store fileAddr in begin of buffer */
}
bctx->fileAddr += sd->msglen; /* update file address */
/*
* Skip block of all zeros
*/
if (allZeros) {
return true;
}
} else if (bit_is_set(FO_OFFSETS, bctx->ff_pkt->flags)) {
ser_declare;
ser_begin(bctx->wbuf, OFFSET_FADDR_SIZE);
ser_uint64(bctx->ff_pkt->bfd.offset); /* store offset in begin of buffer */
}
bctx->jcr->ReadBytes += sd->msglen; /* count bytes read */
/*
* Uncompressed cipher input length
*/
bctx->cipher_input_len = sd->msglen;
/*
* Update checksum if requested
*/
if (bctx->digest) {
crypto_digest_update(bctx->digest, (uint8_t *)bctx->rbuf, sd->msglen);
}
/*
* Update signing digest if requested
*/
if (bctx->signing_digest) {
crypto_digest_update(bctx->signing_digest, (uint8_t *)bctx->rbuf, sd->msglen);
}
/*
* Compress the data.
*/
if (bit_is_set(FO_COMPRESS, bctx->ff_pkt->flags)) {
if (!compress_data(bctx->jcr, bctx->ff_pkt->Compress_algo, bctx->rbuf,
bctx->jcr->store_bsock->msglen, bctx->cbuf,
bctx->max_compress_len, &bctx->compress_len)) {
return false;
}
/*
* See if we need to generate a compression header.
*/
if (bctx->chead) {
ser_declare;
/*
* Complete header
*/
ser_begin(bctx->chead, sizeof(comp_stream_header));
ser_uint32(bctx->ch.magic);
ser_uint32(bctx->compress_len);
ser_uint16(bctx->ch.level);
ser_uint16(bctx->ch.version);
ser_end(bctx->chead, sizeof(comp_stream_header));
bctx->compress_len += sizeof(comp_stream_header); /* add size of header */
}
bctx->jcr->store_bsock->msglen = bctx->compress_len; /* set compressed length */
bctx->cipher_input_len = bctx->compress_len;
}
/*
* Encrypt the data.
*/
need_more_data = false;
if (bit_is_set(FO_ENCRYPT, bctx->ff_pkt->flags) && !encrypt_data(bctx, &need_more_data)) {
if (need_more_data) {
return true;
}
return false;
}
/*
* Send the buffer to the Storage daemon
*/
if (bit_is_set(FO_SPARSE, bctx->ff_pkt->flags) || bit_is_set(FO_OFFSETS, bctx->ff_pkt->flags)) {
sd->msglen += OFFSET_FADDR_SIZE; /* include fileAddr in size */
}
sd->msg = bctx->wbuf; /* set correct write buffer */
if (!sd->send()) {
if (!bctx->jcr->is_job_canceled()) {
Jmsg1(bctx->jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), sd->bstrerror());
}
return false;
}
Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
bctx->jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
sd->msg = bctx->msgsave; /* restore read buffer */
return true;
}
/**
* Called here by find() for each file included.
* This is a callback. The original is find_files() above.
*
* Send the file and its data to the Storage daemon.
*
* Returns: 1 if OK
* 0 if error
* -1 to ignore file/directory (not used here)
*/
int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
{
bool do_read = false;
bool plugin_started = false;
bool do_plugin_set = false;
int status, data_stream;
int rtnstat = 0;
b_save_ctx bsctx;
bool has_file_data = false;
struct save_pkt sp; /* use by option plugin */
BSOCK *sd = jcr->store_bsock;
if (jcr->is_canceled() || jcr->is_incomplete()) {
return 0;
}
jcr->num_files_examined++; /* bump total file count */
switch (ff_pkt->type) {
case FT_LNKSAVED: /* Hard linked, file already saved */
Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
break;
case FT_REGE:
Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
has_file_data = true;
break;
case FT_REG:
Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
has_file_data = true;
break;
case FT_LNK:
Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
break;
case FT_RESTORE_FIRST:
Dmsg1(100, "FT_RESTORE_FIRST saving: %s\n", ff_pkt->fname);
break;
case FT_PLUGIN_CONFIG:
Dmsg1(100, "FT_PLUGIN_CONFIG saving: %s\n", ff_pkt->fname);
break;
case FT_DIRBEGIN:
jcr->num_files_examined--; /* correct file count */
return 1; /* not used */
case FT_NORECURSE:
Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
ff_pkt->top_fname, ff_pkt->fname);
ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
break;
case FT_NOFSCHG:
/* Suppress message for /dev filesystems */
if (!is_in_fileset(ff_pkt)) {
Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into it.\n"),
ff_pkt->fname, ff_pkt->top_fname);
}
ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
break;
case FT_INVALIDFS:
Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
ff_pkt->top_fname, ff_pkt->fname);
ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
break;
case FT_INVALIDDT:
Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
ff_pkt->fname);
break;
case FT_REPARSE:
case FT_JUNCTION:
case FT_DIREND:
Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
break;
case FT_SPEC:
Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
if (S_ISSOCK(ff_pkt->statp.st_mode)) {
Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname);
return 1;
}
break;
case FT_RAW:
Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
has_file_data = true;
break;
case FT_FIFO:
Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
break;
case FT_NOACCESS: {
berrno be;
Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access \"%s\": ERR=%s\n"),
ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
jcr->JobErrors++;
return 1;
}
case FT_NOFOLLOW: {
berrno be;
Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link \"%s\": ERR=%s\n"),
ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
jcr->JobErrors++;
return 1;
}
case FT_NOSTAT: {
berrno be;
Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat \"%s\": ERR=%s\n"),
ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
jcr->JobErrors++;
return 1;
}
case FT_DIRNOCHG:
case FT_NOCHG:
Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
return 1;
case FT_ISARCH:
Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
return 1;
case FT_NOOPEN: {
berrno be;
Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory \"%s\": ERR=%s\n"),
ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
jcr->JobErrors++;
return 1;
}
case FT_DELETED:
Dmsg1(130, "FT_DELETED: %s\n", ff_pkt->fname);
break;
default:
Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
ff_pkt->type, ff_pkt->fname);
jcr->JobErrors++;
return 1;
}
Dmsg1(130, "filed: sending %s to stored\n", ff_pkt->fname);
/*
* Setup backup signing context.
*/
memset(&bsctx, 0, sizeof(b_save_ctx));
bsctx.digest_stream = STREAM_NONE;
bsctx.jcr = jcr;
bsctx.ff_pkt = ff_pkt;
/*
* Digests and encryption are only useful if there's file data
*/
if (has_file_data) {
if (!setup_encryption_digests(bsctx)) {
goto good_rtn;
}
}
/*
* Initialize the file descriptor we use for data and other streams.
*/
binit(&ff_pkt->bfd);
if (bit_is_set(FO_PORTABLE, ff_pkt->flags)) {
set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
}
/*
* Option and cmd plugin are not compatible together
*/
if (ff_pkt->cmd_plugin) {
do_plugin_set = true;
} else if (ff_pkt->opt_plugin) {
/*
* Ask the option plugin what to do with this file
*/
switch (plugin_option_handle_file(jcr, ff_pkt, &sp)) {
case bRC_OK:
Dmsg2(10, "Option plugin %s will be used to backup %s\n", ff_pkt->plugin, ff_pkt->fname);
jcr->opt_plugin = true;
jcr->plugin_sp = &sp;
plugin_update_ff_pkt(ff_pkt, &sp);
do_plugin_set = true;
break;
case bRC_Skip:
Dmsg2(10, "Option plugin %s decided to skip %s\n", ff_pkt->plugin, ff_pkt->fname);
goto good_rtn;
case bRC_Core:
Dmsg2(10, "Option plugin %s decided to let bareos handle %s\n", ff_pkt->plugin, ff_pkt->fname);
break;
default:
goto bail_out;
}
}
if (do_plugin_set) {
/*
* Tell bfile that it needs to call plugin
*/
if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) {
goto bail_out;
}
send_plugin_name(jcr, sd, true); /* signal start of plugin data */
plugin_started = true;
}
/*
* Send attributes -- must be done after binit()
*/
if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
goto bail_out;
}
/*
* Meta data only for restore object
*/
if (IS_FT_OBJECT(ff_pkt->type)) {
goto good_rtn;
}
/*
* Meta data only for deleted files
*/
if (ff_pkt->type == FT_DELETED) {
goto good_rtn;
}
/*
* Set up the encryption context and send the session data to the SD
*/
if (has_file_data && jcr->crypto.pki_encrypt) {
if (!crypto_session_send(jcr, sd)) {
goto bail_out;
}
}
/*
* For a command plugin use the setting from the plugins savepkt no_read field
* which is saved in the ff_pkt->no_read variable. do_read is the inverted
* value of this variable as no_read == TRUE means do_read == FALSE
*/
if (ff_pkt->cmd_plugin) {
do_read = !ff_pkt->no_read;
} else {
/*
* Open any file with data that we intend to save, then save it.
*
* Note, if is_win32_backup, we must open the Directory so that
* the BackupRead will save its permissions and ownership streams.
*/
if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
#ifdef HAVE_WIN32
do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
#else
do_read = ff_pkt->statp.st_size > 0;
#endif
} else if (ff_pkt->type == FT_RAW ||
ff_pkt->type == FT_FIFO ||
ff_pkt->type == FT_REPARSE ||
ff_pkt->type == FT_JUNCTION ||
(!is_portable_backup(&ff_pkt->bfd) &&
ff_pkt->type == FT_DIREND)) {
do_read = true;
}
}
Dmsg2(150, "type=%d do_read=%d\n", ff_pkt->type, do_read);
if (do_read) {
btimer_t *tid;
int noatime;
if (ff_pkt->type == FT_FIFO) {
tid = start_thread_timer(jcr, pthread_self(), 60);
} else {
tid = NULL;
}
noatime = bit_is_set(FO_NOATIME, ff_pkt->flags) ? O_NOATIME : 0;
ff_pkt->bfd.reparse_point = (ff_pkt->type == FT_REPARSE ||
ff_pkt->type == FT_JUNCTION);
if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0, ff_pkt->statp.st_rdev) < 0) {
ff_pkt->ff_errno = errno;
berrno be;
Jmsg(jcr, M_NOTSAVED, 0,
_(" Cannot open \"%s\": ERR=%s.\n"),
ff_pkt->fname, be.bstrerror());
jcr->JobErrors++;
if (tid) {
stop_thread_timer(tid);
tid = NULL;
}
goto good_rtn;
}
if (tid) {
stop_thread_timer(tid);
tid = NULL;
}
status = send_data(jcr, data_stream, ff_pkt, bsctx.digest, bsctx.signing_digest);
if (bit_is_set(FO_CHKCHANGES, ff_pkt->flags)) {
has_file_changed(jcr, ff_pkt);
}
bclose(&ff_pkt->bfd);
if (!status) {
goto bail_out;
}
}
if (have_darwin_os) {
/*
* Regular files can have resource forks and Finder Info
*/
if (ff_pkt->type != FT_LNKSAVED &&
(S_ISREG(ff_pkt->statp.st_mode) &&
bit_is_set(FO_HFSPLUS, ff_pkt->flags))) {
if (!save_rsrc_and_finder(bsctx)) {
goto bail_out;
}
}
}
/*
* Save ACLs when requested and available for anything not being a symlink.
*/
if (have_acl) {
if (bit_is_set(FO_ACL, ff_pkt->flags) && ff_pkt->type != FT_LNK) {
if (!do_backup_acl(jcr, ff_pkt)) {
goto bail_out;
}
}
}
/*
* Save Extended Attributes when requested and available for all files.
*/
if (have_xattr) {
if (bit_is_set(FO_XATTR, ff_pkt->flags)) {
if (!do_backup_xattr(jcr, ff_pkt)) {
goto bail_out;
}
}
}
/*
* Terminate the signing digest and send it to the Storage daemon
*/
if (bsctx.signing_digest) {
if (!terminate_signing_digest(bsctx)) {
goto bail_out;
}
}
/*
* Terminate any digest and send it to Storage daemon
*/
if (bsctx.digest) {
if (!terminate_digest(bsctx)) {
goto bail_out;
}
}
/*
* Check if original file has a digest, and send it
*/
if (ff_pkt->type == FT_LNKSAVED && ff_pkt->digest) {
Dmsg2(300, "Link %s digest %d\n", ff_pkt->fname, ff_pkt->digest_len);
sd->fsend("%ld %d 0", jcr->JobFiles, ff_pkt->digest_stream);
sd->msg = check_pool_memory_size(sd->msg, ff_pkt->digest_len);
memcpy(sd->msg, ff_pkt->digest, ff_pkt->digest_len);
sd->msglen = ff_pkt->digest_len;
sd->send();
sd->signal(BNET_EOD); /* end of hardlink record */
}
good_rtn:
rtnstat = jcr->is_canceled() ? 0 : 1; /* good return if not canceled */
bail_out:
if (jcr->is_incomplete() || jcr->is_canceled()) {
rtnstat = 0;
}
if (plugin_started) {
send_plugin_name(jcr, sd, false); /* signal end of plugin data */
}
if (ff_pkt->opt_plugin) {
jcr->plugin_sp = NULL; /* sp is local to this function */
jcr->opt_plugin = false;
}
if (bsctx.digest) {
crypto_digest_free(bsctx.digest);
}
if (bsctx.signing_digest) {
crypto_digest_free(bsctx.signing_digest);
}
return rtnstat;
}
bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_stream)
{
BSOCK *sd = jcr->store_bsock;
POOL_MEM attribs(PM_NAME),
attribsExBuf(PM_NAME);
char *attribsEx = NULL;
int attr_stream;
int comp_len;
bool status;
int hangup = get_hangup();
#ifdef FD_NO_SEND_TEST
return true;
#endif
Dmsg1(300, "encode_and_send_attrs fname=%s\n", ff_pkt->fname);
/** Find what data stream we will use, then encode the attributes */
if ((data_stream = select_data_stream(ff_pkt, me->compatible)) == STREAM_NONE) {
/* This should not happen */
Jmsg0(jcr, M_FATAL, 0, _("Invalid file flags, no supported data stream type.\n"));
return false;
}
encode_stat(attribs.c_str(), &ff_pkt->statp, sizeof(ff_pkt->statp), ff_pkt->LinkFI, data_stream);
/** Now possibly extend the attributes */
if (IS_FT_OBJECT(ff_pkt->type)) {
attr_stream = STREAM_RESTORE_OBJECT;
} else {
attribsEx = attribsExBuf.c_str();
attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt);
}
Dmsg3(300, "File %s\nattribs=%s\nattribsEx=%s\n", ff_pkt->fname, attribs.c_str(), attribsEx);
jcr->lock();
jcr->JobFiles++; /* increment number of files sent */
ff_pkt->FileIndex = jcr->JobFiles; /* return FileIndex */
pm_strcpy(jcr->last_fname, ff_pkt->fname);
jcr->unlock();
/*
* Debug code: check if we must hangup
*/
if (hangup && (jcr->JobFiles > (uint32_t)hangup)) {
jcr->setJobStatus(JS_Incomplete);
Jmsg1(jcr, M_FATAL, 0, "Debug hangup requested after %d files.\n", hangup);
set_hangup(0);
return false;
}
/**
* Send Attributes header to Storage daemon
* <file-index> <stream> <info>
*/
if (!sd->fsend("%ld %d 0", jcr->JobFiles, attr_stream)) {
if (!jcr->is_canceled() && !jcr->is_incomplete()) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), sd->bstrerror());
}
return false;
}
Dmsg1(300, ">stored: attrhdr %s", sd->msg);
/**
* Send file attributes to Storage daemon
* File_index
* File type
* Filename (full path)
* Encoded attributes
* Link name (if type==FT_LNK or FT_LNKSAVED)
* Encoded extended-attributes (for Win32)
* Delta Sequence Number
*
* or send Restore Object to Storage daemon
* File_index
* File_type
* Object_index
* Object_len (possibly compressed)
* Object_full_len (not compressed)
* Object_compression
* Plugin_name
* Object_name
* Binary Object data
*
* For a directory, link is the same as fname, but with trailing
* slash. For a linked file, link is the link.
*/
if (!IS_FT_OBJECT(ff_pkt->type) && ff_pkt->type != FT_DELETED) { /* already stripped */
strip_path(ff_pkt);
}
switch (ff_pkt->type) {
case FT_JUNCTION:
case FT_LNK:
case FT_LNKSAVED:
Dmsg3(300, "Link %d %s to %s\n", jcr->JobFiles, ff_pkt->fname, ff_pkt->link);
status = sd->fsend("%ld %d %s%c%s%c%s%c%s%c%u%c", jcr->JobFiles,
ff_pkt->type, ff_pkt->fname, 0, attribs.c_str(), 0,
ff_pkt->link, 0, attribsEx, 0, ff_pkt->delta_seq, 0);
break;
case FT_DIREND:
case FT_REPARSE:
/* Here link is the canonical filename (i.e. with trailing slash) */
status = sd->fsend("%ld %d %s%c%s%c%c%s%c%u%c", jcr->JobFiles,
ff_pkt->type, ff_pkt->link, 0, attribs.c_str(), 0, 0,
attribsEx, 0, ff_pkt->delta_seq, 0);
break;
case FT_PLUGIN_CONFIG:
case FT_RESTORE_FIRST:
comp_len = ff_pkt->object_len;
ff_pkt->object_compression = 0;
if (ff_pkt->object_len > 1000) {
/*
* Big object, compress it
*/
comp_len = compressBound(ff_pkt->object_len);
POOLMEM *comp_obj = get_memory(comp_len);
/*
* FIXME: check Zdeflate error
*/
Zdeflate(ff_pkt->object, ff_pkt->object_len, comp_obj, comp_len);
if (comp_len < ff_pkt->object_len) {
ff_pkt->object = comp_obj;
ff_pkt->object_compression = 1; /* zlib level 9 compression */
} else {
/*
* Uncompressed object smaller, use it
*/
comp_len = ff_pkt->object_len;
}
Dmsg2(100, "Object compressed from %d to %d bytes\n", ff_pkt->object_len, comp_len);
}
sd->msglen = Mmsg(sd->msg, "%d %d %d %d %d %d %s%c%s%c",
jcr->JobFiles, ff_pkt->type, ff_pkt->object_index,
comp_len, ff_pkt->object_len, ff_pkt->object_compression,
ff_pkt->fname, 0, ff_pkt->object_name, 0);
sd->msg = check_pool_memory_size(sd->msg, sd->msglen + comp_len + 2);
memcpy(sd->msg + sd->msglen, ff_pkt->object, comp_len);
/*
* Note we send one extra byte so Dir can store zero after object
*/
sd->msglen += comp_len + 1;
status = sd->send();
if (ff_pkt->object_compression) {
free_and_null_pool_memory(ff_pkt->object);
}
break;
case FT_REG:
status = sd->fsend("%ld %d %s%c%s%c%c%s%c%d%c", jcr->JobFiles,
ff_pkt->type, ff_pkt->fname, 0, attribs.c_str(), 0, 0,
attribsEx, 0, ff_pkt->delta_seq, 0);
break;
default:
status = sd->fsend("%ld %d %s%c%s%c%c%s%c%u%c", jcr->JobFiles,
ff_pkt->type, ff_pkt->fname, 0, attribs.c_str(), 0, 0,
attribsEx, 0, ff_pkt->delta_seq, 0);
break;
}
if (!IS_FT_OBJECT(ff_pkt->type) && ff_pkt->type != FT_DELETED) {
unstrip_path(ff_pkt);
}
Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
if (!status && !jcr->is_job_canceled()) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), sd->bstrerror());
}
sd->signal(BNET_EOD); /* indicate end of attributes data */
return status;
}
/**
* Send data read from an already open file descriptor.
*
* We return 1 on sucess and 0 on errors.
*
* ***FIXME***
* We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop reading.
* Currently this is not a problem as the only other stream, resource forks,
* are not handled as sparse files.
*/
static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt,
DIGEST *digest, DIGEST *signing_digest)
{
b_ctx bctx;
BSOCK *sd = jcr->store_bsock;
#ifdef FD_NO_SEND_TEST
return 1;
#endif
/*
* Setup backup context.
*/
memset(&bctx, 0, sizeof(b_ctx));
bctx.jcr = jcr;
bctx.ff_pkt = ff_pkt;
bctx.msgsave = sd->msg; /* save the original sd buffer */
bctx.rbuf = sd->msg; /* read buffer */
bctx.wbuf = sd->msg; /* write buffer */
bctx.rsize = jcr->buf_size; /* read buffer size */
bctx.cipher_input = (uint8_t *)bctx.rbuf; /* encrypt uncompressed data */
bctx.digest = digest; /* encryption digest */
bctx.signing_digest = signing_digest; /* signing digest */
Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
if (!setup_compression_context(bctx)) {
goto bail_out;
}
if (!setup_encryption_context(bctx)) {
goto bail_out;
}
/*
* Send Data header to Storage daemon
* <file-index> <stream> <info>
*/
if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
if (!jcr->is_job_canceled()) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), sd->bstrerror());
}
goto bail_out;
}
Dmsg1(300, ">stored: datahdr %s", sd->msg);
/*
* Make space at beginning of buffer for fileAddr because this
* same buffer will be used for writing if compression is off.
*/
if (bit_is_set(FO_SPARSE, ff_pkt->flags) || bit_is_set(FO_OFFSETS, ff_pkt->flags)) {
bctx.rbuf += OFFSET_FADDR_SIZE;
bctx.rsize -= OFFSET_FADDR_SIZE;
#ifdef HAVE_FREEBSD_OS
/*
* To read FreeBSD partitions, the read size must be a multiple of 512.
*/
bctx.rsize = (bctx.rsize / 512) * 512;
#endif
}
/*
* A RAW device read on win32 only works if the buffer is a multiple of 512
*/
#ifdef HAVE_WIN32
if (S_ISBLK(ff_pkt->statp.st_mode)) {
bctx.rsize = (bctx.rsize / 512) * 512;
}
if (ff_pkt->statp.st_rdev & FILE_ATTRIBUTE_ENCRYPTED) {
if (!send_encrypted_data(bctx)) {
goto bail_out;
}
} else {
if (!send_plain_data(bctx)) {
goto bail_out;
}
}
#else
if (!send_plain_data(bctx)) {
goto bail_out;
}
#endif
if (sd->msglen < 0) { /* error */
berrno be;
Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"), ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
if (jcr->JobErrors++ > 1000) { /* insanity check */
Jmsg(jcr, M_FATAL, 0, _("Too many errors. JobErrors=%d.\n"), jcr->JobErrors);
}
} else if (bit_is_set(FO_ENCRYPT, ff_pkt->flags)) {
/*
* For encryption, we must call finalize to push out any buffered data.
*/
if (!crypto_cipher_finalize(bctx.cipher_ctx,
(uint8_t *)jcr->crypto.crypto_buf,
&bctx.encrypted_len)) {
/*
* Padding failed. Shouldn't happen.
*/
Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
goto bail_out;
}
/*
* Note, on SSL pre-0.9.7, there is always some output
*/
if (bctx.encrypted_len > 0) {
sd->msglen = bctx.encrypted_len; /* set encrypted length */
sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
if (!sd->send()) {
if (!jcr->is_job_canceled()) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), sd->bstrerror());
}
goto bail_out;
}
Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
sd->msg = bctx.msgsave; /* restore bnet buffer */
}
}
if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
if (!jcr->is_job_canceled()) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), sd->bstrerror());
}
goto bail_out;
}
/*
* Free the cipher context
*/
if (bctx.cipher_ctx) {
crypto_cipher_free(bctx.cipher_ctx);
}
return 1;
bail_out:
/*
* Free the cipher context
*/
if (bctx.cipher_ctx) {
crypto_cipher_free(bctx.cipher_ctx);
}
sd->msg = bctx.msgsave; /* restore bnet buffer */
sd->msglen = 0;
return 0;
}
/*
* Called here for each record from read_records()
* This function is used when we do a external clone of a Job e.g.
* this SD is the reading SD. And a remote SD is the writing SD.
*
* Returns: true if OK
* false if error
*/
static bool clone_record_to_remote_sd(DCR *dcr, DEV_RECORD *rec)
{
POOLMEM *msgsave;
JCR *jcr = dcr->jcr;
char buf1[100], buf2[100];
BSOCK *sd = jcr->store_bsock;
bool send_eod, send_header;
#ifdef xxx
Dmsg5(000, "on entry JobId=%d FI=%s SessId=%d Strm=%s len=%d\n",
jcr->JobId, FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId,
stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len);
#endif
/*
* If label discard it
*/
if (rec->FileIndex < 0) {
return true;
}
/*
* See if this is the first record being processed.
*/
if (rec->last_FileIndex == 0) {
/*
* Initialize the last counters so we can compare
* things in the next run through here.
*/
rec->last_VolSessionId = rec->VolSessionId;
rec->last_VolSessionTime = rec->VolSessionTime;
rec->last_FileIndex = rec->FileIndex;
rec->last_Stream = rec->Stream;
jcr->JobFiles = 1;
/*
* Need to send both a new header only.
*/
send_eod = false;
send_header = true;
} else {
/*
* See if we are changing file or stream type.
*/
if (rec->VolSessionId != rec->last_VolSessionId ||
rec->VolSessionTime != rec->last_VolSessionTime ||
rec->FileIndex != rec->last_FileIndex ||
rec->Stream != rec->last_Stream) {
/*
* See if we are changing the FileIndex e.g.
* start processing the next file in the backup stream.
*/
if (rec->FileIndex != rec->last_FileIndex) {
jcr->JobFiles++;
}
/*
* Keep track of the new state.
*/
rec->last_VolSessionId = rec->VolSessionId;
rec->last_VolSessionTime = rec->VolSessionTime;
rec->last_FileIndex = rec->FileIndex;
rec->last_Stream = rec->Stream;
/*
* Need to send both a EOD and a new header.
*/
send_eod = true;
send_header = true;
} else {
send_eod = false;
send_header = false;
}
}
/*
* Send a EOD when needed.
*/
if (send_eod) {
if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
if (!jcr->is_job_canceled()) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
sd->bstrerror());
}
return false;
}
}
/*
* Send a header when needed.
*/
if (send_header) {
if (!sd->fsend("%ld %d 0", rec->FileIndex, rec->Stream)) {
if (!jcr->is_job_canceled()) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
sd->bstrerror());
}
return false;
}
}
/*
* Send the record data.
* We don't want to copy the data from the record to the socket structure
* so we save the original msg pointer and use the record data pointer for
* sending and restore the original msg pointer when done.
*/
msgsave = sd->msg;
sd->msg = rec->data;
sd->msglen = rec->data_len;
if (!sd->send()) {
sd->msg = msgsave;
sd->msglen = 0;
if (!jcr->is_job_canceled()) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
sd->bstrerror());
}
return false;
}
jcr->JobBytes += sd->msglen;
sd->msg = msgsave;
Dmsg5(500, "wrote_record JobId=%d FI=%s SessId=%d Strm=%s len=%d\n",
jcr->JobId, FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId,
stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len);
return true;
}
/*
* Send data read from an already open file descriptor.
*
* We return 1 on sucess and 0 on errors.
*
* ***FIXME***
* We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop
* reading.
* Currently this is not a problem as the only other stream, resource forks,
* are not handled as sparse files.
*/
static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest,
DIGEST *signing_digest)
{
BSOCK *sd = jcr->store_bsock;
uint64_t fileAddr = 0; /* file address */
char *rbuf, *wbuf;
int32_t rsize = jcr->buf_size; /* read buffer size */
POOLMEM *msgsave;
CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */
const uint8_t *cipher_input;
uint32_t cipher_input_len;
uint32_t cipher_block_size;
uint32_t encrypted_len;
#ifdef FD_NO_SEND_TEST
return 1;
#endif
msgsave = sd->msg;
rbuf = sd->msg; /* read buffer */
wbuf = sd->msg; /* write buffer */
cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */
Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type);
#ifdef HAVE_LIBZ
uLong compress_len = 0;
uLong max_compress_len = 0;
const Bytef *cbuf = NULL;
int zstat;
if (ff_pkt->flags & FO_GZIP) {
if (ff_pkt->flags & FO_SPARSE) {
cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE;
max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE;
} else {
cbuf = (Bytef *)jcr->compress_buf;
max_compress_len = jcr->compress_buf_size; /* set max length */
}
wbuf = jcr->compress_buf; /* compressed output here */
cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
/*
* Only change zlib parameters if there is no pending operation.
* This should never happen as deflatereset is called after each
* deflate.
*/
if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) {
/* set gzip compression level - must be done per file */
if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset,
ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) {
Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat);
set_jcr_job_status(jcr, JS_ErrorTerminated);
goto err;
}
}
}
#else
const uint32_t max_compress_len = 0;
#endif
if (ff_pkt->flags & FO_ENCRYPT) {
if (ff_pkt->flags & FO_SPARSE) {
Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n"));
goto err;
}
/* Allocate the cipher context */
if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true,
&cipher_block_size)) == NULL) {
/* Shouldn't happen! */
Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n"));
goto err;
}
/*
* Grow the crypto buffer, if necessary.
* crypto_cipher_update() will buffer up to (cipher_block_size - 1).
* We grow crypto_buf to the maximum number of blocks that
* could be returned for the given read buffer size.
* (Using the larger of either rsize or max_compress_len)
*/
jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf,
(MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) +
cipher_block_size - 1) / cipher_block_size * cipher_block_size);
wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */
}
/*
* Send Data header to Storage daemon
* <file-index> <stream> <info>
*/
if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
sd->bstrerror());
goto err;
}
Dmsg1(300, ">stored: datahdr %s\n", sd->msg);
/*
* Make space at beginning of buffer for fileAddr because this
* same buffer will be used for writing if compression is off.
*/
if (ff_pkt->flags & FO_SPARSE) {
rbuf += SPARSE_FADDR_SIZE;
rsize -= SPARSE_FADDR_SIZE;
#ifdef HAVE_FREEBSD_OS
/*
* To read FreeBSD partitions, the read size must be
* a multiple of 512.
*/
rsize = (rsize/512) * 512;
#endif
}
/* a RAW device read on win32 only works if the buffer is a multiple of 512 */
#ifdef HAVE_WIN32
if (S_ISBLK(ff_pkt->statp.st_mode))
rsize = (rsize/512) * 512;
#endif
/*
* Read the file data
*/
while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) {
/* Check for sparse blocks */
if (ff_pkt->flags & FO_SPARSE) {
ser_declare;
bool allZeros = false;
if ((sd->msglen == rsize &&
fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size) ||
((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) &&
(uint64_t)ff_pkt->statp.st_size == 0)) {
allZeros = is_buf_zero(rbuf, rsize);
}
if (!allZeros) {
/* Put file address as first data in buffer */
ser_begin(wbuf, SPARSE_FADDR_SIZE);
ser_uint64(fileAddr); /* store fileAddr in begin of buffer */
}
fileAddr += sd->msglen; /* update file address */
/* Skip block of all zeros */
if (allZeros) {
continue; /* skip block of zeros */
}
}
jcr->ReadBytes += sd->msglen; /* count bytes read */
/* Uncompressed cipher input length */
cipher_input_len = sd->msglen;
/* Update checksum if requested */
if (digest) {
crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen);
}
/* Update signing digest if requested */
if (signing_digest) {
crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen);
}
#ifdef HAVE_LIBZ
/* Do compression if turned on */
if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) {
Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen);
((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf;
((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen;
((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf;
((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len;
if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) {
Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat);
set_jcr_job_status(jcr, JS_ErrorTerminated);
goto err;
}
compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out;
/* reset zlib stream to be able to begin from scratch again */
if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) {
Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat);
set_jcr_job_status(jcr, JS_ErrorTerminated);
goto err;
}
Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len,
sd->msglen);
sd->msglen = compress_len; /* set compressed length */
cipher_input_len = compress_len;
}
#endif
/*
* Note, here we prepend the current record length to the beginning
* of the encrypted data. This is because both sparse and compression
* restore handling want records returned to them with exactly the
* same number of bytes that were processed in the backup handling.
* That is, both are block filters rather than a stream. When doing
* compression, the compression routines may buffer data, so that for
* any one record compressed, when it is decompressed the same size
* will not be obtained. Of course, the buffered data eventually comes
* out in subsequent crypto_cipher_update() calls or at least
* when crypto_cipher_finalize() is called. Unfortunately, this
* "feature" of encryption enormously complicates the restore code.
*/
if (ff_pkt->flags & FO_ENCRYPT) {
uint32_t initial_len = 0;
ser_declare;
if (ff_pkt->flags & FO_SPARSE) {
cipher_input_len += SPARSE_FADDR_SIZE;
}
/* Encrypt the length of the input block */
uint8_t packet_len[sizeof(uint32_t)];
ser_begin(packet_len, sizeof(uint32_t));
ser_uint32(cipher_input_len); /* store data len in begin of buffer */
Dmsg1(20, "Encrypt len=%d\n", cipher_input_len);
if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len),
(uint8_t *)jcr->crypto.crypto_buf, &initial_len)) {
/* Encryption failed. Shouldn't happen. */
Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
goto err;
}
/* Encrypt the input block */
if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len,
(uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) {
if ((initial_len + encrypted_len) == 0) {
/* No full block of data available, read more data */
continue;
}
Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len,
sd->msglen);
sd->msglen = initial_len + encrypted_len; /* set encrypted length */
} else {
/* Encryption failed. Shouldn't happen. */
Jmsg(jcr, M_FATAL, 0, _("Encryption error\n"));
goto err;
}
}
/* Send the buffer to the Storage daemon */
if (ff_pkt->flags & FO_SPARSE) {
sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */
}
sd->msg = wbuf; /* set correct write buffer */
if (!sd->send()) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
sd->bstrerror());
goto err;
}
Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
/* #endif */
jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
sd->msg = msgsave; /* restore read buffer */
} /* end while read file data */
if (sd->msglen < 0) { /* error */
berrno be;
Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"),
ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno));
if (jcr->JobErrors++ > 1000) { /* insanity check */
Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n"));
}
} else if (ff_pkt->flags & FO_ENCRYPT) {
/*
* For encryption, we must call finalize to push out any
* buffered data.
*/
if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf,
&encrypted_len)) {
/* Padding failed. Shouldn't happen. */
Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n"));
goto err;
}
/* Note, on SSL pre-0.9.7, there is always some output */
if (encrypted_len > 0) {
sd->msglen = encrypted_len; /* set encrypted length */
sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */
if (!sd->send()) {
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
sd->bstrerror());
goto err;
}
Dmsg1(130, "Send data to SD len=%d\n", sd->msglen);
jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */
sd->msg = msgsave; /* restore bnet buffer */
}
}
if (!sd->signal(BNET_EOD)) { /* indicate end of file data */
Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
sd->bstrerror());
goto err;
}
/* Free the cipher context */
if (cipher_ctx) {
crypto_cipher_free(cipher_ctx);
}
return 1;
err:
/* Free the cipher context */
if (cipher_ctx) {
crypto_cipher_free(cipher_ctx);
}
sd->msg = msgsave; /* restore bnet buffer */
sd->msglen = 0;
return 0;
}
/*
* Called here by find() for each file included.
* This is a callback. The original is find_files() above.
*
* Send the file and its data to the Storage daemon.
*
* Returns: 1 if OK
* 0 if error
* -1 to ignore file/directory (not used here)
*/
int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
{
bool do_read = false;
int stat, data_stream;
int rtnstat = 0;
DIGEST *digest = NULL;
DIGEST *signing_digest = NULL;
int digest_stream = STREAM_NONE;
SIGNATURE *sig = NULL;
bool has_file_data = false;
// TODO landonf: Allow the user to specify the digest algorithm
#ifdef HAVE_SHA2
crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA256;
#else
crypto_digest_t signing_algorithm = CRYPTO_DIGEST_SHA1;
#endif
BSOCK *sd = jcr->store_bsock;
if (job_canceled(jcr)) {
return 0;
}
jcr->num_files_examined++; /* bump total file count */
switch (ff_pkt->type) {
case FT_LNKSAVED: /* Hard linked, file already saved */
Dmsg2(130, "FT_LNKSAVED hard link: %s => %s\n", ff_pkt->fname, ff_pkt->link);
break;
case FT_REGE:
Dmsg1(130, "FT_REGE saving: %s\n", ff_pkt->fname);
has_file_data = true;
break;
case FT_REG:
Dmsg1(130, "FT_REG saving: %s\n", ff_pkt->fname);
has_file_data = true;
break;
case FT_LNK:
Dmsg2(130, "FT_LNK saving: %s -> %s\n", ff_pkt->fname, ff_pkt->link);
break;
case FT_DIRBEGIN:
jcr->num_files_examined--; /* correct file count */
return 1; /* not used */
case FT_NORECURSE:
Jmsg(jcr, M_INFO, 1, _(" Recursion turned off. Will not descend from %s into %s\n"),
ff_pkt->top_fname, ff_pkt->fname);
ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
break;
case FT_NOFSCHG:
/* Suppress message for /dev filesystems */
if (!is_in_fileset(ff_pkt)) {
Jmsg(jcr, M_INFO, 1, _(" %s is a different filesystem. Will not descend from %s into %s\n"),
ff_pkt->fname, ff_pkt->top_fname, ff_pkt->fname);
}
ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
break;
case FT_INVALIDFS:
Jmsg(jcr, M_INFO, 1, _(" Disallowed filesystem. Will not descend from %s into %s\n"),
ff_pkt->top_fname, ff_pkt->fname);
ff_pkt->type = FT_DIREND; /* Backup only the directory entry */
break;
case FT_INVALIDDT:
Jmsg(jcr, M_INFO, 1, _(" Disallowed drive type. Will not descend into %s\n"),
ff_pkt->fname);
break;
case FT_REPARSE:
case FT_DIREND:
Dmsg1(130, "FT_DIREND: %s\n", ff_pkt->link);
break;
case FT_SPEC:
Dmsg1(130, "FT_SPEC saving: %s\n", ff_pkt->fname);
if (S_ISSOCK(ff_pkt->statp.st_mode)) {
Jmsg(jcr, M_SKIPPED, 1, _(" Socket file skipped: %s\n"), ff_pkt->fname);
return 1;
}
break;
case FT_RAW:
Dmsg1(130, "FT_RAW saving: %s\n", ff_pkt->fname);
has_file_data = true;
break;
case FT_FIFO:
Dmsg1(130, "FT_FIFO saving: %s\n", ff_pkt->fname);
break;
case FT_NOACCESS: {
berrno be;
Jmsg(jcr, M_NOTSAVED, 0, _(" Could not access \"%s\": ERR=%s\n"), ff_pkt->fname,
be.bstrerror(ff_pkt->ff_errno));
jcr->JobErrors++;
return 1;
}
case FT_NOFOLLOW: {
berrno be;
Jmsg(jcr, M_NOTSAVED, 0, _(" Could not follow link \"%s\": ERR=%s\n"),
ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
jcr->JobErrors++;
return 1;
}
case FT_NOSTAT: {
berrno be;
Jmsg(jcr, M_NOTSAVED, 0, _(" Could not stat \"%s\": ERR=%s\n"), ff_pkt->fname,
be.bstrerror(ff_pkt->ff_errno));
jcr->JobErrors++;
return 1;
}
case FT_DIRNOCHG:
case FT_NOCHG:
Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
return 1;
case FT_ISARCH:
Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
return 1;
case FT_NOOPEN: {
berrno be;
Jmsg(jcr, M_NOTSAVED, 0, _(" Could not open directory \"%s\": ERR=%s\n"),
ff_pkt->fname, be.bstrerror(ff_pkt->ff_errno));
jcr->JobErrors++;
return 1;
}
default:
Jmsg(jcr, M_NOTSAVED, 0, _(" Unknown file type %d; not saved: %s\n"),
ff_pkt->type, ff_pkt->fname);
jcr->JobErrors++;
return 1;
}
Dmsg1(130, "bfiled: sending %s to stored\n", ff_pkt->fname);
/* Digests and encryption are only useful if there's file data */
if (has_file_data) {
/*
* Setup for digest handling. If this fails, the digest will be set to NULL
* and not used. Note, the digest (file hash) can be any one of the four
* algorithms below.
*
* The signing digest is a single algorithm depending on
* whether or not we have SHA2.
* ****FIXME**** the signing algoritm should really be
* determined a different way!!!!!! What happens if
* sha2 was available during backup but not restore?
*/
if (ff_pkt->flags & FO_MD5) {
digest = crypto_digest_new(jcr, CRYPTO_DIGEST_MD5);
digest_stream = STREAM_MD5_DIGEST;
} else if (ff_pkt->flags & FO_SHA1) {
digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA1);
digest_stream = STREAM_SHA1_DIGEST;
} else if (ff_pkt->flags & FO_SHA256) {
digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA256);
digest_stream = STREAM_SHA256_DIGEST;
} else if (ff_pkt->flags & FO_SHA512) {
digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
digest_stream = STREAM_SHA512_DIGEST;
}
/* Did digest initialization fail? */
if (digest_stream != STREAM_NONE && digest == NULL) {
Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),
stream_to_ascii(digest_stream));
}
/*
* Set up signature digest handling. If this fails, the signature digest will be set to
* NULL and not used.
*/
// TODO landonf: We should really only calculate the digest once, for both verification and signing.
if (jcr->crypto.pki_sign) {
signing_digest = crypto_digest_new(jcr, signing_algorithm);
/* Full-stop if a failure occurred initializing the signature digest */
if (signing_digest == NULL) {
Jmsg(jcr, M_NOTSAVED, 0, _("%s signature digest initialization failed\n"),
stream_to_ascii(signing_algorithm));
jcr->JobErrors++;
goto good_rtn;
}
}
/* Enable encryption */
if (jcr->crypto.pki_encrypt) {
ff_pkt->flags |= FO_ENCRYPT;
}
}
/* Initialize the file descriptor we use for data and other streams. */
binit(&ff_pkt->bfd);
if (ff_pkt->flags & FO_PORTABLE) {
set_portable_backup(&ff_pkt->bfd); /* disable Win32 BackupRead() */
}
if (ff_pkt->cmd_plugin) {
if (!set_cmd_plugin(&ff_pkt->bfd, jcr)) {
goto bail_out;
}
send_plugin_name(jcr, sd, true); /* signal start of plugin data */
}
/* Send attributes -- must be done after binit() */
if (!encode_and_send_attributes(jcr, ff_pkt, data_stream)) {
goto bail_out;
}
/* Set up the encryption context and send the session data to the SD */
if (has_file_data && jcr->crypto.pki_encrypt) {
if (!crypto_session_send(jcr, sd)) {
goto bail_out;
}
}
/*
* Open any file with data that we intend to save, then save it.
*
* Note, if is_win32_backup, we must open the Directory so that
* the BackupRead will save its permissions and ownership streams.
*/
if (ff_pkt->type != FT_LNKSAVED && S_ISREG(ff_pkt->statp.st_mode)) {
#ifdef HAVE_WIN32
do_read = !is_portable_backup(&ff_pkt->bfd) || ff_pkt->statp.st_size > 0;
#else
do_read = ff_pkt->statp.st_size > 0;
#endif
} else if (ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO ||
ff_pkt->type == FT_REPARSE ||
(!is_portable_backup(&ff_pkt->bfd) && ff_pkt->type == FT_DIREND)) {
do_read = true;
}
if (ff_pkt->cmd_plugin) {
do_read = true;
}
Dmsg1(400, "do_read=%d\n", do_read);
if (do_read) {
btimer_t *tid;
if (ff_pkt->type == FT_FIFO) {
tid = start_thread_timer(jcr, pthread_self(), 60);
} else {
tid = NULL;
}
int noatime = ff_pkt->flags & FO_NOATIME ? O_NOATIME : 0;
ff_pkt->bfd.reparse_point = ff_pkt->type == FT_REPARSE;
if (bopen(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY | noatime, 0) < 0) {
ff_pkt->ff_errno = errno;
berrno be;
Jmsg(jcr, M_NOTSAVED, 0, _(" Cannot open \"%s\": ERR=%s.\n"), ff_pkt->fname,
be.bstrerror());
jcr->JobErrors++;
if (tid) {
stop_thread_timer(tid);
tid = NULL;
}
goto good_rtn;
}
if (tid) {
stop_thread_timer(tid);
tid = NULL;
}
stat = send_data(jcr, data_stream, ff_pkt, digest, signing_digest);
if (ff_pkt->flags & FO_CHKCHANGES) {
has_file_changed(jcr, ff_pkt);
}
bclose(&ff_pkt->bfd);
if (!stat) {
goto bail_out;
}
}
#ifdef HAVE_DARWIN_OS
/* Regular files can have resource forks and Finder Info */
if (ff_pkt->type != FT_LNKSAVED && (S_ISREG(ff_pkt->statp.st_mode) &&
ff_pkt->flags & FO_HFSPLUS)) {
if (ff_pkt->hfsinfo.rsrclength > 0) {
int flags;
int rsrc_stream;
if (!bopen_rsrc(&ff_pkt->bfd, ff_pkt->fname, O_RDONLY | O_BINARY, 0) < 0) {
ff_pkt->ff_errno = errno;
berrno be;
Jmsg(jcr, M_NOTSAVED, -1, _(" Cannot open resource fork for \"%s\": ERR=%s.\n"),
ff_pkt->fname, be.bstrerror());
jcr->JobErrors++;
if (is_bopen(&ff_pkt->bfd)) {
bclose(&ff_pkt->bfd);
}
goto good_rtn;
}
flags = ff_pkt->flags;
ff_pkt->flags &= ~(FO_GZIP|FO_SPARSE);
if (flags & FO_ENCRYPT) {
rsrc_stream = STREAM_ENCRYPTED_MACOS_FORK_DATA;
} else {
rsrc_stream = STREAM_MACOS_FORK_DATA;
}
stat = send_data(jcr, rsrc_stream, ff_pkt, digest, signing_digest);
ff_pkt->flags = flags;
bclose(&ff_pkt->bfd);
if (!stat) {
goto bail_out;
}
}
Dmsg1(300, "Saving Finder Info for \"%s\"\n", ff_pkt->fname);
sd->fsend("%ld %d 0", jcr->JobFiles, STREAM_HFSPLUS_ATTRIBUTES);
Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
pm_memcpy(sd->msg, ff_pkt->hfsinfo.fndrinfo, 32);
sd->msglen = 32;
if (digest) {
crypto_digest_update(digest, (uint8_t *)sd->msg, sd->msglen);
}
if (signing_digest) {
crypto_digest_update(signing_digest, (uint8_t *)sd->msg, sd->msglen);
}
sd->send();
sd->signal(BNET_EOD);
}
#endif
/*
* Save ACLs for anything not being a symlink and not being a plugin.
*/
if (!ff_pkt->cmd_plugin) {
if (ff_pkt->flags & FO_ACL && ff_pkt->type != FT_LNK) {
if (!build_acl_streams(jcr, ff_pkt))
goto bail_out;
}
}
/*
* Save Extended Attributes for all files not being a plugin.
*/
if (!ff_pkt->cmd_plugin) {
if (ff_pkt->flags & FO_XATTR) {
if (!build_xattr_streams(jcr, ff_pkt))
goto bail_out;
}
}
/* Terminate the signing digest and send it to the Storage daemon */
if (signing_digest) {
uint32_t size = 0;
if ((sig = crypto_sign_new(jcr)) == NULL) {
Jmsg(jcr, M_FATAL, 0, _("Failed to allocate memory for crypto signature.\n"));
goto bail_out;
}
if (!crypto_sign_add_signer(sig, signing_digest, jcr->crypto.pki_keypair)) {
Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
goto bail_out;
}
/* Get signature size */
if (!crypto_sign_encode(sig, NULL, &size)) {
Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
goto bail_out;
}
/* Grow the bsock buffer to fit our message if necessary */
if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
sd->msg = realloc_pool_memory(sd->msg, size);
}
/* Send our header */
sd->fsend("%ld %ld 0", jcr->JobFiles, STREAM_SIGNED_DIGEST);
Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
/* Encode signature data */
if (!crypto_sign_encode(sig, (uint8_t *)sd->msg, &size)) {
Jmsg(jcr, M_FATAL, 0, _("An error occurred while signing the stream.\n"));
goto bail_out;
}
sd->msglen = size;
sd->send();
sd->signal(BNET_EOD); /* end of checksum */
}
/* Terminate any digest and send it to Storage daemon */
if (digest) {
uint32_t size;
sd->fsend("%ld %d 0", jcr->JobFiles, digest_stream);
Dmsg1(300, "bfiled>stored:header %s\n", sd->msg);
size = CRYPTO_DIGEST_MAX_SIZE;
/* Grow the bsock buffer to fit our message if necessary */
if (sizeof_pool_memory(sd->msg) < (int32_t)size) {
sd->msg = realloc_pool_memory(sd->msg, size);
}
if (!crypto_digest_finalize(digest, (uint8_t *)sd->msg, &size)) {
Jmsg(jcr, M_FATAL, 0, _("An error occurred finalizing signing the stream.\n"));
goto bail_out;
}
sd->msglen = size;
sd->send();
sd->signal(BNET_EOD); /* end of checksum */
}
if (ff_pkt->cmd_plugin) {
send_plugin_name(jcr, sd, false); /* signal end of plugin data */
}
good_rtn:
rtnstat = 1; /* good return */
bail_out:
if (digest) {
crypto_digest_free(digest);
}
if (signing_digest) {
crypto_digest_free(signing_digest);
}
if (sig) {
crypto_sign_free(sig);
}
return rtnstat;
}