/** * Send data read from an already open file descriptor. * * We return 1 on sucess and 0 on errors. * * ***FIXME*** * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop reading. * Currently this is not a problem as the only other stream, resource forks, * are not handled as sparse files. */ static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signing_digest) { b_ctx bctx; BSOCK *sd = jcr->store_bsock; #ifdef FD_NO_SEND_TEST return 1; #endif /* * Setup backup context. */ memset(&bctx, 0, sizeof(b_ctx)); bctx.jcr = jcr; bctx.ff_pkt = ff_pkt; bctx.msgsave = sd->msg; /* save the original sd buffer */ bctx.rbuf = sd->msg; /* read buffer */ bctx.wbuf = sd->msg; /* write buffer */ bctx.rsize = jcr->buf_size; /* read buffer size */ bctx.cipher_input = (uint8_t *)bctx.rbuf; /* encrypt uncompressed data */ bctx.digest = digest; /* encryption digest */ bctx.signing_digest = signing_digest; /* signing digest */ Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type); if (!setup_compression_context(bctx)) { goto bail_out; } if (!setup_encryption_context(bctx)) { goto bail_out; } /* * Send Data header to Storage daemon * <file-index> <stream> <info> */ if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) { if (!jcr->is_job_canceled()) { Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), sd->bstrerror()); } goto bail_out; } Dmsg1(300, ">stored: datahdr %s", sd->msg); /* * Make space at beginning of buffer for fileAddr because this * same buffer will be used for writing if compression is off. */ if (bit_is_set(FO_SPARSE, ff_pkt->flags) || bit_is_set(FO_OFFSETS, ff_pkt->flags)) { bctx.rbuf += OFFSET_FADDR_SIZE; bctx.rsize -= OFFSET_FADDR_SIZE; #ifdef HAVE_FREEBSD_OS /* * To read FreeBSD partitions, the read size must be a multiple of 512. */ bctx.rsize = (bctx.rsize / 512) * 512; #endif } /* * A RAW device read on win32 only works if the buffer is a multiple of 512 */ #ifdef HAVE_WIN32 if (S_ISBLK(ff_pkt->statp.st_mode)) { bctx.rsize = (bctx.rsize / 512) * 512; } if (ff_pkt->statp.st_rdev & FILE_ATTRIBUTE_ENCRYPTED) { if (!send_encrypted_data(bctx)) { goto bail_out; } } else { if (!send_plain_data(bctx)) { goto bail_out; } } #else if (!send_plain_data(bctx)) { goto bail_out; } #endif if (sd->msglen < 0) { /* error */ berrno be; Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"), ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno)); if (jcr->JobErrors++ > 1000) { /* insanity check */ Jmsg(jcr, M_FATAL, 0, _("Too many errors. JobErrors=%d.\n"), jcr->JobErrors); } } else if (bit_is_set(FO_ENCRYPT, ff_pkt->flags)) { /* * For encryption, we must call finalize to push out any buffered data. */ if (!crypto_cipher_finalize(bctx.cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf, &bctx.encrypted_len)) { /* * Padding failed. Shouldn't happen. */ Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n")); goto bail_out; } /* * Note, on SSL pre-0.9.7, there is always some output */ if (bctx.encrypted_len > 0) { sd->msglen = bctx.encrypted_len; /* set encrypted length */ sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */ if (!sd->send()) { if (!jcr->is_job_canceled()) { Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), sd->bstrerror()); } goto bail_out; } Dmsg1(130, "Send data to SD len=%d\n", sd->msglen); jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */ sd->msg = bctx.msgsave; /* restore bnet buffer */ } } if (!sd->signal(BNET_EOD)) { /* indicate end of file data */ if (!jcr->is_job_canceled()) { Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), sd->bstrerror()); } goto bail_out; } /* * Free the cipher context */ if (bctx.cipher_ctx) { crypto_cipher_free(bctx.cipher_ctx); } return 1; bail_out: /* * Free the cipher context */ if (bctx.cipher_ctx) { crypto_cipher_free(bctx.cipher_ctx); } sd->msg = bctx.msgsave; /* restore bnet buffer */ sd->msglen = 0; return 0; }
/* * Send data read from an already open file descriptor. * * We return 1 on sucess and 0 on errors. * * ***FIXME*** * We use ff_pkt->statp.st_size when FO_SPARSE to know when to stop * reading. * Currently this is not a problem as the only other stream, resource forks, * are not handled as sparse files. */ static int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *signing_digest) { BSOCK *sd = jcr->store_bsock; uint64_t fileAddr = 0; /* file address */ char *rbuf, *wbuf; int32_t rsize = jcr->buf_size; /* read buffer size */ POOLMEM *msgsave; CIPHER_CONTEXT *cipher_ctx = NULL; /* Quell bogus uninitialized warnings */ const uint8_t *cipher_input; uint32_t cipher_input_len; uint32_t cipher_block_size; uint32_t encrypted_len; #ifdef FD_NO_SEND_TEST return 1; #endif msgsave = sd->msg; rbuf = sd->msg; /* read buffer */ wbuf = sd->msg; /* write buffer */ cipher_input = (uint8_t *)rbuf; /* encrypt uncompressed data */ Dmsg1(300, "Saving data, type=%d\n", ff_pkt->type); #ifdef HAVE_LIBZ uLong compress_len = 0; uLong max_compress_len = 0; const Bytef *cbuf = NULL; int zstat; if (ff_pkt->flags & FO_GZIP) { if (ff_pkt->flags & FO_SPARSE) { cbuf = (Bytef *)jcr->compress_buf + SPARSE_FADDR_SIZE; max_compress_len = jcr->compress_buf_size - SPARSE_FADDR_SIZE; } else { cbuf = (Bytef *)jcr->compress_buf; max_compress_len = jcr->compress_buf_size; /* set max length */ } wbuf = jcr->compress_buf; /* compressed output here */ cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */ /* * Only change zlib parameters if there is no pending operation. * This should never happen as deflatereset is called after each * deflate. */ if (((z_stream*)jcr->pZLIB_compress_workset)->total_in == 0) { /* set gzip compression level - must be done per file */ if ((zstat=deflateParams((z_stream*)jcr->pZLIB_compress_workset, ff_pkt->GZIP_level, Z_DEFAULT_STRATEGY)) != Z_OK) { Jmsg(jcr, M_FATAL, 0, _("Compression deflateParams error: %d\n"), zstat); set_jcr_job_status(jcr, JS_ErrorTerminated); goto err; } } } #else const uint32_t max_compress_len = 0; #endif if (ff_pkt->flags & FO_ENCRYPT) { if (ff_pkt->flags & FO_SPARSE) { Jmsg0(jcr, M_FATAL, 0, _("Encrypting sparse data not supported.\n")); goto err; } /* Allocate the cipher context */ if ((cipher_ctx = crypto_cipher_new(jcr->crypto.pki_session, true, &cipher_block_size)) == NULL) { /* Shouldn't happen! */ Jmsg0(jcr, M_FATAL, 0, _("Failed to initialize encryption context.\n")); goto err; } /* * Grow the crypto buffer, if necessary. * crypto_cipher_update() will buffer up to (cipher_block_size - 1). * We grow crypto_buf to the maximum number of blocks that * could be returned for the given read buffer size. * (Using the larger of either rsize or max_compress_len) */ jcr->crypto.crypto_buf = check_pool_memory_size(jcr->crypto.crypto_buf, (MAX(rsize + (int)sizeof(uint32_t), (int32_t)max_compress_len) + cipher_block_size - 1) / cipher_block_size * cipher_block_size); wbuf = jcr->crypto.crypto_buf; /* Encrypted, possibly compressed output here. */ } /* * Send Data header to Storage daemon * <file-index> <stream> <info> */ if (!sd->fsend("%ld %d 0", jcr->JobFiles, stream)) { Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), sd->bstrerror()); goto err; } Dmsg1(300, ">stored: datahdr %s\n", sd->msg); /* * Make space at beginning of buffer for fileAddr because this * same buffer will be used for writing if compression is off. */ if (ff_pkt->flags & FO_SPARSE) { rbuf += SPARSE_FADDR_SIZE; rsize -= SPARSE_FADDR_SIZE; #ifdef HAVE_FREEBSD_OS /* * To read FreeBSD partitions, the read size must be * a multiple of 512. */ rsize = (rsize/512) * 512; #endif } /* a RAW device read on win32 only works if the buffer is a multiple of 512 */ #ifdef HAVE_WIN32 if (S_ISBLK(ff_pkt->statp.st_mode)) rsize = (rsize/512) * 512; #endif /* * Read the file data */ while ((sd->msglen=(uint32_t)bread(&ff_pkt->bfd, rbuf, rsize)) > 0) { /* Check for sparse blocks */ if (ff_pkt->flags & FO_SPARSE) { ser_declare; bool allZeros = false; if ((sd->msglen == rsize && fileAddr+sd->msglen < (uint64_t)ff_pkt->statp.st_size) || ((ff_pkt->type == FT_RAW || ff_pkt->type == FT_FIFO) && (uint64_t)ff_pkt->statp.st_size == 0)) { allZeros = is_buf_zero(rbuf, rsize); } if (!allZeros) { /* Put file address as first data in buffer */ ser_begin(wbuf, SPARSE_FADDR_SIZE); ser_uint64(fileAddr); /* store fileAddr in begin of buffer */ } fileAddr += sd->msglen; /* update file address */ /* Skip block of all zeros */ if (allZeros) { continue; /* skip block of zeros */ } } jcr->ReadBytes += sd->msglen; /* count bytes read */ /* Uncompressed cipher input length */ cipher_input_len = sd->msglen; /* Update checksum if requested */ if (digest) { crypto_digest_update(digest, (uint8_t *)rbuf, sd->msglen); } /* Update signing digest if requested */ if (signing_digest) { crypto_digest_update(signing_digest, (uint8_t *)rbuf, sd->msglen); } #ifdef HAVE_LIBZ /* Do compression if turned on */ if (ff_pkt->flags & FO_GZIP && jcr->pZLIB_compress_workset) { Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", cbuf, rbuf, sd->msglen); ((z_stream*)jcr->pZLIB_compress_workset)->next_in = (Bytef *)rbuf; ((z_stream*)jcr->pZLIB_compress_workset)->avail_in = sd->msglen; ((z_stream*)jcr->pZLIB_compress_workset)->next_out = (Bytef *)cbuf; ((z_stream*)jcr->pZLIB_compress_workset)->avail_out = max_compress_len; if ((zstat=deflate((z_stream*)jcr->pZLIB_compress_workset, Z_FINISH)) != Z_STREAM_END) { Jmsg(jcr, M_FATAL, 0, _("Compression deflate error: %d\n"), zstat); set_jcr_job_status(jcr, JS_ErrorTerminated); goto err; } compress_len = ((z_stream*)jcr->pZLIB_compress_workset)->total_out; /* reset zlib stream to be able to begin from scratch again */ if ((zstat=deflateReset((z_stream*)jcr->pZLIB_compress_workset)) != Z_OK) { Jmsg(jcr, M_FATAL, 0, _("Compression deflateReset error: %d\n"), zstat); set_jcr_job_status(jcr, JS_ErrorTerminated); goto err; } Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len, sd->msglen); sd->msglen = compress_len; /* set compressed length */ cipher_input_len = compress_len; } #endif /* * Note, here we prepend the current record length to the beginning * of the encrypted data. This is because both sparse and compression * restore handling want records returned to them with exactly the * same number of bytes that were processed in the backup handling. * That is, both are block filters rather than a stream. When doing * compression, the compression routines may buffer data, so that for * any one record compressed, when it is decompressed the same size * will not be obtained. Of course, the buffered data eventually comes * out in subsequent crypto_cipher_update() calls or at least * when crypto_cipher_finalize() is called. Unfortunately, this * "feature" of encryption enormously complicates the restore code. */ if (ff_pkt->flags & FO_ENCRYPT) { uint32_t initial_len = 0; ser_declare; if (ff_pkt->flags & FO_SPARSE) { cipher_input_len += SPARSE_FADDR_SIZE; } /* Encrypt the length of the input block */ uint8_t packet_len[sizeof(uint32_t)]; ser_begin(packet_len, sizeof(uint32_t)); ser_uint32(cipher_input_len); /* store data len in begin of buffer */ Dmsg1(20, "Encrypt len=%d\n", cipher_input_len); if (!crypto_cipher_update(cipher_ctx, packet_len, sizeof(packet_len), (uint8_t *)jcr->crypto.crypto_buf, &initial_len)) { /* Encryption failed. Shouldn't happen. */ Jmsg(jcr, M_FATAL, 0, _("Encryption error\n")); goto err; } /* Encrypt the input block */ if (crypto_cipher_update(cipher_ctx, cipher_input, cipher_input_len, (uint8_t *)&jcr->crypto.crypto_buf[initial_len], &encrypted_len)) { if ((initial_len + encrypted_len) == 0) { /* No full block of data available, read more data */ continue; } Dmsg2(400, "encrypted len=%d unencrypted len=%d\n", encrypted_len, sd->msglen); sd->msglen = initial_len + encrypted_len; /* set encrypted length */ } else { /* Encryption failed. Shouldn't happen. */ Jmsg(jcr, M_FATAL, 0, _("Encryption error\n")); goto err; } } /* Send the buffer to the Storage daemon */ if (ff_pkt->flags & FO_SPARSE) { sd->msglen += SPARSE_FADDR_SIZE; /* include fileAddr in size */ } sd->msg = wbuf; /* set correct write buffer */ if (!sd->send()) { Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), sd->bstrerror()); goto err; } Dmsg1(130, "Send data to SD len=%d\n", sd->msglen); /* #endif */ jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */ sd->msg = msgsave; /* restore read buffer */ } /* end while read file data */ if (sd->msglen < 0) { /* error */ berrno be; Jmsg(jcr, M_ERROR, 0, _("Read error on file %s. ERR=%s\n"), ff_pkt->fname, be.bstrerror(ff_pkt->bfd.berrno)); if (jcr->JobErrors++ > 1000) { /* insanity check */ Jmsg(jcr, M_FATAL, 0, _("Too many errors.\n")); } } else if (ff_pkt->flags & FO_ENCRYPT) { /* * For encryption, we must call finalize to push out any * buffered data. */ if (!crypto_cipher_finalize(cipher_ctx, (uint8_t *)jcr->crypto.crypto_buf, &encrypted_len)) { /* Padding failed. Shouldn't happen. */ Jmsg(jcr, M_FATAL, 0, _("Encryption padding error\n")); goto err; } /* Note, on SSL pre-0.9.7, there is always some output */ if (encrypted_len > 0) { sd->msglen = encrypted_len; /* set encrypted length */ sd->msg = jcr->crypto.crypto_buf; /* set correct write buffer */ if (!sd->send()) { Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), sd->bstrerror()); goto err; } Dmsg1(130, "Send data to SD len=%d\n", sd->msglen); jcr->JobBytes += sd->msglen; /* count bytes saved possibly compressed/encrypted */ sd->msg = msgsave; /* restore bnet buffer */ } } if (!sd->signal(BNET_EOD)) { /* indicate end of file data */ Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"), sd->bstrerror()); goto err; } /* Free the cipher context */ if (cipher_ctx) { crypto_cipher_free(cipher_ctx); } return 1; err: /* Free the cipher context */ if (cipher_ctx) { crypto_cipher_free(cipher_ctx); } sd->msg = msgsave; /* restore bnet buffer */ sd->msglen = 0; return 0; }