static int write_to_changed_file(struct asfd *asfd,
struct asfd *chfd, struct manios *manios,
struct slist *slist, int end_flags)
{
struct sbuf *sb;
if(!slist) return 0;
while((sb=slist->head))
{
if(sb->flags & SBUF_NEED_DATA)
{
switch(sbuf_needs_data(sb, asfd, chfd, manios, slist,
end_flags))
{
case 0: return 0;
case 1: continue;
default: return -1;
}
}
else
{
// No change, can go straight in.
if(manio_write_sbuf(manios->changed, sb)) return -1;
if(write_endfile(sb, manios)) return -1;
// Move along.
slist_advance(slist);
}
}
return 0;
}
static int sbuf_needs_data(struct sbuf *sb, struct asfd *asfd,
struct asfd *chfd, struct manios *manios,
struct slist *slist, int end_flags)
{
int ret=-1;
struct blk *blk;
static struct iobuf wbuf;
struct blist *blist=slist->blist;
if(!(sb->flags & SBUF_HEADER_WRITTEN_TO_MANIFEST))
{
if(manio_write_sbuf(manios->changed, sb)) goto end;
sb->flags |= SBUF_HEADER_WRITTEN_TO_MANIFEST;
}
while((blk=sb->protocol2->bstart)
&& blk->got==BLK_GOT
&& (blk->next || end_flags&END_BACKUP))
{
if(blk->got_save_path
&& !blk_is_zero_length(blk))
{
if(breaking && breakcount--==0)
{
breakpoint(breaking, __func__);
goto end;
}
if(manio_write_sig_and_path(manios->changed, blk))
goto end;
if(manios->changed->sig_count==0)
{
// Have finished a manifest file. Want to start
// using it as a dedup candidate now.
if(manio_component_to_chfd(chfd,
manios->changed->offset->ppath))
goto end;
if(!blk->requested)
{
// Also let the client know, so that it
// can free memory if there was a long
// consecutive number of unrequested
// blocks.
get_wbuf_from_index(&wbuf, blk->index);
if(asfd->write(asfd, &wbuf)) goto end;
}
}
}
if(blk==sb->protocol2->bend)
{
blist_adjust_head(blist, sb);
if(write_endfile(sb, manios)) return -1;
slist_advance(slist);
return 1;
}
if(sb->protocol2->bsighead==sb->protocol2->bstart)
sb->protocol2->bsighead=blk->next;
sb->protocol2->bstart=blk->next;
if(blk==blist->blk_from_champ_chooser)
blist->blk_from_champ_chooser=blk->next;
}
if(!blk && sb && !sb->protocol2->bend && (end_flags&END_BACKUP))
{
// Write endfile for the very last file.
if(write_endfile(sb, manios)) return -1;
}
ret=0;
end:
blist_adjust_head(blist, sb);
return ret;
}
int send_whole_filel(struct asfd *asfd,
#ifdef HAVE_WIN32
enum cmd cmd,
#endif
const char *datapth,
int quick_read, uint64_t *bytes, struct cntr *cntr,
struct BFILE *bfd, const char *extrameta, size_t elen)
{
int ret=0;
size_t s=0;
MD5_CTX md5;
char buf[4096]="";
struct iobuf wbuf;
if(!bfd)
{
logp("No bfd in %s()\n", __func__);
return -1;
}
if(!MD5_Init(&md5))
{
logp("MD5_Init() failed\n");
return -1;
}
if(extrameta)
{
size_t metalen=0;
const char *metadata=NULL;
metadata=extrameta;
metalen=elen;
// Send metadata in chunks, rather than all at once.
while(metalen>0)
{
if(metalen>ZCHUNK) s=ZCHUNK;
else s=metalen;
if(!MD5_Update(&md5, metadata, s))
{
logp("MD5_Update() failed\n");
ret=-1;
}
iobuf_set(&wbuf, CMD_APPEND, (char *)metadata, s);
if(asfd->write(asfd, &wbuf))
{
ret=-1;
}
metadata+=s;
metalen-=s;
*bytes+=s;
}
}
else
{
#ifdef HAVE_WIN32
if(!ret && cmd==CMD_EFS_FILE)
{
struct winbuf mybuf;
mybuf.md5=&md5;
mybuf.quick_read=quick_read;
mybuf.datapth=datapth;
mybuf.cntr=cntr;
mybuf.bytes=bytes;
mybuf.asfd=asfd;
// The EFS read function, ReadEncryptedFileRaw(),
// works in an annoying way. You have to give it a
// function that it calls repeatedly every time the
// read buffer is called.
// So ReadEncryptedFileRaw() will not return until
// it has read the whole file. I have no idea why
// they do not have a plain 'read()' function for it.
ReadEncryptedFileRaw((PFE_EXPORT_FUNC)write_efs,
&mybuf, bfd->pvContext);
}
else
#endif
if(!ret)
{
#ifdef HAVE_WIN32
int do_known_byte_count=0;
size_t datalen=bfd->datalen;
if(datalen>0) do_known_byte_count=1;
#endif
while(1)
{
#ifdef HAVE_WIN32
if(do_known_byte_count)
{
s=(uint32_t)bfd->read(bfd,
buf, min((size_t)4096, datalen));
datalen-=s;
}
else
{
#endif
s=(uint32_t)bfd->read(bfd, buf, 4096);
#ifdef HAVE_WIN32
}
#endif
if(s<=0) break;
*bytes+=s;
if(!MD5_Update(&md5, buf, s))
{
logp("MD5_Update() failed\n");
ret=-1;
break;
}
iobuf_set(&wbuf, CMD_APPEND, buf, s);
if(asfd->write(asfd, &wbuf))
{
ret=-1;
break;
}
if(quick_read)
{
int qr;
if((qr=do_quick_read(asfd, datapth, cntr))<0)
{
ret=-1;
break;
}
if(qr)
{
// client wants to interrupt
break;
}
}
#ifdef HAVE_WIN32
// Windows VSS headers tell us how many bytes to
// expect.
if(do_known_byte_count && datalen<=0) break;
#endif
}
}
}
if(!ret)
{
uint8_t checksum[MD5_DIGEST_LENGTH];
if(!MD5_Final(checksum, &md5))
{
logp("MD5_Final() failed\n");
return -1;
}
return write_endfile(asfd, *bytes, checksum);
}
return ret;
}
static int load_signature_and_send_delta(BFILE *bfd, FILE *in, unsigned long long *bytes, unsigned long long *sentbytes, struct cntr *cntr, size_t datalen)
{
rs_job_t *job;
rs_result r;
rs_signature_t *sumset=NULL;
unsigned char checksum[MD5_DIGEST_LENGTH+1];
rs_filebuf_t *infb=NULL;
rs_filebuf_t *outfb=NULL;
rs_buffers_t rsbuf;
memset(&rsbuf, 0, sizeof(rsbuf));
if(load_signature(&sumset, cntr)) return -1;
//logp("start delta\n");
if(!(job=rs_delta_begin(sumset)))
{
logp("could not start delta job.\n");
rs_free_sumset(sumset);
return RS_IO_ERROR;
}
if(!(infb=rs_filebuf_new(bfd,
in, NULL, -1, ASYNC_BUF_LEN, datalen, cntr))
|| !(outfb=rs_filebuf_new(NULL, NULL,
NULL, async_get_fd(), ASYNC_BUF_LEN, -1, cntr)))
{
logp("could not rs_filebuf_new for delta\n");
if(infb) rs_filebuf_free(infb);
return -1;
}
//logp("start delta loop\n");
while(1)
{
size_t wlen=0;
rs_result delresult;
delresult=rs_async(job, &rsbuf, infb, outfb);
if(delresult==RS_DONE)
{
r=delresult;
// logp("delresult done\n");
break;
}
else if(delresult==RS_BLOCKED || delresult==RS_RUNNING)
{
// logp("delresult running/blocked: %d\n", delresult);
// Keep going
}
else
{
logp("error in rs_async for delta: %d\n", delresult);
r=delresult;
break;
}
// FIX ME: get it to read stuff (errors, for example) here too.
if(async_rw(NULL, NULL, '\0', '\0', NULL, &wlen))
return -1;
}
if(r!=RS_DONE)
logp("delta loop returned: %d\n", r);
//logp("after delta loop: %d\n", r);
//logp("\n");
if(r==RS_DONE)
{
*bytes=infb->bytes;
*sentbytes=outfb->bytes;
if(!MD5_Final(checksum, &(infb->md5)))
{
logp("MD5_Final() failed\n");
r=RS_IO_ERROR;
}
}
rs_filebuf_free(infb);
rs_filebuf_free(outfb);
rs_job_free(job);
rs_free_sumset(sumset);
if(r==RS_DONE && write_endfile(*bytes, checksum)) // finish delta file
return -1;
//logp("end of load_sig_send_delta\n");
return r;
}
/* OK, this function is getting a bit out of control.
One problem is that, if you give deflateInit2 compression=0, it still
writes gzip headers and footers, so I had to add extra
if(compression) and if(!compression) bits all over the place that would
skip the actual compression.
This is needed for the case where encryption is on and compression is off.
Encryption off and compression off uses send_whole_file().
Perhaps a separate function is needed for encryption on compression off.
*/
int send_whole_file_gzl(struct asfd *asfd, const char *datapth,
int quick_read, uint64_t *bytes, const char *encpassword,
struct cntr *cntr, int compression, struct BFILE *bfd,
const char *extrameta, size_t elen)
{
int ret=0;
int zret=0;
MD5_CTX md5;
size_t metalen=0;
const char *metadata=NULL;
struct iobuf wbuf;
int have;
z_stream strm;
int flush=Z_NO_FLUSH;
uint8_t in[ZCHUNK];
uint8_t out[ZCHUNK];
int eoutlen;
uint8_t eoutbuf[ZCHUNK+EVP_MAX_BLOCK_LENGTH];
EVP_CIPHER_CTX *enc_ctx=NULL;
#ifdef HAVE_WIN32
int do_known_byte_count=0;
size_t datalen=bfd->datalen;
if(datalen>0) do_known_byte_count=1;
#endif
if(encpassword && !(enc_ctx=enc_setup(1, encpassword)))
return -1;
if(!MD5_Init(&md5))
{
logp("MD5_Init() failed\n");
return -1;
}
//logp("send_whole_file_gz: %s%s\n", fname, extrameta?" (meta)":"");
if((metadata=extrameta))
{
metalen=elen;
}
/* allocate deflate state */
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
if((zret=deflateInit2(&strm, compression, Z_DEFLATED, (15+16),
8, Z_DEFAULT_STRATEGY))!=Z_OK)
{
return -1;
}
do
{
if(metadata)
{
if(metalen>ZCHUNK)
strm.avail_in=ZCHUNK;
else
strm.avail_in=metalen;
memcpy(in, metadata, strm.avail_in);
metadata+=strm.avail_in;
metalen-=strm.avail_in;
}
else
{
// Windows VSS headers give us how much data to
// expect to read.
#ifdef HAVE_WIN32
if(do_known_byte_count)
{
if(datalen<=0) strm.avail_in=0;
else strm.avail_in=
(uint32_t)bfd->read(bfd, in,
min((size_t)ZCHUNK, datalen));
datalen-=strm.avail_in;
}
else
#endif
strm.avail_in=
(uint32_t)bfd->read(bfd, in, ZCHUNK);
}
if(!compression && !strm.avail_in) break;
*bytes+=strm.avail_in;
// The checksum needs to be later if encryption is being used.
if(!enc_ctx)
{
if(!MD5_Update(&md5, in, strm.avail_in))
{
logp("MD5_Update() failed\n");
ret=-1;
break;
}
}
#ifdef HAVE_WIN32
if(do_known_byte_count && datalen<=0) flush=Z_FINISH;
else
#endif
if(strm.avail_in) flush=Z_NO_FLUSH;
else flush=Z_FINISH;
strm.next_in=in;
/* run deflate() on input until output buffer not full, finish
compression if all of source has been read in */
do
{
if(compression)
{
strm.avail_out = ZCHUNK;
strm.next_out = out;
zret = deflate(&strm, flush); /* no bad return value */
if(zret==Z_STREAM_ERROR) /* state not clobbered */
{
logp("z_stream_error\n");
ret=-1;
break;
}
have = ZCHUNK-strm.avail_out;
}
else
{
have=strm.avail_in;
memcpy(out, in, have);
}
if(enc_ctx)
{
if(do_encryption(asfd, enc_ctx, out, have,
eoutbuf, &eoutlen, &md5))
{
ret=-1;
break;
}
}
else
{
iobuf_set(&wbuf, CMD_APPEND, (char *)out, have);
if(asfd->write(asfd, &wbuf))
{
ret=-1;
break;
}
}
if(quick_read && datapth)
{
int qr;
if((qr=do_quick_read(asfd, datapth, cntr))<0)
{
ret=-1;
break;
}
if(qr) // client wants to interrupt
{
goto cleanup;
}
}
if(!compression) break;
} while (!strm.avail_out);
if(ret) break;
if(!compression) continue;
if(strm.avail_in) /* all input will be used */
{
ret=-1;
logp("strm.avail_in=%d\n", strm.avail_in);
break;
}
} while(flush!=Z_FINISH);
if(!ret)
{
if(compression && zret!=Z_STREAM_END)
{
logp("ret OK, but zstream not finished: %d\n", zret);
ret=-1;
}
else if(enc_ctx)
{
if(!EVP_CipherFinal_ex(enc_ctx, eoutbuf, &eoutlen))
{
logp("Encryption failure at the end\n");
ret=-1;
}
else if(eoutlen>0)
{
iobuf_set(&wbuf, CMD_APPEND,
(char *)eoutbuf, (size_t)eoutlen);
if(asfd->write(asfd, &wbuf))
ret=-1;
else if(!MD5_Update(&md5, eoutbuf, eoutlen))
{
logp("MD5_Update() failed\n");
ret=-1;
}
}
}
}
cleanup:
deflateEnd(&strm);
if(enc_ctx)
{
EVP_CIPHER_CTX_cleanup(enc_ctx);
free(enc_ctx);
}
if(!ret)
{
uint8_t checksum[MD5_DIGEST_LENGTH];
if(!MD5_Final(checksum, &md5))
{
logp("MD5_Final() failed\n");
return -1;
}
return write_endfile(asfd, *bytes, checksum);
}
//logp("end of send\n");
return ret;
}
static int load_signature_and_send_delta(struct asfd *asfd,
BFILE *bfd, FILE *in,
unsigned long long *bytes, unsigned long long *sentbytes,
struct conf *conf, size_t datalen)
{
rs_job_t *job;
rs_result r;
rs_signature_t *sumset=NULL;
unsigned char checksum[MD5_DIGEST_LENGTH+1];
rs_filebuf_t *infb=NULL;
rs_filebuf_t *outfb=NULL;
rs_buffers_t rsbuf;
memset(&rsbuf, 0, sizeof(rsbuf));
if(load_signature(asfd, &sumset, conf)) return -1;
if(!(job=rs_delta_begin(sumset)))
{
logp("could not start delta job.\n");
rs_free_sumset(sumset);
return RS_IO_ERROR;
}
if(!(infb=rs_filebuf_new(asfd, bfd,
in, NULL, -1, ASYNC_BUF_LEN, datalen, conf->cntr))
|| !(outfb=rs_filebuf_new(asfd, NULL, NULL,
NULL, asfd->fd, ASYNC_BUF_LEN, -1, conf->cntr)))
{
logp("could not rs_filebuf_new for delta\n");
if(infb) rs_filebuf_free(infb);
return -1;
}
while(1)
{
rs_result delresult;
delresult=rs_async(job, &rsbuf, infb, outfb);
if(delresult==RS_DONE)
{
r=delresult;
break;
}
else if(delresult==RS_BLOCKED || delresult==RS_RUNNING)
{
// Keep going
}
else
{
logp("error in rs_async for delta: %d\n", delresult);
r=delresult;
break;
}
// FIX ME: get it to read stuff (errors, for example) here too.
if(asfd->as->rw(asfd->as)) return -1;
}
if(r!=RS_DONE)
logp("delta loop returned: %d\n", r);
if(r==RS_DONE)
{
*bytes=infb->bytes;
*sentbytes=outfb->bytes;
if(!MD5_Final(checksum, &(infb->md5)))
{
logp("MD5_Final() failed\n");
r=RS_IO_ERROR;
}
}
rs_filebuf_free(infb);
rs_filebuf_free(outfb);
rs_job_free(job);
rs_free_sumset(sumset);
if(r==RS_DONE && write_endfile(asfd, *bytes, checksum))
return -1;
return r;
}
int send_whole_file_gz(struct asfd *asfd,
const char *fname, const char *datapth, int quick_read,
unsigned long long *bytes, struct conf **confs,
int compression, FILE *fp)
{
int ret=0;
int zret=0;
unsigned have;
z_stream strm;
int flush=Z_NO_FLUSH;
uint8_t in[ZCHUNK];
uint8_t out[ZCHUNK];
struct iobuf wbuf;
//logp("send_whole_file_gz: %s%s\n", fname, extrameta?" (meta)":"");
/* allocate deflate state */
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
if((zret=deflateInit2(&strm, compression, Z_DEFLATED, (15+16),
8, Z_DEFAULT_STRATEGY))!=Z_OK)
return -1;
do
{
strm.avail_in=fread(in, 1, ZCHUNK, fp);
if(!compression && !strm.avail_in) break;
*bytes+=strm.avail_in;
if(strm.avail_in) flush=Z_NO_FLUSH;
else flush=Z_FINISH;
strm.next_in=in;
// Run deflate() on input until output buffer not full, finish
// compression if all of source has been read in.
do
{
if(compression)
{
strm.avail_out=ZCHUNK;
strm.next_out=out;
zret=deflate(&strm, flush);
if(zret==Z_STREAM_ERROR)
{
logp("z_stream_error\n");
ret=-1;
break;
}
have=ZCHUNK-strm.avail_out;
}
else
{
have=strm.avail_in;
memcpy(out, in, have);
}
wbuf.cmd=CMD_APPEND;
wbuf.buf=(char *)out;
wbuf.len=have;
if(asfd->write(asfd, &wbuf))
{
ret=-1;
break;
}
if(quick_read && datapth)
{
int qr;
if((qr=do_quick_read(asfd, datapth, confs))<0)
{
ret=-1;
break;
}
if(qr) // Client wants to interrupt.
{
goto cleanup;
}
}
if(!compression) break;
} while(!strm.avail_out);
if(ret) break;
if(!compression) continue;
if(strm.avail_in) /* all input will be used */
{
ret=-1;
logp("strm.avail_in=%d\n", strm.avail_in);
break;
}
} while(flush!=Z_FINISH);
if(!ret)
{
if(compression && zret!=Z_STREAM_END)
{
logp("ret OK, but zstream not finished: %d\n", zret);
ret=-1;
}
}
cleanup:
deflateEnd(&strm);
if(!ret)
{
return write_endfile(asfd, *bytes);
}
//logp("end of send\n");
return ret;
}
static int sbuf_needs_data(struct sbuf *sb, struct asfd *asfd,
struct asfd *chfd, struct manios *manios,
struct slist *slist, int end_flags)
{
int ret=-1;
struct blk *blk;
static struct iobuf wbuf;
struct blist *blist=slist->blist;
static int unrequested=0;
if(!(sb->flags & SBUF_HEADER_WRITTEN_TO_MANIFEST))
{
if(manio_write_sbuf(manios->changed, sb)) goto end;
sb->flags |= SBUF_HEADER_WRITTEN_TO_MANIFEST;
}
while((blk=sb->protocol2->bstart)
&& blk->got==BLK_GOT
&& (blk->next || end_flags&END_BACKUP))
{
if(blk->got_save_path
&& !blk_is_zero_length(blk))
{
if(breaking && breakcount--==0)
{
breakpoint(breaking, __func__);
goto end;
}
if(manio_write_sig_and_path(manios->changed, blk))
goto end;
if(manios->changed->sig_count==0)
{
// Have finished a manifest file. Want to start
// using it as a dedup candidate now.
if(manio_component_to_chfd(chfd,
manios->changed->offset->ppath))
goto end;
// The champ chooser has the candidate. Now,
// empty our local hash table.
hash_delete_all();
// Add the most recent block, so identical
// adjacent blocks are deduplicated well.
if(hash_load_blk(blk))
goto end;
}
}
if(!blk->requested)
unrequested++;
if(unrequested>BLKS_MAX_UNREQUESTED)
{
unrequested=0;
// Let the client know that it can free memory if there
// was a long consecutive number of unrequested blocks.
get_wbuf_from_index(&wbuf, blk->index);
if(asfd->write(asfd, &wbuf))
goto end;
}
if(blk==sb->protocol2->bend)
{
blist_adjust_head(blist, sb);
if(write_endfile(sb, manios)) return -1;
slist_advance(slist);
return 1;
}
if(sb->protocol2->bsighead==sb->protocol2->bstart)
sb->protocol2->bsighead=blk->next;
sb->protocol2->bstart=blk->next;
if(blk==blist->blk_from_champ_chooser)
blist->blk_from_champ_chooser=blk->next;
}
if(!blk && sb && !sb->protocol2->bend && (end_flags&END_BACKUP))
{
// Write endfile for the very last file.
if(write_endfile(sb, manios)) return -1;
}
ret=0;
end:
blist_adjust_head(blist, sb);
return ret;
}
int send_whole_file(char cmd, const char *fname, const char *datapth, int quick_read, unsigned long long *bytes, struct cntr *cntr, BFILE *bfd, FILE *fp, const char *extrameta, size_t elen)
{
int ret=0;
size_t s=0;
MD5_CTX md5;
char buf[4096]="";
if(!MD5_Init(&md5))
{
logp("MD5_Init() failed\n");
return -1;
}
if(extrameta)
{
size_t metalen=0;
const char *metadata=NULL;
metadata=extrameta;
metalen=elen;
// Send metadata in chunks, rather than all at once.
while(metalen>0)
{
if(metalen>ZCHUNK) s=ZCHUNK;
else s=metalen;
if(!MD5_Update(&md5, metadata, s))
{
logp("MD5_Update() failed\n");
ret=-1;
}
if(async_write(CMD_APPEND, metadata, s))
{
ret=-1;
}
metadata+=s;
metalen-=s;
*bytes+=s;
}
}
else
{
#ifdef HAVE_WIN32
if(!ret && cmd==CMD_EFS_FILE)
{
struct winbuf mybuf;
mybuf.md5=&md5;
mybuf.quick_read=quick_read;
mybuf.datapth=datapth;
mybuf.cntr=cntr;
mybuf.bytes=bytes;
// The EFS read function, ReadEncryptedFileRaw(),
// works in an annoying way. You have to give it a
// function that it calls repeatedly every time the
// read buffer is called.
// So ReadEncryptedFileRaw() will not return until
// it has read the whole file. I have no idea why
// they do not have a plain 'read()' function for it.
ReadEncryptedFileRaw((PFE_EXPORT_FUNC)write_efs,
&mybuf, bfd->pvContext);
}
if(!ret && cmd!=CMD_EFS_FILE)
{
while((s=(uint32_t)bread(bfd, buf, 4096))>0)
{
*bytes+=s;
if(!MD5_Update(&md5, buf, s))
{
logp("MD5_Update() failed\n");
ret=-1;
break;
}
if(async_write(CMD_APPEND, buf, s))
{
ret=-1;
break;
}
if(quick_read)
{
int qr;
if((qr=do_quick_read(datapth, cntr))<0)
{
ret=-1;
break;
}
if(qr)
{
// client wants to interrupt
break;
}
}
}
}
#else
//printf("send_whole_file: %s\n", fname);
if(!ret) while((s=fread(buf, 1, 4096, fp))>0)
{
*bytes+=s;
if(!MD5_Update(&md5, buf, s))
{
logp("MD5_Update() failed\n");
ret=-1;
break;
}
if(async_write(CMD_APPEND, buf, s))
{
ret=-1;
break;
}
if(quick_read)
{
int qr;
if((qr=do_quick_read(datapth, cntr))<0)
{
ret=-1;
break;
}
if(qr)
{
// client wants to interrupt
break;
}
}
}
#endif
}
if(!ret)
{
unsigned char checksum[MD5_DIGEST_LENGTH+1];
if(!MD5_Final(checksum, &md5))
{
logp("MD5_Final() failed\n");
return -1;
}
return write_endfile(*bytes, checksum);
}
return ret;
}
