Exemplo n.º 1
0
Arquivo: browse.c Projeto: Shloub/burp
static int browse_manifest_start(struct asfd *srfd, struct cstat *cstat,
	struct bu *bu, const char *browse, struct conf **confs)
{
	int ret=-1;
	char *manifest=NULL;
	struct sbuf *sb=NULL;
	struct manio *manio=NULL;

	if(!(manifest=prepend_s(bu->path,
		cstat->protocol==PROTO_1?"manifest.gz":"manifest"))
	  || !(manio=manio_alloc())
	  || manio_init_read(manio, manifest)
	  || !(sb=sbuf_alloc_protocol(cstat->protocol)))
		goto end;
	manio_set_protocol(manio, cstat->protocol);
	if(get_int(confs[OPT_MONITOR_BROWSE_CACHE]))
		ret=cache_load(srfd, manio, sb, cstat, bu);
	else
		ret=do_browse_manifest(srfd, manio, sb, browse);
end:
	free_w(&manifest);
	manio_free(&manio);
	sbuf_free(&sb);
	return ret;
}
Exemplo n.º 2
0
// This is basically backup_phase3_server() from protocol1. It used to merge the
// unchanged and changed data into a single file. Now it splits the manifests
// into several files.
int backup_phase3_server_protocol2(struct sdirs *sdirs, struct conf **confs)
{
	int ret=1;
	int pcmp=0;
	char *hooksdir=NULL;
	char *dindexdir=NULL;
	char *manifesttmp=NULL;
	struct sbuf *usb=NULL;
	struct sbuf *csb=NULL;
	struct blk *blk=NULL;
	int finished_ch=0;
	int finished_un=0;
	struct manio *newmanio=NULL;
	struct manio *chmanio=NULL;
	struct manio *unmanio=NULL;
	uint64_t fcount=0;

	logp("Start phase3\n");

	if(!(manifesttmp=get_tmp_filename(sdirs->rmanifest))
	  || !(newmanio=manio_alloc())
	  || !(chmanio=manio_alloc())
	  || !(unmanio=manio_alloc())
	  || !(hooksdir=prepend_s(manifesttmp, "hooks"))
	  || !(dindexdir=prepend_s(manifesttmp, "dindex"))
	  || manio_init_write(newmanio, manifesttmp)
	  || manio_init_write_hooks(newmanio,
		get_string(confs[OPT_DIRECTORY]), hooksdir, sdirs->rmanifest)
	  || manio_init_write_dindex(newmanio, dindexdir)
	  || manio_init_read(chmanio, sdirs->changed)
	  || manio_init_read(unmanio, sdirs->unchanged)
	  || !(usb=sbuf_alloc(confs))
	  || !(csb=sbuf_alloc(confs)))
		goto end;

	while(!finished_ch || !finished_un)
	{
		if(!blk && !(blk=blk_alloc())) goto end;

		if(!finished_un
		  && usb
		  && !usb->path.buf)
		{
			switch(manio_sbuf_fill(unmanio, NULL /* no async */,
				usb, NULL, NULL, confs))
			{
				case -1: goto end;
				case 1: finished_un++;
			}
		}

		if(!finished_ch
		  && csb
		  && !csb->path.buf)
		{
			switch(manio_sbuf_fill(chmanio, NULL /* no async */,
				csb, NULL, NULL, confs))
			{
				case -1: goto end;
				case 1: finished_ch++;
			}
		}

		if((usb && usb->path.buf) && (!csb || !csb->path.buf))
		{
			switch(manio_copy_entry(NULL /* no async */,
				&usb, usb,
				&blk, unmanio, newmanio, confs))
			{
				case -1: goto end;
				case 1: finished_un++;
			}
		}
		else if((!usb || !usb->path.buf) && (csb && csb->path.buf))
		{
			switch(manio_copy_entry(NULL /* no async */,
				&csb, csb, &blk, chmanio, newmanio, confs))
			{
				case -1: goto end;
				case 1: finished_ch++;
			}
		}
		else if((!usb || !usb->path.buf) && (!csb || !(csb->path.buf)))
		{
			continue;
		}
		else if(!(pcmp=sbuf_pathcmp(usb, csb)))
		{
			// They were the same - write one.
			switch(manio_copy_entry(NULL /* no async */,
				&csb, csb, &blk, chmanio, newmanio, confs))
			{
				case -1: goto end;
				case 1: finished_ch++;
			}
		}
		else if(pcmp<0)
		{
			switch(manio_copy_entry(NULL /* no async */,
				&usb, usb, &blk, unmanio, newmanio, confs))
			{
				case -1: goto end;
				case 1: finished_un++;
			}
		}
		else
		{
			switch(manio_copy_entry(NULL /* no async */,
				&csb, csb, &blk, chmanio, newmanio, confs))
			{
				case -1: goto end;
				case 1: finished_ch++;
			}
		}
	}

	fcount=newmanio->fcount;

	// Flush to disk and set up for reading.
	if(manio_free(&newmanio)
	  || !(newmanio=manio_alloc())
	  || manio_init_read(newmanio, sdirs->rmanifest))
		goto end;

	// Rename race condition should be of no consequence here, as the
	// manifest should just get recreated automatically.
	if(do_rename(manifesttmp, sdirs->rmanifest))
		goto end;
	else
	{
		recursive_delete(sdirs->changed, NULL, 1);
		recursive_delete(sdirs->unchanged, NULL, 1);
	}

	if(sparse_generation(newmanio, fcount, sdirs, confs))
		goto end;

	ret=0;

	logp("End phase3\n");
end:
	manio_free(&newmanio);
	manio_free(&chmanio);
	manio_free(&unmanio);
	sbuf_free(&csb);
	sbuf_free(&usb);
	blk_free(&blk);
	free_w(&hooksdir);
	free_w(&dindexdir);
	free_w(&manifesttmp);
	return ret;
}
Exemplo n.º 3
0
int backup_phase2_server(struct async *as, struct sdirs *sdirs,
	const char *manifest_dir, int resume, struct conf *conf)
{
	int ret=-1;
	int sigs_end=0;
	int backup_end=0;
	int requests_end=0;
	int blk_requests_end=0;
	struct slist *slist=NULL;
	struct blist *blist=NULL;
	struct iobuf *wbuf=NULL;
	struct dpth *dpth=NULL;
	struct manio *cmanio=NULL;	// current manifest
	struct manio *p1manio=NULL;	// phase1 scan manifest
	struct manio *chmanio=NULL;	// changed manifest
	struct manio *unmanio=NULL;	// unchanged manifest
	// This is used to tell the client that a number of consecutive blocks
	// have been found and can be freed.
	uint64_t wrap_up=0;
	// Main fd is first in the list.
	struct asfd *asfd=as->asfd;
	// Champ chooser fd is second in the list.
	struct asfd *chfd=asfd->next;

	logp("Phase 2 begin (recv backup data)\n");

	//if(champ_chooser_init(sdirs->data, conf)
	if(!(cmanio=manio_alloc())
	  || !(p1manio=manio_alloc())
	  || !(chmanio=manio_alloc())
	  || !(unmanio=manio_alloc())
	  || manio_init_read(cmanio, sdirs->cmanifest)
	  || manio_init_read(p1manio, sdirs->phase1data)
	  || manio_init_write(chmanio, sdirs->changed)
	  || manio_init_write(unmanio, sdirs->unchanged)
	  || !(slist=slist_alloc())
	  || !(blist=blist_alloc())
	  || !(wbuf=iobuf_alloc())
	  || !(dpth=dpth_alloc(sdirs->data))
	  || dpth_init(dpth))
		goto end;

	// The phase1 manifest looks the same as a burp1 one.
	manio_set_protocol(p1manio, PROTO_BURP1);

	while(!backup_end)
	{
		if(maybe_add_from_scan(asfd,
			p1manio, cmanio, unmanio, slist, conf))
				goto end;

		if(!wbuf->len)
		{
			if(get_wbuf_from_sigs(wbuf, slist, blist,
			  sigs_end, &blk_requests_end, dpth, conf))
				goto end;
			if(!wbuf->len)
			{
				get_wbuf_from_files(wbuf, slist,
					p1manio, &requests_end);
			}
		}

		if(wbuf->len)
			asfd->append_all_to_write_buffer(asfd, wbuf);

		append_for_champ_chooser(chfd, blist, sigs_end);

		if(as->read_write(as))
		{
			logp("error in %s\n", __func__);
			goto end;
		}

		while(asfd->rbuf->buf)
		{
			if(deal_with_read(asfd->rbuf, slist, blist,
				conf, &sigs_end, &backup_end, dpth))
					goto end;
			// Get as much out of the
			// readbuf as possible.
			if(asfd->parse_readbuf(asfd)) goto end;
		}
		while(chfd->rbuf->buf)
		{
			if(deal_with_read_from_chfd(asfd, chfd,
				blist, &wrap_up, dpth)) goto end;
			// Get as much out of the
			// readbuf as possible.
			if(chfd->parse_readbuf(chfd)) goto end;
		}

		if(write_to_changed_file(asfd, chfd, chmanio,
			slist, blist, dpth, backup_end, conf))
				goto end;
	}

	// Hack: If there are some entries left after the last entry that
	// contains block data, it will not be written to the changed file
	// yet because the last entry of block data has not had
	// sb->burp2->bend set.
	if(slist->head && slist->head->next)
	{
		slist->head=slist->head->next;
		if(write_to_changed_file(asfd, chfd, chmanio,
			slist, blist, dpth, backup_end, conf))
				goto end;
	}

	if(manio_close(unmanio)
	  || manio_close(chmanio))
		goto end;

	if(blist->head)
	{
		logp("ERROR: finishing but still want block: %lu\n",
			blist->head->index);
		goto end;
	}

	// Need to release the last left. There should be one at most.
	if(dpth->head && dpth->head->next)
	{
		logp("ERROR: More data locks remaining after: %s\n",
			dpth->head->save_path);
		goto end;
	}
	if(dpth_release_all(dpth)) goto end;

	ret=0;
end:
	logp("End backup\n");
	slist_free(slist);
	blist_free(blist);
	iobuf_free_content(asfd->rbuf);
	iobuf_free_content(chfd->rbuf);
	// Write buffer did not allocate 'buf'. 
	if(wbuf) wbuf->buf=NULL;
	iobuf_free(wbuf);
	dpth_release_all(dpth);
	dpth_free(&dpth);
	manio_free(&cmanio);
	manio_free(&p1manio);
	manio_free(&chmanio);
	manio_free(&unmanio);
	return ret;
}
Exemplo n.º 4
0
int backup_phase4_server_protocol2(struct sdirs *sdirs, struct conf **confs)
{
	int ret=-1;
	uint64_t i=0;
	uint64_t pass=0;
	char *sparse=NULL;
	char *global_sparse=NULL;
	char *h1dir=NULL;
	char *h2dir=NULL;
	char *hooksdir=NULL;
	char *srca=NULL;
	char *srcb=NULL;
	char *dst=NULL;
	char compa[32]="";
	char compb[32]="";
	char compd[32]="";
	struct manio *newmanio=NULL;
	char *logpath=NULL;
	char *fmanifest=NULL; // FIX THIS: should be part of sdirs.

	if(!(logpath=prepend_s(sdirs->finishing, "log")))
		goto end;
	if(set_logfp(logpath, confs))
		goto end;

	logp("Begin phase4 (sparse generation)\n");

	if(!(newmanio=manio_alloc())
	  || !(fmanifest=prepend_s(sdirs->finishing, "manifest"))
	  || manio_init_read(newmanio, fmanifest)
	  || manio_read_fcount(newmanio)
	  || !(hooksdir=prepend_s(fmanifest, "hooks"))
	  || !(h1dir=prepend_s(fmanifest, "h1"))
	  || !(h2dir=prepend_s(fmanifest, "h2")))
		goto end;

	while(1)
	{
		char *srcdir=NULL;
		char *dstdir=NULL;
		if(!pass)
		{
			srcdir=hooksdir;
			dstdir=h1dir;
		}
		else if(pass%2)
		{
			srcdir=h1dir;
			dstdir=h2dir;
		}
		else
		{
			srcdir=h2dir;
			dstdir=h1dir;
		}
		pass++;
		for(i=0; i<newmanio->offset.fcount; i+=2)
		{
			free_w(&srca);
			free_w(&srcb);
			free_w(&dst);
			snprintf(compa, sizeof(compa), "%08"PRIX64, i);
			snprintf(compb, sizeof(compb), "%08"PRIX64, i+1);
			snprintf(compd, sizeof(compd), "%08"PRIX64, i/2);
			if(!(srca=prepend_s(srcdir, compa))
			  || !(dst=prepend_s(dstdir, compd)))
				goto end;
			if(i+1<newmanio->offset.fcount
			  && !(srcb=prepend_s(srcdir, compb)))
				goto end;
			if(merge_sparse_indexes(srca, srcb, dst, confs))
				goto end;
		}
		if((newmanio->offset.fcount=i/2)<2) break;
	}

	if(!(sparse=prepend_s(fmanifest, "sparse"))
	  || !(global_sparse=prepend_s(sdirs->data, "sparse")))
		goto end;

	// FIX THIS: nasty race condition here needs to be automatically
	// recoverable.
	if(do_rename(dst, sparse)) goto end;

	if(merge_into_global_sparse(sparse, global_sparse, confs)) goto end;

	logp("End phase4 (sparse generation)\n");

	ret=0;
end:
	manio_free(&newmanio);
	free_w(&sparse);
	free_w(&global_sparse);
	free_w(&srca);
	free_w(&srcb);
	recursive_delete(h1dir, NULL, 1);
	recursive_delete(h2dir, NULL, 1);
	free_w(&h1dir);
	free_w(&h2dir);
	free_w(&logpath);
	free_w(&fmanifest);
	return ret;
}
Exemplo n.º 5
0
static int list_manifest(struct asfd *asfd,
	const char *fullpath, regex_t *regex,
	const char *browsedir, struct conf *conf)
{
	int ars=0;
	int ret=0;
	struct sbuf *sb=NULL;
	struct manio *manio=NULL;
	char *manifest_dir=NULL;
	char *last_bd_match=NULL;
	size_t bdlen=0;

	if(!(manifest_dir=prepend_s(fullpath,
		conf->protocol==PROTO_BURP1?"manifest.gz":"manifest"))
	  || !(manio=manio_alloc())
	  || manio_init_read(manio, manifest_dir)
	  || !(sb=sbuf_alloc(conf)))
	{
		log_and_send_oom(asfd, __func__);
		goto error;
	}
	manio_set_protocol(manio, conf->protocol);

	if(browsedir) bdlen=strlen(browsedir);

	while(1)
	{
		int show=0;

		if((ars=manio_sbuf_fill(manio, asfd, sb, NULL, NULL, conf))<0)
			goto error;
		else if(ars>0)
			goto end; // Finished OK.

		if(write_status(STATUS_LISTING, sb->path.buf, conf))
			goto error;

		if(browsedir)
		{
			int r;
			if((r=check_browsedir(browsedir,
				&sb->path.buf, bdlen, &last_bd_match))<0)
					goto error;
			if(!r) continue;
			show++;
		}
		else
		{
			if(check_regex(regex, sb->path.buf))
				show++;
		}
		if(show)
		{
			if(write_wrapper(asfd, &sb->attr)
			  || write_wrapper(asfd, &sb->path))
				goto error;
			if(sbuf_is_link(sb)
			  && write_wrapper(asfd, &sb->link))
				goto error;
		}

		sbuf_free_content(sb);
	}

error:
	ret=-1;
end:
	sbuf_free(&sb);
	free_w(&manifest_dir);
	manio_free(&manio);
	free_w(&last_bd_match);
	return ret;
}
Exemplo n.º 6
0
int backup_phase2_server_protocol1(struct async *as, struct sdirs *sdirs,
	const char *incexc, int resume, struct conf **cconfs)
{
	int ret=0;
	struct manio *p1manio=NULL;
	struct dpth *dpth=NULL;
	char *deltmppath=NULL;
	char *last_requested=NULL;
	// Where to write changed data.
	// Data is not getting written to a compressed file.
	// This is important for recovery if the power goes.
	struct fzp *chfp=NULL;
	struct fzp *ucfp=NULL; // unchanged data
	struct fzp *cmanfp=NULL; // previous (current) manifest.
	struct sbuf *cb=NULL; // file list in current manifest
	struct sbuf *p1b=NULL; // file list from client
	struct sbuf *rb=NULL; // receiving file from client
	struct asfd *asfd=as->asfd;
	int breaking=0;
	int breakcount=0;
	if(get_int(cconfs[OPT_BREAKPOINT])>=2000
	  && get_int(cconfs[OPT_BREAKPOINT])<3000)
	{
		breaking=get_int(cconfs[OPT_BREAKPOINT]);
		breakcount=breaking-2000;
	}

	logp("Begin phase2 (receive file data)\n");

	if(!(dpth=dpth_alloc())
	  || dpth_protocol1_init(dpth, sdirs->currentdata,
		get_int(cconfs[OPT_MAX_STORAGE_SUBDIRS])))
			goto error;

	if(open_previous_manifest(&cmanfp, sdirs, incexc, cconfs))
		goto error;

	if(get_int(cconfs[OPT_DIRECTORY_TREE]))
	{
		// Need to make sure we do not try to create a path that is
		// too long.
		if(build_path_w(sdirs->treepath)) goto error;
		treepathlen=strlen(sdirs->treepath);
		init_fs_max(sdirs->treepath);
	}

	if(!(p1manio=manio_alloc())
	  || manio_init_read(p1manio, sdirs->phase1data)
	  || !(cb=sbuf_alloc(cconfs))
	  || !(p1b=sbuf_alloc(cconfs))
	  || !(rb=sbuf_alloc(cconfs)))
		goto error;

	manio_set_protocol(p1manio, PROTO_1);

	if(resume && do_resume(p1manio, sdirs, dpth, cconfs))
		goto error;

	// Unchanged and changed should now be truncated correctly, we just
	// have to open them for appending.
	if(!(ucfp=fzp_open(sdirs->unchanged, "a+b"))
	  || !(chfp=fzp_open(sdirs->changed, "a+b")))
		goto error;

	if(manio_closed(p1manio)
	  && manio_open_next_fpath(p1manio))
		goto error;

	while(1)
	{
		if(breaking)
		{
			if(breakcount--==0) return breakpoint(cconfs, __func__);
		}

		//printf("in loop, %s %s %c\n",
		//	cmanfp?"got cmanfp":"no cmanfp",
		//	rb->path.buf?:"no rb->path",
	 	//	rb->path.buf?'X':rb->path.cmd);
		if(write_status(CNTR_STATUS_BACKUP,
			rb->path.buf?rb->path.buf:p1b->path.buf, cconfs))
				goto error;
		if(last_requested
		  || manio_closed(p1manio)
		  || asfd->writebuflen)
		{
			switch(do_stuff_to_receive(asfd, sdirs,
				cconfs, rb, chfp, dpth, &last_requested))
			{
				case 0: break;
				case 1: goto end; // Finished ok.
				case -1: goto error;
			}
		}

		switch(do_stuff_to_send(asfd, p1b, &last_requested))
		{
			case 0: break;
			case 1: continue;
			case -1: goto error;
		}

		if(manio_closed(p1manio)) continue;

		sbuf_free_content(p1b);

		switch(manio_sbuf_fill_phase1(p1manio, asfd,
			p1b, NULL, sdirs, cconfs))
		{
			case 0: break;
			case 1: manio_close(p1manio);
				if(asfd->write_str(asfd,
				  CMD_GEN, "backupphase2end")) goto error;
				break;
			case -1: goto error;
		}

		if(!cmanfp)
		{
			// No old manifest, need to ask for a new file.
			if(process_new(sdirs, cconfs, p1b, ucfp))
				goto error;
			continue;
		}

		// Have an old manifest, look for it there.

		// Might already have it, or be ahead in the old
		// manifest.
		if(cb->path.buf) switch(maybe_process_file(asfd,
			sdirs, cb, p1b, ucfp, cconfs))
		{
			case 0: break;
			case 1: continue;
			case -1: goto error;
		}

		while(cmanfp)
		{
			sbuf_free_content(cb);
			switch(sbufl_fill(cb, asfd, cmanfp, cconfs))
			{
				case 0: break;
				case 1: fzp_close(&cmanfp);
					if(process_new(sdirs, cconfs, p1b,
						ucfp)) goto error;
					continue;
				case -1: goto error;
			}
			switch(maybe_process_file(asfd, sdirs,
				cb, p1b, ucfp, cconfs))
			{
				case 0: continue;
				case 1: break;
				case -1: goto error;
			}
			break;
		}
	}

error:
	ret=-1;
end:
	if(fzp_close(&chfp))
	{
		logp("error closing %s in %s\n", sdirs->changed, __func__);
		ret=-1;
	}
	if(fzp_close(&ucfp))
	{
		logp("error closing %s in %s\n", sdirs->unchanged, __func__);
		ret=-1;
	}
	free_w(&deltmppath);
	sbuf_free(&cb);
	sbuf_free(&p1b);
	sbuf_free(&rb);
	manio_free(&p1manio);
	fzp_close(&cmanfp);
	dpth_free(&dpth);
	if(!ret) unlink(sdirs->phase1data);

	logp("End phase2 (receive file data)\n");

	return ret;
}