Beispiel #1
0
static int open_log(struct asfd *asfd,
	struct sdirs *sdirs, struct conf **cconfs)
{
	int ret=-1;
	char *logpath=NULL;
	const char *peer_version=get_string(cconfs[OPT_PEER_VERSION]);

	if(!(logpath=prepend_s(sdirs->rworking, "log"))) goto end;
	if(log_fzp_set(logpath, cconfs))
	{
		logp("could not open log file: %s\n", logpath);
		goto end;
	}

	logp("Client version: %s\n", peer_version?:"");
	logp("Protocol: %d\n", (int)get_protocol(cconfs));
	if(get_int(cconfs[OPT_CLIENT_IS_WINDOWS]))
		logp("Client is Windows\n");

	// Make sure a warning appears in the backup log.
	// The client will already have been sent a message with logw.
	// This time, prevent it sending a logw to the client by specifying
	// NULL for cntr.
	if(get_int(cconfs[OPT_VERSION_WARN])) version_warn(asfd, NULL, cconfs);

	ret=0;
end:
	free_w(&logpath);
	return ret;
}
Beispiel #2
0
int reload(struct conf **confs, const char *conffile, bool firsttime)
{
	if(!firsttime) logp("Reloading config\n");

	if(confs_init(confs)) return -1;

	if(conf_load_global_only(conffile, confs)) return -1;

	umask(get_mode_t(confs[OPT_UMASK]));

	// This will turn on syslogging which could not be turned on before
	// conf_load.
	log_fzp_set(NULL, confs);

#ifndef HAVE_WIN32
	if(get_e_burp_mode(confs[OPT_BURP_MODE])==BURP_MODE_SERVER)
		setup_signals();
#endif

	// Do not try to change user or group after the first time.
	if(firsttime && chuser_and_or_chgrp(
		get_string(confs[OPT_USER]), get_string(confs[OPT_GROUP])))
			return -1;

	return 0;
}
Beispiel #3
0
static int backup_phase4_server(struct sdirs *sdirs, struct conf **cconfs)
{
	int breaking=get_int(cconfs[OPT_BREAKPOINT]);
	if(breaking==4)
		return breakpoint(breaking, __func__);

	log_fzp_set(NULL, cconfs);
	// Phase4 will open logfp again (in case it is resuming).
	switch(get_protocol(cconfs))
	{
		case PROTO_1:
			return backup_phase4_server_protocol1(sdirs, cconfs);
		default:
			return backup_phase4_server_protocol2(sdirs, cconfs);
	}
}
Beispiel #4
0
static int champ_chooser_fork(struct sdirs *sdirs, struct conf **confs,
	int resume)
{
	pid_t childpid=-1;
	int cret;

	if(!get_int(confs[OPT_FORK]))
	{
		logp("Not forking a champ chooser process.\n");
		// They need to manually run a separate process.
		return 0;
	}

	switch((childpid=fork()))
	{
		case -1:
			logp("fork failed in %s: %s\n",
				__func__, strerror(errno));
			return -1;
		case 0:
			// Child.
			log_fzp_set(NULL, confs);
			switch(champ_chooser_server(sdirs, confs, resume))
			{
				case 0:
					cret=0;
					break;
				default:
					cret=1;
					break;
			}
			exit(cret);
		default:
			// Parent.
			logp("forked champ chooser pid %d\n", childpid);
			return 0;
	}
	return -1; // Not reached.
}
Beispiel #5
0
static void run_test(
	int expected_result,
	int entries,
	void setup_datadir_tmp_callback(
		struct slist *slist, struct fdirs *fdirs, struct conf **confs))
{
	struct conf **confs;
	struct sdirs *sdirs;
	struct fdirs *fdirs;
	struct slist *slist;

	setup(&sdirs, &fdirs, &confs);

	build_storage_dirs(sdirs, sd1, ARR_LEN(sd1));
	slist=build_manifest(
		fdirs->manifest,
		PROTO_1,
		entries,
		/*phase*/ 3);

	setup_datadir_tmp_callback(slist, fdirs, confs);

clock_t start;
clock_t diff;
start = clock();
	fail_unless(backup_phase4_server_protocol1(sdirs, confs)
		==expected_result);
diff = clock() - start;
int msec = diff * 1000 / CLOCKS_PER_SEC;
printf("%d.%d\n", msec/1000, msec%1000);

	log_fzp_set(NULL, confs);

	assert_datadir(slist, fdirs);

	slist_free(&slist);
	tear_down(&sdirs, &fdirs, &confs);
}
Beispiel #6
0
static int do_backup_server(struct async *as, struct sdirs *sdirs,
	struct conf **cconfs, const char *incexc, int resume)
{
	int ret=0;
	int do_phase2=1;
	struct asfd *asfd=as->asfd;
	enum protocol protocol=get_protocol(cconfs);
	struct cntr *cntr=get_cntr(cconfs);

	logp("in do_backup_server\n");

	log_rshash(cconfs);

	if(resume)
	{
		if(sdirs_get_real_working_from_symlink(sdirs)
		  || sdirs_get_real_manifest(sdirs, protocol)
		  || open_log(asfd, sdirs, cconfs))
			goto error;
	}
	else
	{
		// Not resuming - need to set everything up fresh.
		if(sdirs_create_real_working(sdirs,
			get_string(cconfs[OPT_TIMESTAMP_FORMAT]))
		  || sdirs_get_real_manifest(sdirs, protocol)
		  || open_log(asfd, sdirs, cconfs))
			goto error;

		if(write_incexc(sdirs->rworking, incexc))
		{
			logp("unable to write incexc\n");
			goto error;
		}

		if(backup_phase1_server(as, sdirs, cconfs))
		{
			logp("error in phase 1\n");
			goto error;
		}
	}

	if(resume)
	{
		struct stat statp;
		if(lstat(sdirs->phase1data, &statp)
		  && !lstat(sdirs->changed, &statp)
		  && !lstat(sdirs->unchanged, &statp))
		{
			// In this condition, it looks like there was an
			// interruption during phase3. Skip phase2.
			do_phase2=0;
		}
	}

	if(do_phase2)
	{
		if(backup_phase2_server(as, sdirs, incexc, resume, cconfs))
		{
			logp("error in backup phase 2\n");
			goto error;
		}

		asfd->write_str(asfd, CMD_GEN, "okbackupend");
	}

	// Close the connection with the client, the rest of the job we can do
	// by ourselves.
	logp("Backup ending - disconnect from client.\n");
	if(asfd_flush_asio(asfd)) goto end;
	as->asfd_remove(as, asfd);
	asfd_close(asfd);

	if(backup_phase3_server(sdirs, cconfs))
	{
		logp("error in backup phase 3\n");
		goto error;
	}

	if(do_rename(sdirs->working, sdirs->finishing))
		goto error;

	if(backup_phase4_server(sdirs, cconfs))
	{
		logp("error in backup phase 4\n");
		goto error;
	}

	cntr_print(cntr, ACTION_BACKUP, asfd);
	cntr_stats_to_file(cntr, sdirs->rworking, ACTION_BACKUP);

	if(protocol==PROTO_2)
	{
		// Regenerate dindex before the symlink is renamed, so that the
		// champ chooser cleanup does not try to remove data files
		// whilst the dindex regeneration is happening.
		if(regenerate_client_dindex(sdirs))
			goto error;
	}

	// Move the symlink to indicate that we are now in the end phase. The
	// rename() race condition is automatically recoverable here.
	if(do_rename(sdirs->finishing, sdirs->current))
		goto error;

	logp("Backup completed.\n");
	log_fzp_set(NULL, cconfs);
	compress_filename(sdirs->rworking,
		"log", "log.gz", get_int(cconfs[OPT_COMPRESSION]));

	goto end;
error:
	ret=-1;
end:

	log_fzp_set(NULL, cconfs);
	return ret;
}
Beispiel #7
0
static int recover_working(struct async *as,
	struct sdirs *sdirs, const char *incexc,
	int *resume, struct conf **cconfs)
{
	int ret=-1;
	char msg[256]="";
	char *logpath=NULL;
	struct stat statp;
	char *phase1datatmp=NULL;
	enum recovery_method recovery_method=get_e_recovery_method(
		cconfs[OPT_WORKING_DIR_RECOVERY_METHOD]);

	// The working directory has not finished being populated.
	// Check what to do.
	if(get_fullrealwork(as->asfd, sdirs, cconfs)) goto end;
	if(!sdirs->rworking) goto end;

	log_recovery_method(sdirs, recovery_method);

	if(!(phase1datatmp=get_tmp_filename(sdirs->phase1data)))
		goto end;
	// If there is still a phase1 tmp file...
	if(!lstat(phase1datatmp, &statp)
	  ||
		// ...or phase1 has not even got underway yet...
		(lstat(phase1datatmp, &statp)
		  && lstat(sdirs->phase1data, &statp)
		  && lstat(sdirs->changed, &statp)
		  && lstat(sdirs->unchanged, &statp)))
	{
		// ...phase 1 did not complete - delete everything.
		logp("Phase 1 has not completed.\n");
		recovery_method=RECOVERY_METHOD_DELETE;
	}

	if(recovery_method==RECOVERY_METHOD_DELETE)
	{
		ret=working_delete(as, sdirs, cconfs);
		goto end;
	}

	// We are not deleting the old working directory - open the log inside
	// for appending.
	if(!(logpath=prepend_s(sdirs->rworking, "log"))
	  || log_fzp_set(logpath, cconfs))
		goto end;

	switch(recovery_method)
	{
		case RECOVERY_METHOD_DELETE:
			// Dealt with above.
			break;
		case RECOVERY_METHOD_RESUME:
			ret=working_resume(as, sdirs, incexc, resume, cconfs);
			break;
		case RECOVERY_METHOD_UNSET:
		default:
			snprintf(msg, sizeof(msg),
				"Unknown working_dir_recovery_method: %d\n",
					(int)recovery_method);
			log_and_send(as->asfd, msg);
			break;
	}

end:
	free_w(&logpath);
	free_w(&phase1datatmp);
	log_fzp_set(NULL, cconfs); // fclose the logfzp
	return ret;
}
Beispiel #8
0
int champ_chooser_server(struct sdirs *sdirs, struct conf **confs,
	int resume)
{
	int s;
	int ret=-1;
	int len;
	struct asfd *asfd=NULL;
	struct sockaddr_un local;
	struct lock *lock=NULL;
	struct async *as=NULL;
	int started=0;
	struct scores *scores=NULL;
	const char *directory=get_string(confs[OPT_DIRECTORY]);

	if(!(lock=lock_alloc_and_init(sdirs->champlock))
	  || build_path_w(sdirs->champlock))
		goto end;
	lock_get(lock);
	switch(lock->status)
	{
		case GET_LOCK_GOT:
			log_fzp_set(sdirs->champlog, confs);
			logp("Got champ lock for dedup_group: %s\n",
				get_string(confs[OPT_DEDUP_GROUP]));
			break;
		case GET_LOCK_NOT_GOT:
		case GET_LOCK_ERROR:
		default:
			//logp("Did not get champ lock\n");
			goto end;
	}

	if((s=socket(AF_UNIX, SOCK_STREAM, 0))<0)
	{
		logp("socket error in %s: %s\n", __func__, strerror(errno));
		goto end;
	}

	memset(&local, 0, sizeof(struct sockaddr_un));
	local.sun_family=AF_UNIX;
	snprintf(local.sun_path, sizeof(local.sun_path),
		"%s", sdirs->champsock);
	len=strlen(local.sun_path)+sizeof(local.sun_family)+1;
	unlink(sdirs->champsock);
	if(bind(s, (struct sockaddr *)&local, len)<0)
	{
		logp("bind error in %s: %s\n", __func__, strerror(errno));
		goto end;
	}

	if(listen(s, 5)<0)
	{
		logp("listen error in %s: %s\n", __func__, strerror(errno));
		goto end;
	}

	if(!(as=async_alloc())
	  || as->init(as, 0)
	  || !(asfd=setup_asfd(as, "champ chooser main socket", &s,
		/*listen*/"")))
			goto end;
	asfd->fdtype=ASFD_FD_SERVER_LISTEN_MAIN;

	// I think that this is probably the best point at which to run a
	// cleanup job to delete unused data files, because no other process
	// can fiddle with the dedup_group at this point.
	// Cannot do it on a resume, or it will delete files that are
	// referenced in the backup we are resuming.
	if(delete_unused_data_files(sdirs, resume))
		goto end;

	// Load the sparse indexes for this dedup group.
	if(!(scores=champ_chooser_init(sdirs->data)))
		goto end;

	while(1)
	{
		for(asfd=as->asfd->next; asfd; asfd=asfd->next)
		{
			if(!asfd->blist->head
			  || asfd->blist->head->got==BLK_INCOMING) continue;
			if(results_to_fd(asfd)) goto end;
		}

		int removed;

		switch(as->read_write(as))
		{
			case 0:
				// Check the main socket last, as it might add
				// a new client to the list.
				for(asfd=as->asfd->next; asfd; asfd=asfd->next)
				{
					while(asfd->rbuf->buf)
					{
						if(deal_with_client_rbuf(asfd,
							directory, scores))
								goto end;
						// Get as much out of the
						// readbuf as possible.
						if(asfd->parse_readbuf(asfd))
							goto end;
					}
				}
				if(as->asfd->new_client)
				{
					// Incoming client.
					as->asfd->new_client=0;
					if(champ_chooser_new_client(as, confs))
						goto end;
					started=1;
				}
				break;
			default:
				removed=0;
				// Maybe one of the fds had a problem.
				// Find and remove it and carry on if possible.
				for(asfd=as->asfd->next; asfd; )
				{
					struct asfd *a;
					if(!asfd->want_to_remove)
					{
						asfd=asfd->next;
						continue;
					}
					as->asfd_remove(as, asfd);
					logp("%s: disconnected fd %d\n",
						asfd->desc, asfd->fd);
					a=asfd->next;
					asfd_free(&asfd);
					asfd=a;
					removed++;
				}
				if(removed) break;
				// If we got here, there was no fd to remove.
				// It is a fatal error.
				goto end;
		}
				
		if(started && !as->asfd->next)
		{
			logp("All clients disconnected.\n");
			ret=0;
			break;
		}
	}

end:
	logp("champ chooser exiting: %d\n", ret);
	champ_chooser_free(&scores);
	log_fzp_set(NULL, confs);
	async_free(&as);
	asfd_free(&asfd); // This closes s for us.
	close_fd(&s);
	unlink(sdirs->champsock);
// FIX THIS: free asfds.
	lock_release(lock);
	lock_free(&lock);
	return ret;
}
Beispiel #9
0
int backup_phase4_server_protocol1(struct sdirs *sdirs, struct conf **cconfs)
{
	int ret=-1;
	struct stat statp;
	char realcurrent[256]="";
	uint64_t bno=0;
	int hardlinked_current=0;
	char tstmp[64]="";
	int previous_backup=0;
	struct fdirs *fdirs=NULL;

	readlink_w(sdirs->current, realcurrent, sizeof(realcurrent));

	if(!(fdirs=fdirs_alloc())
	  || fdirs_init(fdirs, sdirs, realcurrent))
		goto end;

	if(log_fzp_set(fdirs->logpath, cconfs))
		goto end;

	logp("Begin phase4 (shuffle files)\n");

	if(write_status(CNTR_STATUS_SHUFFLING, NULL, get_cntr(cconfs)))
		goto end;

	if(!lstat(sdirs->current, &statp)) // Had a previous backup.
	{
		previous_backup++;

		if(lstat(fdirs->hlinkedcurrent, &statp))
		{
			hardlinked_current=0;
			logp("Previous backup is not a hardlinked_archive\n");
			logp(" will generate reverse deltas\n");
		}
		else
		{
			hardlinked_current=1;
			logp("Previous backup is a hardlinked_archive\n");
			logp(" will not generate reverse deltas\n");
		}

		// If current was not a hardlinked_archive, need to duplicate
		// it.
		if(!hardlinked_current && lstat(fdirs->currentdup, &statp))
		{
			// Have not duplicated the current backup yet.
			if(!lstat(fdirs->currentduptmp, &statp))
			{
				logp("Removing previous directory: %s\n",
					fdirs->currentduptmp);
				if(recursive_delete(fdirs->currentduptmp))
				{
					logp("Could not delete %s\n",
						fdirs->currentduptmp);
					goto end;
				}
			}
			logp("Duplicating current backup.\n");
			if(recursive_hardlink(sdirs->current,
				fdirs->currentduptmp, cconfs)
			// The rename race condition is of no consequence here
			// because currentdup does not exist.
			  || do_rename(fdirs->currentduptmp, fdirs->currentdup))
				goto end;
		}
	}

	if(timestamp_read(fdirs->timestamp, tstmp, sizeof(tstmp)))
	{
		logp("could not read timestamp file: %s\n",
			fdirs->timestamp);
		goto end;
	}
	// Get the backup number.
	bno=strtoull(tstmp, NULL, 10);

	// Determine whether the new backup should be a hardlinked
	// archive or not, from the confs and the backup number...
	if(need_hardlinked_archive(cconfs, bno))
	{
		// Create a file to indicate that the previous backup
		// does not have others depending on it.
		struct fzp *hfp=NULL;
		if(!(hfp=fzp_open(fdirs->hlinked, "wb"))) goto end;

		// Stick the next backup timestamp in it. It might
		// be useful one day when wondering when the next
		// backup, now deleted, was made.
		fzp_printf(hfp, "%s\n", tstmp);
		if(fzp_close(&hfp))
		{
			logp("error closing hardlinked indication\n");
			goto end;
		}
	}
	else
		unlink(fdirs->hlinked);

	if(atomic_data_jiggle(sdirs, fdirs, hardlinked_current, cconfs))
	{
		logp("could not finish up backup.\n");
		goto end;
	}

	if(write_status(CNTR_STATUS_SHUFFLING,
		"deleting temporary files", get_cntr(cconfs)))
			goto end;

	// Remove the temporary data directory, we have now removed
	// everything useful from it.
	recursive_delete(fdirs->datadirtmp);

	// Clean up the currentdata directory - this is now the 'old'
	// currentdata directory. Any files that were deleted from
	// the client will be left in there, so call recursive_delete
	// with the option that makes it not delete files.
	// This will have the effect of getting rid of unnecessary
	// directories.
	recursive_delete_dirs_only(fdirs->currentdupdata);

	// Rename the old current to something that we know to delete.
	if(previous_backup && !hardlinked_current)
	{
		if(deleteme_move(sdirs,
			fdirs->fullrealcurrent, realcurrent, cconfs)
		// I have tested that potential race conditions on the
		// rename() are automatically recoverable here.
		  || do_rename(fdirs->currentdup, fdirs->fullrealcurrent))
			goto end;
	}

	if(deleteme_maybe_delete(cconfs, sdirs))
		goto end;

	logp("End phase4 (shuffle files)\n");

	ret=0;
end:
	fdirs_free(&fdirs);
	return ret;
}
Beispiel #10
0
static int restore_manifest(struct asfd *asfd, struct bu *bu,
	regex_t *regex, int srestore, enum action act, struct sdirs *sdirs,
	char **dir_for_notify, struct conf **cconfs)
{
	int ret=-1;
	char *manifest=NULL;
	char *logpath=NULL;
	char *logpathz=NULL;
	// For sending status information up to the server.
	enum cntr_status cntr_status=CNTR_STATUS_RESTORING;

	if(act==ACTION_RESTORE) cntr_status=CNTR_STATUS_RESTORING;
	else if(act==ACTION_VERIFY) cntr_status=CNTR_STATUS_VERIFYING;

	if((act==ACTION_RESTORE && get_logpaths(bu, "restorelog",
		&logpath, &logpathz))
	  || (act==ACTION_VERIFY && get_logpaths(bu, "verifylog",
		&logpath, &logpathz))
	  || !(manifest=prepend_s(bu->path,
		get_protocol(cconfs)==PROTO_1?
			"manifest.gz":"manifest")))
	{
		log_and_send_oom(asfd, __func__);
		goto end;
	}

	if(log_fzp_set(logpath, cconfs))
	{
		char msg[256]="";
		snprintf(msg, sizeof(msg),
				"could not open log file: %s", logpath);
		log_and_send(asfd, msg);
		goto end;
	}

	*dir_for_notify=strdup_w(bu->path, __func__);

	log_restore_settings(cconfs, srestore);

	// First, do a pass through the manifest to set up cntr.
	// This is the equivalent of a phase1 scan during backup.

	if(setup_cntr(asfd, manifest,
		regex, srestore, act, cntr_status, cconfs))
			goto end;

	if(get_int(cconfs[OPT_SEND_CLIENT_CNTR])
	  && cntr_send(get_cntr(cconfs)))
		goto end;

	// Now, do the actual restore.
	ret=actual_restore(asfd, bu, manifest,
		  regex, srestore, act, sdirs, cntr_status, cconfs);
end:
	log_fzp_set(NULL, cconfs);
	compress_file(logpath, logpathz, get_int(cconfs[OPT_COMPRESSION]));
	if(manifest) free(manifest);
	if(logpath) free(logpath);
	if(logpathz) free(logpathz);
	return ret;
}