static int do_lock_dirs(struct sdirs *sdirs, struct conf *conf) { int ret=-1; char *lockbase=NULL; char *lockfile=NULL; if(conf->client_lockdir) { if(!(sdirs->lockdir=strdup_w(conf->client_lockdir, __func__)) || !(lockbase=prepend_s(sdirs->lockdir, conf->cname))) goto end; } else { if(!(sdirs->lockdir=strdup_w(sdirs->client, __func__)) || !(lockbase=strdup_w(sdirs->client, __func__))) goto end; } if(!(lockfile=prepend_s(lockbase, "lockfile")) || !(sdirs->lock=lock_alloc_and_init(lockfile))) goto end; ret=0; end: free_w(&lockbase); free_w(&lockfile); return ret; }
END_TEST static void init_and_add_to_list(struct lock **locklist, const char *path) { struct lock *lock; fail_unless((lock=lock_alloc_and_init(path))!=NULL); lock_add_to_list(locklist, lock); }
static void do_fork(int child_exit_early) { switch(fork()) { case -1: fail_unless(0==1); break; case 0: // Child. { struct lock *lock; lock=lock_alloc_and_init(lockfile); lock_get_quick(lock); if(!child_exit_early) { sleep(2); lock_release(lock); lock_free(&lock); } exit(0); } default: break; } // Parent. }
static int merge_into_global_sparse(const char *sparse, const char *global, struct conf **confs) { int ret=-1; char *tmpfile=NULL; struct stat statp; char *lockfile=NULL; struct lock *lock=NULL; const char *globalsrc=NULL; if(!(tmpfile=prepend_n(global, "tmp", strlen("tmp"), "."))) goto end; // Get a lock before messing with the global sparse index. if(!(lockfile=prepend_n(global, "lock", strlen("lock"), ".")) || !(lock=lock_alloc_and_init(lockfile))) goto end; if(try_to_get_lock(lock)) goto end; if(!lstat(global, &statp)) globalsrc=global; if(merge_sparse_indexes(sparse, globalsrc, tmpfile, confs)) goto end; // FIX THIS: nasty race condition needs to be recoverable. if(do_rename(tmpfile, global)) goto end; ret=0; end: lock_release(lock); lock_free(&lock); if(lockfile) free(lockfile); if(tmpfile) free(tmpfile); return ret; }
int run_bedup(int argc, char *argv[]) { int i=1; int ret=0; int option=0; int nonburp=0; unsigned int maxlinks=DEF_MAX_LINKS; char *groups=NULL; char ext[16]=""; int givenconfigfile=0; const char *configfile=NULL; configfile=get_config_path(); snprintf(ext, sizeof(ext), ".bedup.%d", getpid()); while((option=getopt(argc, argv, "c:dg:hlm:nvV?"))!=-1) { switch(option) { case 'c': configfile=optarg; givenconfigfile=1; break; case 'd': deletedups=1; break; case 'g': groups=optarg; break; case 'l': makelinks=1; break; case 'm': maxlinks=atoi(optarg); break; case 'n': nonburp=1; break; case 'V': printf("%s-%s\n", prog, VERSION); return 0; case 'v': verbose=1; break; case 'h': case '?': return usage(); } } if(nonburp && givenconfigfile) { logp("-n and -c options are mutually exclusive\n"); return 1; } if(nonburp && groups) { logp("-n and -g options are mutually exclusive\n"); return 1; } if(!nonburp && maxlinks!=DEF_MAX_LINKS) { logp("-m option is specified via the configuration file in burp mode (max_hardlinks=)\n"); return 1; } if(deletedups && makelinks) { logp("-d and -l options are mutually exclusive\n"); return 1; } if(deletedups && !nonburp) { logp("-d option requires -n option\n"); return 1; } if(optind>=argc) { if(nonburp) { logp("No directories found after options\n"); return 1; } } else { if(!nonburp) { logp("Do not specify extra arguments.\n"); return 1; } } if(maxlinks<2) { logp("The argument to -m needs to be greater than 1.\n"); return 1; } if(nonburp) { // Read directories from command line. for(i=optind; i<argc; i++) { // Strip trailing slashes, for tidiness. if(argv[i][strlen(argv[i])-1]=='/') argv[i][strlen(argv[i])-1]='\0'; if(process_dir("", argv[i], ext, maxlinks, 0 /* not burp mode */, 0 /* level */)) { ret=1; break; } } } else { struct conf **globalcs=NULL; struct strlist *grouplist=NULL; struct lock *globallock=NULL; if(groups) { char *tok=NULL; if((tok=strtok(groups, ",\n"))) { do { if(strlist_add(&grouplist, tok, 1)) { log_out_of_memory(__func__); return -1; } } while((tok=strtok(NULL, ",\n"))); } if(!grouplist) { logp("unable to read list of groups\n"); return -1; } } // Read directories from config files, and get locks. if(!(globalcs=confs_alloc())) return -1; if(confs_init(globalcs)) return -1; if(conf_load_global_only(configfile, globalcs)) return 1; if(get_e_burp_mode(globalcs[OPT_BURP_MODE])!=BURP_MODE_SERVER) { logp("%s is not a server config file\n", configfile); confs_free(&globalcs); return 1; } logp("Dedup clients from %s\n", get_string(globalcs[OPT_CLIENTCONFDIR])); maxlinks=get_int(globalcs[OPT_MAX_HARDLINKS]); if(grouplist) { struct strlist *g=NULL; logp("in dedup groups:\n"); for(g=grouplist; g; g=g->next) logp("%s\n", g->path); } else { char *lockpath=NULL; // Only get the global lock when doing a global run. // If you are doing individual groups, you are likely // to want to do many different dedup jobs and a // global lock would get in the way. if(!(lockpath=prepend( get_string(globalcs[OPT_LOCKFILE]), ".bedup", "")) || !(globallock=lock_alloc_and_init(lockpath))) return 1; lock_get(globallock); if(globallock->status!=GET_LOCK_GOT) { logp("Could not get lock %s (%d)\n", lockpath, globallock->status); free_w(&lockpath); return 1; } logp("Got %s\n", lockpath); } ret=iterate_over_clients(globalcs, grouplist, ext, maxlinks); confs_free(&globalcs); lock_release(globallock); lock_free(&globallock); strlists_free(&grouplist); } if(!nonburp) { logp("%d client storages scanned\n", ccount); } logp("%llu duplicate %s found\n", count, count==1?"file":"files"); logp("%llu bytes %s%s\n", savedbytes, (makelinks || deletedups)?"saved":"saveable", bytes_to_human(savedbytes)); return ret; }
static int iterate_over_clients(struct conf **globalcs, struct strlist *grouplist, const char *ext, unsigned int maxlinks) { int ret=0; DIR *dirp=NULL; struct conf **cconfs=NULL; struct dirent *dirinfo=NULL; const char *globalclientconfdir=get_string(globalcs[OPT_CLIENTCONFDIR]); signal(SIGABRT, &sighandler); signal(SIGTERM, &sighandler); signal(SIGINT, &sighandler); if(!(cconfs=confs_alloc())) return -1; if(confs_init(cconfs)) return -1; if(!(dirp=opendir(globalclientconfdir))) { logp("Could not opendir '%s': %s\n", globalclientconfdir, strerror(errno)); return 0; } while((dirinfo=readdir(dirp))) { char *lockfile=NULL; char *lockfilebase=NULL; char *client_lockdir=NULL; struct lock *lock=NULL; if(dirinfo->d_ino==0 // looks_like...() also avoids '.' and '..'. || looks_like_tmp_or_hidden_file(dirinfo->d_name) || !is_regular_file(globalclientconfdir, dirinfo->d_name)) continue; confs_free_content(cconfs); if(confs_init(cconfs)) return -1; if(set_string(cconfs[OPT_CNAME], dirinfo->d_name)) return -1; if(conf_load_clientconfdir(globalcs, cconfs)) { logp("could not load config for client %s\n", dirinfo->d_name); return 0; } if(grouplist) { const char *dedup_group= get_string(cconfs[OPT_DEDUP_GROUP]); if(!dedup_group || !in_group(grouplist, dedup_group)) continue; } if(!(client_lockdir=get_string(cconfs[OPT_CLIENT_LOCKDIR]))) client_lockdir=get_string(cconfs[OPT_DIRECTORY]); if(!(lockfilebase=prepend(client_lockdir, dirinfo->d_name, "/")) || !(lockfile=prepend(lockfilebase, BEDUP_LOCKFILE_NAME, "/"))) { free_w(&lockfilebase); free_w(&lockfile); ret=-1; break; } free_w(&lockfilebase); if(!(lock=lock_alloc_and_init(lockfile))) { ret=-1; break; } lock_get(lock); free_w(&lockfile); if(lock->status!=GET_LOCK_GOT) { logp("Could not get %s\n", lock->path); continue; } logp("Got %s\n", lock->path); // Remember that we got that lock. lock_add_to_list(&locklist, lock); if(process_dir(get_string(cconfs[OPT_DIRECTORY]), dirinfo->d_name, ext, maxlinks, 1 /* burp mode */, 0 /* level */)) { ret=-1; break; } ccount++; } closedir(dirp); locks_release_and_free(&locklist); confs_free(&cconfs); return ret; }
static #endif int real_main(int argc, char *argv[]) { int ret=1; int option=0; int daemon=1; int forking=1; int strip=0; int randomise=0; struct lock *lock=NULL; struct conf **confs=NULL; int forceoverwrite=0; enum action act=ACTION_LIST; const char *backup=NULL; const char *backup2=NULL; char *restoreprefix=NULL; char *stripfrompath=NULL; const char *regex=NULL; const char *browsefile=NULL; char *browsedir=NULL; const char *conffile=get_conf_path(); const char *orig_client=NULL; const char *logfile=NULL; // The orig_client is the original client that the normal client // would like to restore from. #ifndef HAVE_WIN32 int generate_ca_only=0; #endif int vss_restore=1; int test_confs=0; enum burp_mode mode; log_init(argv[0]); #ifndef HAVE_WIN32 if(!strcmp(prog, "bedup")) return run_bedup(argc, argv); if(!strcmp(prog, "bsigs")) return run_bsigs(argc, argv); #endif while((option=getopt(argc, argv, "a:b:c:C:d:fFghil:nq:Qr:s:tvxjz:?"))!=-1) { switch(option) { case 'a': if(parse_action(&act, optarg)) goto end; break; case 'b': // The diff command may have two backups // specified. if(!backup2 && backup) backup2=optarg; if(!backup) backup=optarg; break; case 'c': conffile=optarg; break; case 'C': orig_client=optarg; break; case 'd': restoreprefix=optarg; // for restores browsedir=optarg; // for lists break; case 'f': forceoverwrite=1; break; case 'F': daemon=0; break; case 'g': #ifndef HAVE_WIN32 generate_ca_only=1; #endif break; case 'i': cmd_print_all(); ret=0; goto end; case 'l': logfile=optarg; break; case 'n': forking=0; break; case 'q': randomise=atoi(optarg); break; case 'Q': log_force_quiet(); break; case 'r': regex=optarg; break; case 's': strip=atoi(optarg); break; case 'v': printf("%s-%s\n", progname(), VERSION); ret=0; goto end; case 'x': vss_restore=0; break; case 't': test_confs=1; break; case 'z': browsefile=optarg; break; case 'h': case '?': default: usage(); goto end; } } if(optind<argc) { usage(); goto end; } if(act==ACTION_MONITOR) { // Try to output everything in JSON. log_set_json(1); #ifndef HAVE_WIN32 // Need to do this so that processes reading stdout get the // result of the printfs of logp straight away. setlinebuf(stdout); #endif } if(!(confs=confs_alloc())) goto end; if(reload(confs, conffile, 1)) goto end; // Dry run to test config file syntax. if(test_confs) { ret=run_test_confs(confs, orig_client); goto end; } if(!backup) switch(act) { case ACTION_DELETE: logp("No backup specified for deletion.\n"); goto end; case ACTION_RESTORE: case ACTION_VERIFY: case ACTION_DIFF: case ACTION_DIFF_LONG: logp("No backup specified. Using the most recent.\n"); backup="0"; default: break; } if(!backup2) switch(act) { case ACTION_DIFF: case ACTION_DIFF_LONG: logp("No second backup specified. Using file system scan.\n"); backup2="n"; // For 'next'. default: break; } // The logfile option is only used for the status client stuff. if(logfile && (act!=ACTION_STATUS && act!=ACTION_STATUS_SNAPSHOT)) logp("-l <logfile> option obsoleted\n"); if(orig_client && *orig_client && set_string(confs[OPT_ORIG_CLIENT], orig_client)) goto end; // The random delay needs to happen before the lock is got, otherwise // you would never be able to use burp by hand. if(randomise) set_int(confs[OPT_RANDOMISE], randomise); mode=get_e_burp_mode(confs[OPT_BURP_MODE]); if(mode==BURP_MODE_CLIENT && (act==ACTION_BACKUP_TIMED || act==ACTION_TIMER_CHECK)) random_delay(confs); if(mode==BURP_MODE_SERVER && act==ACTION_CHAMP_CHOOSER) { // These server modes need to run without getting the lock. } else if(mode==BURP_MODE_CLIENT && (act==ACTION_LIST || act==ACTION_LIST_LONG || act==ACTION_DIFF || act==ACTION_DIFF_LONG || act==ACTION_STATUS || act==ACTION_STATUS_SNAPSHOT || act==ACTION_MONITOR)) { // These client modes need to run without getting the lock. } else { const char *lockfile=confs_get_lockfile(confs); if(!(lock=lock_alloc_and_init(lockfile))) goto end; lock_get(lock); switch(lock->status) { case GET_LOCK_GOT: break; case GET_LOCK_NOT_GOT: logp("Could not get lockfile.\n"); logp("Another process is probably running,\n"); goto end; case GET_LOCK_ERROR: default: logp("Could not get lockfile.\n"); logp("Maybe you do not have permissions to write to %s.\n", lockfile); goto end; } } set_int(confs[OPT_OVERWRITE], forceoverwrite); set_int(confs[OPT_STRIP], strip); set_int(confs[OPT_FORK], forking); set_int(confs[OPT_DAEMON], daemon); strip_trailing_slashes(&restoreprefix); strip_trailing_slashes(&browsedir); if(replace_conf_str(confs[OPT_BACKUP], backup) || replace_conf_str(confs[OPT_BACKUP2], backup2) || replace_conf_str(confs[OPT_RESTOREPREFIX], restoreprefix) || replace_conf_str(confs[OPT_STRIP_FROM_PATH], stripfrompath) || replace_conf_str(confs[OPT_REGEX], regex) || replace_conf_str(confs[OPT_BROWSEFILE], browsefile) || replace_conf_str(confs[OPT_BROWSEDIR], browsedir) || replace_conf_str(confs[OPT_MONITOR_LOGFILE], logfile)) goto end; base64_init(); hexmap_init(); if(mode==BURP_MODE_SERVER) { #ifdef HAVE_WIN32 logp("Sorry, server mode is not implemented for Windows.\n"); #else ret=server_modes(act, conffile, lock, generate_ca_only, confs); #endif } else { ret=client(confs, act, vss_restore); } end: lock_release(lock); lock_free(&lock); confs_free(&confs); return ret; }
int main (int argc, char *argv[]) { int ret=1; int option=0; int daemon=1; int forking=1; int strip=0; struct lock *lock=NULL; struct conf *conf=NULL; int forceoverwrite=0; enum action act=ACTION_LIST; const char *backup=NULL; const char *restoreprefix=NULL; const char *regex=NULL; const char *browsefile=NULL; const char *browsedir=NULL; const char *conffile=get_conf_path(); const char *orig_client=NULL; // The orig_client is the original client that the normal client // would like to restore from. #ifndef HAVE_WIN32 const char *sclient=NULL; // Status monitor client to view. int generate_ca_only=0; #endif int vss_restore=1; int json=0; init_log(argv[0]); while((option=getopt(argc, argv, "a:b:c:C:d:ghfFil:nr:s:vxjz:?"))!=-1) { switch(option) { case 'a': if(parse_action(&act, optarg)) goto end; break; case 'b': backup=optarg; break; case 'c': conffile=optarg; break; case 'C': orig_client=optarg; #ifndef HAVE_WIN32 sclient=optarg; #endif break; case 'd': restoreprefix=optarg; // for restores browsedir=optarg; // for lists break; case 'f': forceoverwrite=1; break; case 'F': daemon=0; break; case 'g': #ifndef HAVE_WIN32 generate_ca_only=1; #endif break; case 'i': cmd_print_all(); ret=0; goto end; case 'l': logp("-l <logfile> option obsoleted\n"); break; case 'n': forking=0; break; case 'r': regex=optarg; break; case 's': strip=atoi(optarg); break; case 'v': printf("%s-%s\n", progname(), VERSION); ret=0; goto end; case 'x': vss_restore=0; break; case 'j': json=1; break; case 'z': browsefile=optarg; break; case 'h': case '?': default: usage(); goto end; } } if(optind<argc) { usage(); goto end; } if(!(conf=conf_alloc())) goto end; if(reload(conf, conffile, 1 /* first time */, 0 /* no oldmax_children setting */, 0 /* no oldmax_status_children setting */, json)) goto end; if((act==ACTION_RESTORE || act==ACTION_VERIFY) && !backup) { logp("No backup specified. Using the most recent.\n"); backup="0"; } if(act==ACTION_DELETE && !backup) { logp("No backup specified for deletion.\n"); goto end; } if(conf->mode==MODE_CLIENT) { if(orig_client && *orig_client) { if(!(conf->orig_client=strdup(orig_client))) { log_out_of_memory(__func__); goto end; } } } if(conf->mode==MODE_SERVER && (act==ACTION_STATUS || act==ACTION_STATUS_SNAPSHOT || act==ACTION_CHAMP_CHOOSER)) { // These server modes need to run without getting the lock. } else { if(!(lock=lock_alloc_and_init(conf->lockfile))) goto end; lock_get(lock); switch(lock->status) { case GET_LOCK_GOT: break; case GET_LOCK_NOT_GOT: logp("Could not get lockfile.\n"); logp("Another process is probably running,\n"); goto end; case GET_LOCK_ERROR: default: logp("Could not get lockfile.\n"); logp("Maybe you do not have permissions to write to %s.\n", conf->lockfile); goto end; } } conf->overwrite=forceoverwrite; conf->strip=strip; conf->forking=forking; conf->daemon=daemon; if(replace_conf_str(backup, &conf->backup) || replace_conf_str(restoreprefix, &conf->restoreprefix) || replace_conf_str(regex, &conf->regex) || replace_conf_str(browsefile, &conf->browsefile) || replace_conf_str(browsedir, &conf->browsedir)) goto end; if(conf->mode==MODE_SERVER) { #ifdef HAVE_WIN32 logp("Sorry, server mode is not implemented for Windows.\n"); #else if(act==ACTION_STATUS || act==ACTION_STATUS_SNAPSHOT) { // We are running on the server machine, being a client // of the burp server, getting status information. ret=status_client_ncurses(conf, act, sclient); } else if(act==ACTION_CHAMP_CHOOSER) { // We are running on the server machine, wanting to // be a standalone champion chooser process. if(!sclient || !*sclient) { logp("No client name specified for standalone champion chooser process.\n"); logp("Try using the '-C' option.\n"); ret=1; } else ret=champ_chooser_server_standalone(conf, sclient); } else ret=server(conf, conffile, lock, generate_ca_only); #endif } else { logp("before client\n"); ret=client(conf, act, vss_restore, json); logp("after client\n"); } end: lock_release(lock); lock_free(&lock); conf_free(conf); return ret; }
int champ_chooser_server(struct sdirs *sdirs, struct conf **confs, int resume) { int s; int ret=-1; int len; struct asfd *asfd=NULL; struct sockaddr_un local; struct lock *lock=NULL; struct async *as=NULL; int started=0; struct scores *scores=NULL; const char *directory=get_string(confs[OPT_DIRECTORY]); if(!(lock=lock_alloc_and_init(sdirs->champlock)) || build_path_w(sdirs->champlock)) goto end; lock_get(lock); switch(lock->status) { case GET_LOCK_GOT: log_fzp_set(sdirs->champlog, confs); logp("Got champ lock for dedup_group: %s\n", get_string(confs[OPT_DEDUP_GROUP])); break; case GET_LOCK_NOT_GOT: case GET_LOCK_ERROR: default: //logp("Did not get champ lock\n"); goto end; } if((s=socket(AF_UNIX, SOCK_STREAM, 0))<0) { logp("socket error in %s: %s\n", __func__, strerror(errno)); goto end; } memset(&local, 0, sizeof(struct sockaddr_un)); local.sun_family=AF_UNIX; snprintf(local.sun_path, sizeof(local.sun_path), "%s", sdirs->champsock); len=strlen(local.sun_path)+sizeof(local.sun_family)+1; unlink(sdirs->champsock); if(bind(s, (struct sockaddr *)&local, len)<0) { logp("bind error in %s: %s\n", __func__, strerror(errno)); goto end; } if(listen(s, 5)<0) { logp("listen error in %s: %s\n", __func__, strerror(errno)); goto end; } if(!(as=async_alloc()) || as->init(as, 0) || !(asfd=setup_asfd(as, "champ chooser main socket", &s, /*listen*/""))) goto end; asfd->fdtype=ASFD_FD_SERVER_LISTEN_MAIN; // I think that this is probably the best point at which to run a // cleanup job to delete unused data files, because no other process // can fiddle with the dedup_group at this point. // Cannot do it on a resume, or it will delete files that are // referenced in the backup we are resuming. if(delete_unused_data_files(sdirs, resume)) goto end; // Load the sparse indexes for this dedup group. if(!(scores=champ_chooser_init(sdirs->data))) goto end; while(1) { for(asfd=as->asfd->next; asfd; asfd=asfd->next) { if(!asfd->blist->head || asfd->blist->head->got==BLK_INCOMING) continue; if(results_to_fd(asfd)) goto end; } int removed; switch(as->read_write(as)) { case 0: // Check the main socket last, as it might add // a new client to the list. for(asfd=as->asfd->next; asfd; asfd=asfd->next) { while(asfd->rbuf->buf) { if(deal_with_client_rbuf(asfd, directory, scores)) goto end; // Get as much out of the // readbuf as possible. if(asfd->parse_readbuf(asfd)) goto end; } } if(as->asfd->new_client) { // Incoming client. as->asfd->new_client=0; if(champ_chooser_new_client(as, confs)) goto end; started=1; } break; default: removed=0; // Maybe one of the fds had a problem. // Find and remove it and carry on if possible. for(asfd=as->asfd->next; asfd; ) { struct asfd *a; if(!asfd->want_to_remove) { asfd=asfd->next; continue; } as->asfd_remove(as, asfd); logp("%s: disconnected fd %d\n", asfd->desc, asfd->fd); a=asfd->next; asfd_free(&asfd); asfd=a; removed++; } if(removed) break; // If we got here, there was no fd to remove. // It is a fatal error. goto end; } if(started && !as->asfd->next) { logp("All clients disconnected.\n"); ret=0; break; } } end: logp("champ chooser exiting: %d\n", ret); champ_chooser_free(&scores); log_fzp_set(NULL, confs); async_free(&as); asfd_free(&asfd); // This closes s for us. close_fd(&s); unlink(sdirs->champsock); // FIX THIS: free asfds. lock_release(lock); lock_free(&lock); return ret; }