// calls either process_dir or process_file, above
static void process(const char *what, int level, int recursive,
char *overlapping_dems[], int *next_dem_number,
meta_parameters *meta, int *n_dems_total)
{
struct stat stbuf;
if (stat(what, &stbuf) == -1) {
asfPrintStatus(" Cannot access: %s\n", what);
return;
}
char *base = get_filename(what);
if ((stbuf.st_mode & S_IFMT) == S_IFDIR) {
if (level==0 || recursive) {
asfPrintStatus(" %s%s/\n", spaces(level), base);
process_dir(what, level+1, recursive, overlapping_dems,
next_dem_number, meta, n_dems_total);
}
else {
asfPrintStatus(" %s%s (skipped)\n", spaces(level), base);
}
}
else {
process_file(what, level, overlapping_dems, next_dem_number,
meta, n_dems_total);
}
FREE(base);
}
bool state::should_hash(const tstring &fn)
{
file_types type = state::file_type(fn,&ocb,0,0,0,0);
if (mode_expert)
return should_hash_expert(fn,type);
if (type == stat_directory) {
if (mode_recursive){
process_dir(fn);
} else {
ocb.error_filename(fn,"Is a directory");
}
return false;
}
if (type == stat_symlink){
return should_hash_symlink(fn,NULL);
}
if (type == stat_unknown){
return false;
}
// By default we hash anything we can't identify as a "bad thing"
return true;
}
/* Deterine if a symlink should be hashed or not.
* Returns TRUE if a symlink should be hashed.
*/
bool state::should_hash_symlink(const tstring &fn, file_types *link_type)
{
/**
* We must look at what this symlink points to before we process it.
* The file_type() function uses lstat to examine the file.
* Here we use the normal stat to examine what this symlink points to.
*/
struct __stat64 sb;
if (TSTAT(fn.c_str(),&sb)) {
ocb.error_filename(fn,"%s",strerror(errno));
return false;
}
file_types type = file_metadata_t::decode_file_type(sb);
if (type == stat_directory) {
if (mode_recursive){
process_dir(fn);
} else {
ocb.error_filename(fn,"Is a directory");
}
return false;
}
if (link_type) *link_type = type;
return true;
}
void
process_dir (char *dirname)
{
DIR *dir;
struct dirent *de;
struct stat st;
if ((dir = opendir (dirname)) == NULL)
{
perror ("opendir");
return;
}
if (chdir (dirname) == -1)
{
perror ("chdir");
return;
}
while ((de = readdir (dir)) != NULL)
{
if (stat (de->d_name, &st) == -1)
{
perror ("stat");
continue;
}
if (S_ISDIR (st.st_mode) && !strcmp (de->d_name, ".")
&& !strcmp (de->d_name, ".."))
process_dir (de->d_name);
else if (S_ISREG (st.st_mode) || S_ISLNK (st.st_mode))
process_file (de->d_name);
}
if (chdir ("..") == -1)
perror ("chdir ..");
}
int main()
{
process_dir();
printf("total: %d\n", total);
return 0;
}
static void
process_dir (xmonitor_t *mon, gchar *dirpath)
{
GDir *dir;
const gchar *tmp;
gchar *path;
dir = g_dir_open (dirpath, 0, NULL);
if (!dir) {
ERR ("Error when opening %s", dirpath);
return;
}
while ((tmp = g_dir_read_name (dir))) {
path = g_strdup_printf ("%s/%s", dirpath, tmp);
if (g_file_test (path, G_FILE_TEST_IS_DIR)) {
if (!monitor_add_dir (mon, path)) {
ERR ("Couldn't watch directory: %s!", path);
g_free (path);
continue;
}
DBG ("Now watching dir %s", path);
mon->dir_list = g_list_append (mon->dir_list, path);
process_dir (mon, path);
} else {
g_free (path);
}
}
g_main_context_iteration (NULL, FALSE);
g_dir_close (dir);
}
void
process_objects(CIRCLE_handle *handle)
{
process_objects_total[0] = MPI_Wtime();
static char temp[CIRCLE_MAX_STRING_LEN];
static char stat_temp[CIRCLE_MAX_STRING_LEN];
static char redis_cmd_buf[CIRCLE_MAX_STRING_LEN];
static char filekey[512];
struct stat st;
int status = 0;
int crc = 0;
/* Pop an item off the queue */
handle->dequeue(temp);
/* Try and stat it, checking to see if it is a link */
stat_time[0] = MPI_Wtime();
status = lstat(temp,&st);
stat_time[1] += MPI_Wtime()-stat_time[0];
if(status != EXIT_SUCCESS)
{
LOG(PURGER_LOG_ERR, "Error: Couldn't stat \"%s\"", temp);
}
/* Check to see if it is a directory. If so, put its children in the queue */
else if(S_ISDIR(st.st_mode) && !(S_ISLNK(st.st_mode)))
{
readdir_time[0] = MPI_Wtime();
process_dir(stat_temp,temp,handle);
readdir_time[1] += MPI_Wtime() - readdir_time[0];
}
else if(!benchmarking_flag && S_ISREG(st.st_mode))
{
/* Hash the file */
hash_time[0] = MPI_Wtime();
treewalk_redis_keygen(filekey, temp);
crc = (int)crc32(filekey,32) % sharded_count;
hash_time[1] += MPI_Wtime() - hash_time[0];
/* Create and hset with basic attributes. */
treewalk_create_redis_attr_cmd(redis_cmd_buf, &st, temp, filekey);
/* Execute the redis command */
redis_time[0] = MPI_Wtime();
(*redis_command_ptr)(crc,redis_cmd_buf);
redis_time[1] += MPI_Wtime() - redis_time[0];
/* Check to see if the file is expired.
If so, zadd it by mtime and add the user id
to warnlist */
if(difftime(time_started,st.st_mtime) > expire_threshold)
{
LOG(PURGER_LOG_DBG,"File expired: \"%s\"",temp);
redis_time[0] = MPI_Wtime();
/* The mtime of the file as a zadd. */
treewalk_redis_run_zadd(filekey, (long)st.st_mtime, "mtime",crc);
/* add user to warn list */
treewalk_redis_run_sadd(&st);
redis_time[1] += MPI_Wtime() - redis_time[0];
}
}
process_objects_total[1] += MPI_Wtime() - process_objects_total[0];
}
/*
* Type should be the result of calling lstat on the file.
* We want to know what this file is, not what it points to
* This expert calls itself recursively. The dir_table
* makes sure that we do not loop.
*/
bool state::should_hash_expert(const tstring &fn, file_types type)
{
file_types link_type=stat_unknown;
if (stat_directory == type) {
if (mode_recursive){
process_dir(fn);
}
else {
ocb.error_filename(fn,"Is a directory");
}
return false;
}
if (mode_winpe) {
// The user could have requested PE files *and* something else
// therefore we don't return false here if the file is not a PE.
// Note that we have to check for directories first!
if (should_hash_winpe(fn)){
return true;
}
}
switch(type) {
// We can't just return s->mode & mode_X because mode_X is
// a 64-bit value. When that value gets converted back to int,
// the high part of it is lost.
#define RETURN_IF_MODE(A) if (A) return true; break;
case stat_directory:
// This case should be handled above. This statement is
// here to avoid compiler warnings
ocb.internal_error("Did not handle directory entry in should_hash_expert()");
case stat_regular: RETURN_IF_MODE(mode_regular);
case stat_block: RETURN_IF_MODE(mode_block);
case stat_character: RETURN_IF_MODE(mode_character);
case stat_pipe: RETURN_IF_MODE(mode_pipe);
case stat_socket: RETURN_IF_MODE(mode_socket);
case stat_door: RETURN_IF_MODE(mode_door);
case stat_symlink:
// Although it might appear that we need nothing more than
// return (s->mode & mode_symlink);
// that doesn't work. That logic gets into trouble when we're
// running in recursive mode on a symlink to a directory.
// The program attempts to open the directory entry itself
// and gets into an infinite loop.
if (!(mode_symlink)) return false;
if (should_hash_symlink(fn,&link_type)) {
return should_hash_expert(fn,link_type);
}
return false;
case stat_unknown:
ocb.error_filename(fn,"unknown file type");
return false;
}
return false;
}
bool
process_path(const std::string& path_actual,
const std::string& path_output,
std::ostream& os)
{
struct stat s;
if (lstat(path_actual.c_str(), &s) < 0) {
std::cerr << progname
<< ": error: " << strerror(errno)
<< ": " << path_actual << std::endl;
return false;
}
if (S_ISREG(s.st_mode)) {
if (! process_file(path_actual, path_output, &s, os)) {
return false;
}
} else if (S_ISDIR(s.st_mode)) {
if (! process_dir(path_actual, path_output, os)) {
return false;
}
} else if (S_ISLNK(s.st_mode)) {
if (! process_symlink(path_actual, path_output, os)) {
return false;
}
} else {
std::cerr << progname
<< ": error: unexpected file type: " << path_actual
<< " (" << (s.st_mode & S_IFMT) << ")" << std::endl;
return false;
}
// S_ISCHR character device
// S_ISBLK block device
// S_ISFIFO
// S_ISSOCK
if (! os) {
std::cerr << progname
<< ": error: write error" << std::endl;
return false;
}
return true;
}
/**
* Recursively add files under a path to the media library.
*
* @param medialib the medialib object
* @param path the directory to scan for files
* @param error If an error occurs, it will be stored in there.
*
* @return an IDLIST collection with the added entries
*/
xmmsv_t *
xmms_medialib_add_recursive (xmms_medialib_t *medialib, const gchar *path,
xmms_error_t *error)
{
xmmsv_t *entries;
entries = xmmsv_new_coll (XMMS_COLLECTION_TYPE_IDLIST);
g_return_val_if_fail (medialib, entries);
g_return_val_if_fail (path, entries);
process_dir (medialib, entries, path, error);
return entries;
}
int process_dir(void)
{
DIR *dp = NULL;
char *path = NULL;
struct stat statbuff;
struct dirent * dirbuf = NULL;
int namlen = 0;
path = getcwd(NULL, 0);
if (NULL == (dp = opendir(path)))
error_exit("opendir");
while (NULL != (dirbuf = readdir(dp))) {
/* get file stat */
if (-1 == stat(dirbuf->d_name, &statbuff))
error_exit(dirbuf->d_name);
if ('.' == dirbuf->d_name[0])
continue;
if (S_ISDIR(statbuff.st_mode)) {
/* if dir, then recursion */
if (-1 == chdir(dirbuf->d_name))
error_exit(dirbuf->d_name);
process_dir();
} else {
/*regular file, comput line*/
namlen = strlen(dirbuf->d_name);
#if 1
if (strcmp(".c", dirbuf->d_name+namlen-2)
&& strcmp(".h", dirbuf->d_name+namlen-2)
&& strcmp(".sh", dirbuf->d_name+namlen-3)
&& strcmp("Makefile", dirbuf->d_name)
)
continue;
#endif
namlen = process_file(dirbuf->d_name);
/* exclude sqlite3 */
if (namlen < 400) {
printf("%-30s %d\n", dirbuf->d_name, namlen);
total += namlen;
}
}
}
closedir(dp);
chdir("..");
return 0;
}
static int
handle_addpath (xmmsv_t *v, void *data)
{
xmonitor_t *mon = data;
if (!monitor_add_dir (mon, mon->watch_dir)) {
ERR ("Couldn't watch directory!");
return FALSE;
}
mon->dir_list = g_list_append (mon->dir_list, mon->watch_dir);
process_dir (mon, mon->watch_dir);
DBG ("Watching %d dirs", g_list_length (mon->dir_list));
return FALSE;
}
void process_dir( const char* c ) {
//printf("Processing %s\n", c) ;
DIR* d = opendir( c ) ;
struct dirent* e;
while ( (e=readdir(d)) ) {
// skip dot, dot-dot
if ( !strcmp(e->d_name,"." ) ) continue;
if ( !strcmp(e->d_name,".." ) ) continue;
// skip player info
if ( !strcmp(e->d_name,"level.dat" ) ) continue;
// only directories and .dat files
if ( e->d_type == DT_DIR ) {
int len = strlen(c) + 1 + strlen(e->d_name) + 1 ;
char *f = (char*)malloc(sizeof(char)*len);
strcpy(f,c);
strcat(f,"/");
strcat(f,e->d_name);
process_dir(f);
free(f);
continue;
}
if ( !strstr(e->d_name+strlen(e->d_name)-4, ".dat" )) continue ;
// ok, we have a file to process
char* fname = (char*)malloc(sizeof(char)*(strlen(c)+strlen(e->d_name)+2));
strcpy(fname,c) ; strcat(fname,"/") ; strcat(fname,e->d_name);
Named_Tag* n = process_file(fname);
first_file = 0 ;
//print_named_tag( n ) ;
//printf("\n");
//printf("%s: %i %i\n", fname, current_x, current_z) ;
add_chunk( current_x, current_z, current_chunk ) ;
free(fname);
NT_dispose(n);
//printf("%s\n", e->d_name ) ;
}
closedir(d);
}
// relative path
int process_dir(char *path, struct s_category *category)
{
DIR *dir;
struct dirent entry;
struct dirent *result;
struct stat entryInfo;
char path_name[PATH_MAX + 1];
char absolute_path[PATH_MAX + 1];
char new_dir[PATH_MAX + 1];
sprintf(absolute_path, "%s/%s", config_site.source_directory, path);
if ((dir = opendir(absolute_path)) == NULL) {
printf("Couldn't open %s : %s\n", path, strerror(errno));
return -1;
}
while ((readdir_r(dir, &entry, &result) == 0) && result != 0) {
if (strcmp(entry.d_name, ".") == 0 ||
strcmp(entry.d_name, "..") == 0) {
continue;
}
strncpy(path_name, path, PATH_MAX);
strncat(path_name, "/", PATH_MAX);
strncat(path_name, entry.d_name, PATH_MAX);
sprintf(absolute_path, "%s/%s", config_site.source_directory, path_name);
if (lstat(absolute_path, &entryInfo) == 0) {
if (S_ISDIR (entryInfo.st_mode)) {
sprintf(new_dir, "%s/%s", config_site.dest_directory, path_name+1); // +1 is to delete _
create_dir(new_dir);
process_dir(path_name, category);
} else {
process_file(path_name, category);
}
}
}
closedir(dir);
return 0;
}
//Given dirs just for this thread and global file queue
void process_dirs(queue_t * loc_dir_queue, queue_t * glb_file_queue, queue_t * glb_dir_queue)
{
//We have a list of dirs
//Loop through each of them
while(1)
{
queue_element_t * item = remove_from_queue(loc_dir_queue);
if(item == NULL)
{
return;
}
//Get the str from this item
char * str = item->path_name;
//Process this dir
process_dir(str,glb_file_queue, glb_dir_queue);
}
}
/**
* Recursively scan a directory for media files.
*
* @return a reverse sorted list of encoded urls
*/
static gboolean
process_dir (xmms_medialib_t *medialib, xmmsv_t *entries,
const gchar *directory, xmms_error_t *error)
{
xmmsv_list_iter_t *it;
xmmsv_t *list, *val;
list = xmms_xform_browse (directory, error);
if (!list) {
return FALSE;
}
xmmsv_get_list_iter (list, &it);
while (xmmsv_list_iter_entry (it, &val)) {
const gchar *str;
gint isdir;
xmmsv_dict_entry_get_string (val, "path", &str);
xmmsv_dict_entry_get_int (val, "isdir", &isdir);
if (isdir == 1) {
process_dir (medialib, entries, str, error);
} else {
xmms_medialib_session_t *session;
xmms_medialib_entry_t entry;
do {
session = xmms_medialib_session_begin (medialib);
entry = xmms_medialib_entry_new_encoded (session, str, error);
} while (!xmms_medialib_session_commit (session));
if (entry) {
xmmsv_coll_idlist_append (entries, entry);
}
}
xmmsv_list_iter_remove (it);
}
xmmsv_unref (list);
return TRUE;
}
int process_category(char *dirname)
{
struct s_category *category;
char new_dir[PATH_MAX + 1];
category = add_category(dirname);
if (category == NULL) {
printf("Couldn't properly add category : %s\n", dirname);
return -1;
}
sprintf(new_dir, "%s/%s", config_site.dest_directory, category->name);
if (create_dir(new_dir) < 0) {
return -1;
}
else {
/* Category is created, now we can dig into it */
return process_dir(dirname, category);
}
}
/*
* main
*/
int main(int argc, const char * const argv[])
{
apr_off_t max;
apr_time_t current, repeat, delay, previous;
apr_status_t status;
apr_pool_t *pool, *instance;
apr_getopt_t *o;
apr_finfo_t info;
int retries, isdaemon, limit_found, intelligent, dowork;
char opt;
const char *arg;
char *proxypath, *path;
interrupted = 0;
repeat = 0;
isdaemon = 0;
dryrun = 0;
limit_found = 0;
max = 0;
verbose = 0;
realclean = 0;
benice = 0;
deldirs = 0;
intelligent = 0;
previous = 0; /* avoid compiler warning */
proxypath = NULL;
if (apr_app_initialize(&argc, &argv, NULL) != APR_SUCCESS) {
return 1;
}
atexit(apr_terminate);
if (argc) {
shortname = apr_filepath_name_get(argv[0]);
}
if (apr_pool_create(&pool, NULL) != APR_SUCCESS) {
return 1;
}
apr_pool_abort_set(oom, pool);
apr_file_open_stderr(&errfile, pool);
apr_signal(SIGINT, setterm);
apr_signal(SIGTERM, setterm);
apr_getopt_init(&o, pool, argc, argv);
while (1) {
status = apr_getopt(o, "iDnvrtd:l:L:p:", &opt, &arg);
if (status == APR_EOF) {
break;
}
else if (status != APR_SUCCESS) {
usage();
}
else {
switch (opt) {
case 'i':
if (intelligent) {
usage();
}
intelligent = 1;
break;
case 'D':
if (dryrun) {
usage();
}
dryrun = 1;
break;
case 'n':
if (benice) {
usage();
}
benice = 1;
break;
case 't':
if (deldirs) {
usage();
}
deldirs = 1;
break;
case 'v':
if (verbose) {
usage();
}
verbose = 1;
break;
case 'r':
if (realclean) {
usage();
}
realclean = 1;
deldirs = 1;
break;
case 'd':
if (isdaemon) {
usage();
}
isdaemon = 1;
repeat = apr_atoi64(arg);
repeat *= SECS_PER_MIN;
repeat *= APR_USEC_PER_SEC;
break;
case 'l':
if (limit_found) {
usage();
}
limit_found = 1;
do {
apr_status_t rv;
char *end;
rv = apr_strtoff(&max, arg, &end, 10);
if (rv == APR_SUCCESS) {
if ((*end == 'K' || *end == 'k') && !end[1]) {
max *= KBYTE;
}
else if ((*end == 'M' || *end == 'm') && !end[1]) {
max *= MBYTE;
}
else if ((*end == 'G' || *end == 'g') && !end[1]) {
max *= GBYTE;
}
else if (*end && /* neither empty nor [Bb] */
((*end != 'B' && *end != 'b') || end[1])) {
rv = APR_EGENERAL;
}
}
if (rv != APR_SUCCESS) {
apr_file_printf(errfile, "Invalid limit: %s"
APR_EOL_STR APR_EOL_STR, arg);
usage();
}
} while(0);
break;
case 'p':
if (proxypath) {
usage();
}
proxypath = apr_pstrdup(pool, arg);
if (apr_filepath_set(proxypath, pool) != APR_SUCCESS) {
usage();
}
break;
} /* switch */
} /* else */
} /* while */
if (o->ind != argc) {
usage();
}
if (isdaemon && (repeat <= 0 || verbose || realclean || dryrun)) {
usage();
}
if (!isdaemon && intelligent) {
usage();
}
if (!proxypath || max <= 0) {
usage();
}
if (apr_filepath_get(&path, 0, pool) != APR_SUCCESS) {
usage();
}
baselen = strlen(path);
#ifndef DEBUG
if (isdaemon) {
apr_file_close(errfile);
apr_proc_detach(APR_PROC_DETACH_DAEMONIZE);
}
#endif
do {
apr_pool_create(&instance, pool);
now = apr_time_now();
APR_RING_INIT(&root, _entry, link);
delcount = 0;
unsolicited = 0;
dowork = 0;
switch (intelligent) {
case 0:
dowork = 1;
break;
case 1:
retries = STAT_ATTEMPTS;
status = APR_SUCCESS;
do {
if (status != APR_SUCCESS) {
apr_sleep(STAT_DELAY);
}
status = apr_stat(&info, path, APR_FINFO_MTIME, instance);
} while (status != APR_SUCCESS && !interrupted && --retries);
if (status == APR_SUCCESS) {
previous = info.mtime;
intelligent = 2;
}
dowork = 1;
break;
case 2:
retries = STAT_ATTEMPTS;
status = APR_SUCCESS;
do {
if (status != APR_SUCCESS) {
apr_sleep(STAT_DELAY);
}
status = apr_stat(&info, path, APR_FINFO_MTIME, instance);
} while (status != APR_SUCCESS && !interrupted && --retries);
if (status == APR_SUCCESS) {
if (previous != info.mtime) {
dowork = 1;
}
previous = info.mtime;
break;
}
intelligent = 1;
dowork = 1;
break;
}
if (dowork && !interrupted) {
if (!process_dir(path, instance) && !interrupted) {
purge(path, instance, max);
}
else if (!isdaemon && !interrupted) {
apr_file_printf(errfile, "An error occurred, cache cleaning "
"aborted." APR_EOL_STR);
return 1;
}
if (intelligent && !interrupted) {
retries = STAT_ATTEMPTS;
status = APR_SUCCESS;
do {
if (status != APR_SUCCESS) {
apr_sleep(STAT_DELAY);
}
status = apr_stat(&info, path, APR_FINFO_MTIME, instance);
} while (status != APR_SUCCESS && !interrupted && --retries);
if (status == APR_SUCCESS) {
previous = info.mtime;
intelligent = 2;
}
else {
intelligent = 1;
}
}
}
apr_pool_destroy(instance);
current = apr_time_now();
if (current < now) {
delay = repeat;
}
else if (current - now >= repeat) {
delay = repeat;
}
else {
delay = now + repeat - current;
}
/* we can't sleep the whole delay time here apiece as this is racy
* with respect to interrupt delivery - think about what happens
* if we have tested for an interrupt, then get scheduled
* before the apr_sleep() call and while waiting for the cpu
* we do get an interrupt
*/
if (isdaemon) {
while (delay && !interrupted) {
if (delay > APR_USEC_PER_SEC) {
apr_sleep(APR_USEC_PER_SEC);
delay -= APR_USEC_PER_SEC;
}
else {
apr_sleep(delay);
delay = 0;
}
}
}
} while (isdaemon && !interrupted);
if (!isdaemon && interrupted) {
apr_file_printf(errfile, "Cache cleaning aborted due to user "
"request." APR_EOL_STR);
return 1;
}
return 0;
}
/**
* Main program. Returns 1 if all programs run successfully, 0 if
* something failed and -1 if there was an error running programs.
*/
int main(int argc, char *argv[])
{
int retcode;
struct sigaction zig;
pid_t pid;
char *c;
/* Check parameters */
if (argc < 2) {
fprintf(stderr, "Usage: %s test_prog\n", argv[0]);
return 1;
}
if (argc > 2 && strcmp(argv[2], "--leader") == 0) {
pounder_fprintf(stdout,
"Logging this test output to %s/POUNDERLOG.\n",
getenv("POUNDER_LOGDIR"));
is_leader = 1;
record_pid();
}
progname = argv[0];
/* Set up signals */
memset(&zig, 0x00, sizeof(zig));
zig.sa_handler = jump_out;
sigaction(SIGHUP, &zig, NULL);
sigaction(SIGINT, &zig, NULL);
sigaction(SIGTERM, &zig, NULL);
if (is_directory(argv[1])) {
retcode = process_dir(argv[1]);
} else {
if (is_executable(argv[1])) {
c = rindex(argv[1], '/');
c++;
// Start the test
pid = spawn_test(argv[1]);
if (pid < 0) {
perror("fork");
retcode = -1;
goto out;
}
// Track the test
note_process(pid, argv[1]);
if (wait_for_pids() == 0) {
retcode = 1;
} else {
retcode = 0;
}
} else {
pounder_fprintf(stderr,
"%s: Not a directory or a test.\n",
argv[1]);
retcode = -1;
}
}
out:
kill_daemons();
wait_for_daemons();
if (is_leader) {
if (retcode == 0) {
pounder_fprintf(stdout, "%s: %s.\n", argv[1], pass_msg);
} else if (retcode < 0 || retcode == 255) {
pounder_fprintf(stdout, "%s: %s with code %d.\n",
argv[1], abort_msg, retcode);
} else {
pounder_fprintf(stdout, "%s: %s with code %d.\n",
argv[1], fail_msg, retcode);
}
unlink(pidfile);
}
exit(retcode);
}
int
process_entry (const char *path, const struct stat *sb,
int typeflag, struct FTW *ftwbuf)
{
int ret = 0;
char *name = NULL;
char *bname = NULL;
char *dname = NULL;
int i = 0;
/* The if condition below helps in ignoring some directories in
the given path. If the name of the entry is one of the directory
names that the user told to ignore, then that directory will not
be processed and will return FTW_SKIP_SUBTREE to nftw which will
not crawl this directory and move on to other siblings.
Note that for nftw to recognize FTW_SKIP_TREE, FTW_ACTIONRETVAL
should be passed as an argument to nftw.
This mainly helps in calculating the checksum of network filesystems
(client-server), where the server might have some hidden directories
for managing the filesystem. So to calculate the sanity of filesytem
one has to get the checksum of the client and then the export directory
of server by telling arequal to ignore some of the directories which
are not part of the namespace.
*/
if (arequal_config.ignored_directory) {
#ifndef FTW_SKIP_SUBTREE
char *cp;
name = strdup (path);
dname = dirname (name);
for (cp = strtok(name, "/"); cp; cp = strtok(NULL, "/")) {
if (ignore_entry(cp, dname)) {
DBG ("ignoring %s\n", path);
if (name)
free (name);
return 0;
}
}
#else /* FTW_SKIP_SUBTREE */
name = strdup (path);
name[strlen(name)] = '\0';
bname = strrchr (name, '/');
if (bname)
bname++;
dname = dirname (name);
if (ignore_entry(bname, dname)) {
DBG ("ignoring %s\n", bname);
ret = FTW_SKIP_SUBTREE;
if (name)
free (name);
return ret;
}
#endif /* FTW_SKIP_SUBTREE */
}
DBG ("processing entry %s\n", path);
switch ((S_IFMT & sb->st_mode)) {
case S_IFDIR:
ret = process_dir (path, sb);
break;
case S_IFREG:
ret = process_file (path, sb);
break;
case S_IFLNK:
ret = process_symlink (path, sb);
break;
default:
ret = process_other (path, sb);
break;
}
if (name)
free (name);
return ret;
}
/*
* walk the cache directory tree
*/
static int process_dir(char *path, apr_pool_t *pool)
{
apr_dir_t *dir;
apr_pool_t *p;
apr_hash_t *h;
apr_hash_index_t *i;
apr_file_t *fd;
apr_status_t status;
apr_finfo_t info;
apr_size_t len;
apr_time_t current, deviation;
char *nextpath, *base, *ext, *orig_basename;
APR_RING_ENTRY(_direntry) anchor;
DIRENTRY *d, *t, *n;
ENTRY *e;
int skip, retries;
disk_cache_info_t disk_info;
APR_RING_INIT(&anchor, _direntry, link);
apr_pool_create(&p, pool);
h = apr_hash_make(p);
fd = NULL;
skip = 0;
deviation = MAXDEVIATION * APR_USEC_PER_SEC;
if (apr_dir_open(&dir, path, p) != APR_SUCCESS) {
return 1;
}
while (apr_dir_read(&info, 0, dir) == APR_SUCCESS && !interrupted) {
if (!strcmp(info.name, ".") || !strcmp(info.name, "..")) {
continue;
}
d = apr_pcalloc(p, sizeof(DIRENTRY));
d->basename = apr_pstrcat(p, path, "/", info.name, NULL);
APR_RING_INSERT_TAIL(&anchor, d, _direntry, link);
}
apr_dir_close(dir);
if (interrupted) {
return 1;
}
skip = baselen + 1;
for (d = APR_RING_FIRST(&anchor);
!interrupted && d != APR_RING_SENTINEL(&anchor, _direntry, link);
d=n) {
n = APR_RING_NEXT(d, link);
base = strrchr(d->basename, '/');
if (!base++) {
base = d->basename;
}
ext = strchr(base, '.');
/* there may be temporary files which may be gone before
* processing, always skip these if not in realclean mode
*/
if (!ext && !realclean) {
if (!strncasecmp(base, AP_TEMPFILE_BASE, AP_TEMPFILE_BASELEN)
&& strlen(base) == AP_TEMPFILE_NAMELEN) {
continue;
}
}
/* this may look strange but apr_stat() may return errno which
* is system dependent and there may be transient failures,
* so just blindly retry for a short while
*/
retries = STAT_ATTEMPTS;
status = APR_SUCCESS;
do {
if (status != APR_SUCCESS) {
apr_sleep(STAT_DELAY);
}
status = apr_stat(&info, d->basename, DIRINFO, p);
} while (status != APR_SUCCESS && !interrupted && --retries);
/* what may happen here is that apache did create a file which
* we did detect but then does delete the file before we can
* get file information, so if we don't get any file information
* we will ignore the file in this case
*/
if (status != APR_SUCCESS) {
if (!realclean && !interrupted) {
continue;
}
return 1;
}
if (info.filetype == APR_DIR) {
/* Make a copy of the basename, as process_dir modifies it */
orig_basename = apr_pstrdup(pool, d->basename);
if (process_dir(d->basename, pool)) {
return 1;
}
/* If asked to delete dirs, do so now. We don't care if it fails.
* If it fails, it likely means there was something else there.
*/
if (deldirs && !dryrun) {
apr_dir_remove(orig_basename, pool);
}
continue;
}
if (info.filetype != APR_REG) {
continue;
}
if (!ext) {
if (!strncasecmp(base, AP_TEMPFILE_BASE, AP_TEMPFILE_BASELEN)
&& strlen(base) == AP_TEMPFILE_NAMELEN) {
d->basename += skip;
d->type = TEMP;
d->dsize = info.size;
apr_hash_set(h, d->basename, APR_HASH_KEY_STRING, d);
}
continue;
}
if (!strcasecmp(ext, CACHE_HEADER_SUFFIX)) {
*ext = '\0';
d->basename += skip;
/* if a user manually creates a '.header' file */
if (d->basename[0] == '\0') {
continue;
}
t = apr_hash_get(h, d->basename, APR_HASH_KEY_STRING);
if (t) {
d = t;
}
d->type |= HEADER;
d->htime = info.mtime;
d->hsize = info.size;
apr_hash_set(h, d->basename, APR_HASH_KEY_STRING, d);
continue;
}
if (!strcasecmp(ext, CACHE_DATA_SUFFIX)) {
*ext = '\0';
d->basename += skip;
/* if a user manually creates a '.data' file */
if (d->basename[0] == '\0') {
continue;
}
t = apr_hash_get(h, d->basename, APR_HASH_KEY_STRING);
if (t) {
d = t;
}
d->type |= DATA;
d->dtime = info.mtime;
d->dsize = info.size;
apr_hash_set(h, d->basename, APR_HASH_KEY_STRING, d);
}
}
if (interrupted) {
return 1;
}
path[baselen] = '\0';
for (i = apr_hash_first(p, h); i && !interrupted; i = apr_hash_next(i)) {
void *hvalue;
apr_uint32_t format;
apr_hash_this(i, NULL, NULL, &hvalue);
d = hvalue;
switch(d->type) {
case HEADERDATA:
nextpath = apr_pstrcat(p, path, "/", d->basename,
CACHE_HEADER_SUFFIX, NULL);
if (apr_file_open(&fd, nextpath, APR_FOPEN_READ | APR_FOPEN_BINARY,
APR_OS_DEFAULT, p) == APR_SUCCESS) {
len = sizeof(format);
if (apr_file_read_full(fd, &format, len,
&len) == APR_SUCCESS) {
if (format == DISK_FORMAT_VERSION) {
apr_off_t offset = 0;
apr_file_seek(fd, APR_SET, &offset);
len = sizeof(disk_cache_info_t);
if (apr_file_read_full(fd, &disk_info, len,
&len) == APR_SUCCESS) {
apr_file_close(fd);
e = apr_palloc(pool, sizeof(ENTRY));
APR_RING_INSERT_TAIL(&root, e, _entry, link);
e->expire = disk_info.expire;
e->response_time = disk_info.response_time;
e->htime = d->htime;
e->dtime = d->dtime;
e->hsize = d->hsize;
e->dsize = d->dsize;
e->basename = apr_pstrdup(pool, d->basename);
break;
}
else {
apr_file_close(fd);
}
}
else if (format == VARY_FORMAT_VERSION) {
/* This must be a URL that added Vary headers later,
* so kill the orphaned .data file
*/
apr_file_close(fd);
apr_file_remove(apr_pstrcat(p, path, "/", d->basename,
CACHE_DATA_SUFFIX, NULL),
p);
}
}
else {
apr_file_close(fd);
}
}
/* we have a somehow unreadable headers file which is associated
* with a data file. this may be caused by apache currently
* rewriting the headers file. thus we may delete the file set
* either in realclean mode or if the headers file modification
* timestamp is not within a specified positive or negative offset
* to the current time.
*/
current = apr_time_now();
if (realclean || d->htime < current - deviation
|| d->htime > current + deviation) {
delete_entry(path, d->basename, p);
unsolicited += d->hsize;
unsolicited += d->dsize;
}
break;
/* single data and header files may be deleted either in realclean
* mode or if their modification timestamp is not within a
* specified positive or negative offset to the current time.
* this handling is necessary due to possible race conditions
* between apache and this process
*/
case HEADER:
current = apr_time_now();
nextpath = apr_pstrcat(p, path, "/", d->basename,
CACHE_HEADER_SUFFIX, NULL);
if (apr_file_open(&fd, nextpath, APR_FOPEN_READ | APR_FOPEN_BINARY,
APR_OS_DEFAULT, p) == APR_SUCCESS) {
len = sizeof(format);
if (apr_file_read_full(fd, &format, len,
&len) == APR_SUCCESS) {
if (format == VARY_FORMAT_VERSION) {
apr_time_t expires;
len = sizeof(expires);
apr_file_read_full(fd, &expires, len, &len);
apr_file_close(fd);
if (expires < current) {
delete_entry(path, d->basename, p);
}
break;
}
}
apr_file_close(fd);
}
if (realclean || d->htime < current - deviation
|| d->htime > current + deviation) {
delete_entry(path, d->basename, p);
unsolicited += d->hsize;
}
break;
case DATA:
current = apr_time_now();
if (realclean || d->dtime < current - deviation
|| d->dtime > current + deviation) {
delete_entry(path, d->basename, p);
unsolicited += d->dsize;
}
break;
/* temp files may only be deleted in realclean mode which
* is asserted above if a tempfile is in the hash array
*/
case TEMP:
delete_file(path, d->basename, p);
unsolicited += d->dsize;
break;
}
}
if (interrupted) {
return 1;
}
apr_pool_destroy(p);
if (benice) {
apr_sleep(NICE_DELAY);
}
if (interrupted) {
return 1;
}
return 0;
}
static int iterate_over_clients(struct conf **globalcs,
struct strlist *grouplist, const char *ext, unsigned int maxlinks)
{
int ret=0;
DIR *dirp=NULL;
struct conf **cconfs=NULL;
struct dirent *dirinfo=NULL;
const char *globalclientconfdir=get_string(globalcs[OPT_CLIENTCONFDIR]);
signal(SIGABRT, &sighandler);
signal(SIGTERM, &sighandler);
signal(SIGINT, &sighandler);
if(!(cconfs=confs_alloc())) return -1;
if(confs_init(cconfs)) return -1;
if(!(dirp=opendir(globalclientconfdir)))
{
logp("Could not opendir '%s': %s\n",
globalclientconfdir, strerror(errno));
return 0;
}
while((dirinfo=readdir(dirp)))
{
char *lockfile=NULL;
char *lockfilebase=NULL;
char *client_lockdir=NULL;
struct lock *lock=NULL;
if(dirinfo->d_ino==0
// looks_like...() also avoids '.' and '..'.
|| looks_like_tmp_or_hidden_file(dirinfo->d_name)
|| !is_regular_file(globalclientconfdir, dirinfo->d_name))
continue;
confs_free_content(cconfs);
if(confs_init(cconfs)) return -1;
if(set_string(cconfs[OPT_CNAME], dirinfo->d_name))
return -1;
if(conf_load_clientconfdir(globalcs, cconfs))
{
logp("could not load config for client %s\n",
dirinfo->d_name);
return 0;
}
if(grouplist)
{
const char *dedup_group=
get_string(cconfs[OPT_DEDUP_GROUP]);
if(!dedup_group
|| !in_group(grouplist, dedup_group))
continue;
}
if(!(client_lockdir=get_string(cconfs[OPT_CLIENT_LOCKDIR])))
client_lockdir=get_string(cconfs[OPT_DIRECTORY]);
if(!(lockfilebase=prepend(client_lockdir,
dirinfo->d_name, "/"))
|| !(lockfile=prepend(lockfilebase,
BEDUP_LOCKFILE_NAME, "/")))
{
free_w(&lockfilebase);
free_w(&lockfile);
ret=-1;
break;
}
free_w(&lockfilebase);
if(!(lock=lock_alloc_and_init(lockfile)))
{
ret=-1;
break;
}
lock_get(lock);
free_w(&lockfile);
if(lock->status!=GET_LOCK_GOT)
{
logp("Could not get %s\n", lock->path);
continue;
}
logp("Got %s\n", lock->path);
// Remember that we got that lock.
lock_add_to_list(&locklist, lock);
if(process_dir(get_string(cconfs[OPT_DIRECTORY]),
dirinfo->d_name,
ext, maxlinks, 1 /* burp mode */, 0 /* level */))
{
ret=-1;
break;
}
ccount++;
}
closedir(dirp);
locks_release_and_free(&locklist);
confs_free(&cconfs);
return ret;
}
int run_bedup(int argc, char *argv[])
{
int i=1;
int ret=0;
int option=0;
int nonburp=0;
unsigned int maxlinks=DEF_MAX_LINKS;
char *groups=NULL;
char ext[16]="";
int givenconfigfile=0;
const char *configfile=NULL;
configfile=get_config_path();
snprintf(ext, sizeof(ext), ".bedup.%d", getpid());
while((option=getopt(argc, argv, "c:dg:hlm:nvV?"))!=-1)
{
switch(option)
{
case 'c':
configfile=optarg;
givenconfigfile=1;
break;
case 'd':
deletedups=1;
break;
case 'g':
groups=optarg;
break;
case 'l':
makelinks=1;
break;
case 'm':
maxlinks=atoi(optarg);
break;
case 'n':
nonburp=1;
break;
case 'V':
printf("%s-%s\n", prog, VERSION);
return 0;
case 'v':
verbose=1;
break;
case 'h':
case '?':
return usage();
}
}
if(nonburp && givenconfigfile)
{
logp("-n and -c options are mutually exclusive\n");
return 1;
}
if(nonburp && groups)
{
logp("-n and -g options are mutually exclusive\n");
return 1;
}
if(!nonburp && maxlinks!=DEF_MAX_LINKS)
{
logp("-m option is specified via the configuration file in burp mode (max_hardlinks=)\n");
return 1;
}
if(deletedups && makelinks)
{
logp("-d and -l options are mutually exclusive\n");
return 1;
}
if(deletedups && !nonburp)
{
logp("-d option requires -n option\n");
return 1;
}
if(optind>=argc)
{
if(nonburp)
{
logp("No directories found after options\n");
return 1;
}
}
else
{
if(!nonburp)
{
logp("Do not specify extra arguments.\n");
return 1;
}
}
if(maxlinks<2)
{
logp("The argument to -m needs to be greater than 1.\n");
return 1;
}
if(nonburp)
{
// Read directories from command line.
for(i=optind; i<argc; i++)
{
// Strip trailing slashes, for tidiness.
if(argv[i][strlen(argv[i])-1]=='/')
argv[i][strlen(argv[i])-1]='\0';
if(process_dir("", argv[i], ext, maxlinks,
0 /* not burp mode */, 0 /* level */))
{
ret=1;
break;
}
}
}
else
{
struct conf **globalcs=NULL;
struct strlist *grouplist=NULL;
struct lock *globallock=NULL;
if(groups)
{
char *tok=NULL;
if((tok=strtok(groups, ",\n")))
{
do
{
if(strlist_add(&grouplist, tok, 1))
{
log_out_of_memory(__func__);
return -1;
}
} while((tok=strtok(NULL, ",\n")));
}
if(!grouplist)
{
logp("unable to read list of groups\n");
return -1;
}
}
// Read directories from config files, and get locks.
if(!(globalcs=confs_alloc())) return -1;
if(confs_init(globalcs)) return -1;
if(conf_load_global_only(configfile, globalcs)) return 1;
if(get_e_burp_mode(globalcs[OPT_BURP_MODE])!=BURP_MODE_SERVER)
{
logp("%s is not a server config file\n", configfile);
confs_free(&globalcs);
return 1;
}
logp("Dedup clients from %s\n",
get_string(globalcs[OPT_CLIENTCONFDIR]));
maxlinks=get_int(globalcs[OPT_MAX_HARDLINKS]);
if(grouplist)
{
struct strlist *g=NULL;
logp("in dedup groups:\n");
for(g=grouplist; g; g=g->next)
logp("%s\n", g->path);
}
else
{
char *lockpath=NULL;
// Only get the global lock when doing a global run.
// If you are doing individual groups, you are likely
// to want to do many different dedup jobs and a
// global lock would get in the way.
if(!(lockpath=prepend(
get_string(globalcs[OPT_LOCKFILE]),
".bedup", ""))
|| !(globallock=lock_alloc_and_init(lockpath)))
return 1;
lock_get(globallock);
if(globallock->status!=GET_LOCK_GOT)
{
logp("Could not get lock %s (%d)\n", lockpath,
globallock->status);
free_w(&lockpath);
return 1;
}
logp("Got %s\n", lockpath);
}
ret=iterate_over_clients(globalcs, grouplist, ext, maxlinks);
confs_free(&globalcs);
lock_release(globallock);
lock_free(&globallock);
strlists_free(&grouplist);
}
if(!nonburp)
{
logp("%d client storages scanned\n", ccount);
}
logp("%llu duplicate %s found\n",
count, count==1?"file":"files");
logp("%llu bytes %s%s\n",
savedbytes, (makelinks || deletedups)?"saved":"saveable",
bytes_to_human(savedbytes));
return ret;
}
/*
* cmdc() -- command completion function.
*
* cmdc takes a char* and returns a GList* of possible completions.
*
* Initial version by Travis Hume <*****@*****.**>.
*
*/
static GList *
cmdc( char *s )
{
GCompletion *completion = NULL;
GList *ret_list = NULL;
static GHashTable *path_hash = NULL;
static char *path = NULL;
gchar *path_elem;
struct stat buf;
static gboolean inited = FALSE;
gpointer hash_key = NULL;
/*
* Only want to build the GCompletion once. At some point I'd like to add
* code to refresh the GCompletion, either at a regular interval, or when
* there is a completion failure, ...
*
*/
if(!inited)
{
/* Make a local copy of the path variable. Otherwise the path
environment variable would be modified. */
path = (char *) malloc(sizeof(char) * (strlen(getenv("PATH")) + 1));
strcpy(path, getenv("PATH"));
path_hash = g_hash_table_new( g_str_hash, g_str_equal );
for( path_elem = strtok( path, ":" ); path_elem;
path_elem = strtok( NULL, ":" ))
{
if( stat( path_elem, &buf ))
continue;
if( buf.st_mode & S_IFDIR )
{
/* keep a hash of processed paths, to avoid reprocessing
* dupped path entries.
*/
hash_key = g_hash_table_lookup( path_hash, path_elem );
if( hash_key )
continue; /* duplicate $PATH entry */
else
{
g_hash_table_insert(
path_hash, (gpointer)path_elem, (gpointer)path_elem );
process_dir( path_elem );
}
}
}
/* atexit() we want to free the completion. */
g_atexit( cleanup );
inited = TRUE;
}
completion = g_completion_new( NULL );
g_completion_add_items( completion, path_elements );
ret_list = g_list_copy( g_completion_complete( completion, s, NULL ));
g_completion_free( completion );
return g_list_sort( ret_list, (GCompareFunc)g_list_str_cmp );
}
static int process_dir(const char *oldpath, const char *newpath,
const char *ext, unsigned int maxlinks, int burp_mode, int level)
{
int ret=-1;
DIR *dirp=NULL;
char *path=NULL;
struct stat info;
struct dirent *dirinfo=NULL;
struct file newfile;
struct mystruct *find=NULL;
static char working[256]="";
static char finishing[256]="";
newfile.path=NULL;
if(!(path=prepend(oldpath, newpath, "/"))) goto end;
if(burp_mode && level==0)
{
if(get_link(path, "working", working, sizeof(working))
|| get_link(path, "finishing", finishing, sizeof(finishing)))
goto end;
}
if(!(dirp=opendir(path)))
{
logp("Could not opendir '%s': %s\n", path, strerror(errno));
ret=0;
goto end;
}
while((dirinfo=readdir(dirp)))
{
if(!strcmp(dirinfo->d_name, ".")
|| !strcmp(dirinfo->d_name, ".."))
continue;
//printf("try %s\n", dirinfo->d_name);
if(burp_mode)
{
if(level==0)
{
/* Be careful not to try to dedup the lockfiles.
The lock actually gets lost if you open one to do a
checksum
and then close it. This caused me major headaches to
figure out. */
if(!strcmp(dirinfo->d_name, LOCKFILE_NAME)
|| !strcmp(dirinfo->d_name, BEDUP_LOCKFILE_NAME))
continue;
/* Skip places where backups are going on. */
if(!strcmp(dirinfo->d_name, working)
|| !strcmp(dirinfo->d_name, finishing))
continue;
if(!strcmp(dirinfo->d_name, "deleteme"))
continue;
}
else if(level==1)
{
// Do not dedup stuff that might be appended to later.
if(!strncmp(dirinfo->d_name, "log",
strlen("log"))
|| !strncmp(dirinfo->d_name, "verifylog",
strlen("verifylog"))
|| !strncmp(dirinfo->d_name, "restorelog",
strlen("restorelog")))
continue;
}
}
free_w(&newfile.path);
if(!(newfile.path=prepend(path, dirinfo->d_name, "/")))
goto end;
if(lstat(newfile.path, &info))
continue;
if(S_ISDIR(info.st_mode))
{
if(process_dir(path, dirinfo->d_name, ext, maxlinks, burp_mode, level+1))
goto end;
continue;
}
else if(!S_ISREG(info.st_mode)
|| !info.st_size) // ignore zero-length files
continue;
newfile.dev=info.st_dev;
newfile.ino=info.st_ino;
newfile.nlink=info.st_nlink;
newfile.full_cksum=0;
newfile.part_cksum=0;
newfile.next=NULL;
//printf("%s\n", newfile.path);
if((find=find_key(info.st_size)))
{
if(check_files(find, &newfile, &info, ext, maxlinks))
goto end;
}
else
{
//printf("add: %s\n", newfile.path);
if(add_key(info.st_size, &newfile))
goto end;
}
}
ret=0;
end:
closedir(dirp);
free_w(&newfile.path);
free_w(&path);
return ret;
}
static int
process_path (char *pathname, char *name, bool leaf, char *parent,
mode_t mode, ino_t inum)
{
struct stat stat_buf;
static dev_t root_dev; /* Device ID of current argument pathname. */
int i;
struct predicate *eval_tree;
eval_tree = get_eval_tree ();
/* Assume it is a non-directory initially. */
stat_buf.st_mode = 0;
/* The caller usually knows the inode number, either from readdir or
* a *stat call. We use that value (the caller passes 0 to indicate
* ignorance of the inode number).
*/
stat_buf.st_ino = inum;
state.rel_pathname = name;
state.type = 0;
state.have_stat = false;
state.have_type = false;
state.already_issued_stat_error_msg = false;
if (!digest_mode (&mode, pathname, name, &stat_buf, leaf))
return 0;
if (!S_ISDIR (state.type))
{
if (state.curdepth >= options.mindepth)
apply_predicate (pathname, &stat_buf, eval_tree);
return 0;
}
/* From here on, we're working on a directory. */
/* Now we really need to stat the directory, even if we know the
* type, because we need information like struct stat.st_rdev.
*/
if (get_statinfo (pathname, name, &stat_buf) != 0)
return 0;
state.have_stat = true;
mode = state.type = stat_buf.st_mode; /* use full info now that we have it. */
state.stop_at_current_level =
options.maxdepth >= 0
&& state.curdepth >= options.maxdepth;
/* If we've already seen this directory on this branch,
don't descend it again. */
for (i = 0; i <= dir_curr; i++)
if (stat_buf.st_ino == dir_ids[i].ino &&
stat_buf.st_dev == dir_ids[i].dev)
{
state.stop_at_current_level = true;
issue_loop_warning (name, pathname, i);
}
if (dir_alloc <= ++dir_curr)
{
dir_alloc += DIR_ALLOC_STEP;
dir_ids = (struct dir_id *)
xrealloc ((char *) dir_ids, dir_alloc * sizeof (struct dir_id));
}
dir_ids[dir_curr].ino = stat_buf.st_ino;
dir_ids[dir_curr].dev = stat_buf.st_dev;
if (options.stay_on_filesystem)
{
if (state.curdepth == 0)
root_dev = stat_buf.st_dev;
else if (stat_buf.st_dev != root_dev)
state.stop_at_current_level = true;
}
if (options.do_dir_first && state.curdepth >= options.mindepth)
apply_predicate (pathname, &stat_buf, eval_tree);
if (options.debug_options & DebugSearch)
fprintf (stderr, "pathname = %s, stop_at_current_level = %d\n",
pathname, state.stop_at_current_level);
if (state.stop_at_current_level == false)
{
/* Scan directory on disk. */
process_dir (pathname, name, strlen (pathname), &stat_buf, parent);
}
if (options.do_dir_first == false && state.curdepth >= options.mindepth)
{
/* The fields in 'state' are now out of date. Correct them.
*/
if (!digest_mode (&mode, pathname, name, &stat_buf, leaf))
return 0;
if (0 == dir_curr)
{
at_top (pathname, mode, stat_buf.st_ino, &stat_buf,
do_process_predicate);
}
else
{
do_process_predicate (pathname, name, mode, stat_buf.st_ino,
&stat_buf);
}
}
dir_curr--;
return 1;
}
// Return 0 for directory processed, -1 for error, 1 for not processed.
static int process_dir(const char *oldpath, const char *newpath,
const char *ext, unsigned int maxlinks, int burp_mode, int level)
{
int ret=-1;
DIR *dirp=NULL;
char *path=NULL;
struct stat info;
struct dirent *dirinfo=NULL;
struct file newfile;
struct mystruct *find=NULL;
static char working[256]="";
static char finishing[256]="";
newfile.path=NULL;
if(!(path=prepend_s(oldpath, newpath))) goto end;
if(burp_mode && level==0)
{
if(get_link(path, "working", working, sizeof(working))
|| get_link(path, "finishing", finishing, sizeof(finishing)))
goto end;
if(!looks_like_protocol1(path))
{
logp("%s does not look like a protocol 1 storage directory - skipping\n", path);
ret=1;
goto end;
}
}
if(!(dirp=opendir(path)))
{
logp("Could not opendir '%s': %s\n", path, strerror(errno));
ret=1;
goto end;
}
while((dirinfo=readdir(dirp)))
{
if(!strcmp(dirinfo->d_name, ".")
|| !strcmp(dirinfo->d_name, ".."))
continue;
//printf("try %s\n", dirinfo->d_name);
if(burp_mode
&& level_exclusion(level, dirinfo->d_name,
working, finishing))
continue;
free_w(&newfile.path);
if(!(newfile.path=prepend_s(path, dirinfo->d_name)))
goto end;
if(lstat(newfile.path, &info))
continue;
if(S_ISDIR(info.st_mode))
{
if(process_dir(path, dirinfo->d_name, ext, maxlinks, burp_mode, level+1))
goto end;
continue;
}
else if(!S_ISREG(info.st_mode)
|| !info.st_size) // ignore zero-length files
continue;
newfile.dev=info.st_dev;
newfile.ino=info.st_ino;
newfile.nlink=info.st_nlink;
newfile.full_cksum=0;
newfile.part_cksum=0;
newfile.next=NULL;
if((find=find_key(info.st_size)))
{
//printf("check %d: %s\n", info.st_size, newfile.path);
if(check_files(find, &newfile, &info, ext, maxlinks))
goto end;
}
else
{
//printf("add: %s\n", newfile.path);
if(add_key(info.st_size, &newfile))
goto end;
}
}
ret=0;
end:
closedir(dirp);
free_w(&newfile.path);
free_w(&path);
return ret;
}
void
load_database(cron_db *old_db) {
struct stat spool_stat, syscron_stat, crond_stat;
cron_db new_db;
user *u, *nu;
time_t new_mtime;
Debug(DLOAD, ("[%ld] load_database()\n", (long)getpid()));
/* before we start loading any data, do a stat on SPOOL_DIR
* so that if anything changes as of this moment (i.e., before we've
* cached any of the database), we'll see the changes next time.
*/
if (stat(SPOOL_DIR, &spool_stat) < OK) {
log_it("CRON", getpid(), "STAT FAILED", SPOOL_DIR);
(void) exit(ERROR_EXIT);
}
/* track system crontab directory
*/
if (stat(CROND_DIR, &crond_stat) < OK)
crond_stat.st_mtime = 0;
/* track system crontab file
*/
if (stat(SYSCRONTAB, &syscron_stat) < OK)
syscron_stat.st_mtime = 0;
/* if spooldir's mtime has not changed, we don't need to fiddle with
* the database.
*
* Note that old_db->mtime is initialized to 0 in main(), and
* so is guaranteed to be different than the stat() mtime the first
* time this function is called.
*/
new_mtime = TMAX(crond_stat.st_mtime, TMAX(spool_stat.st_mtime,
syscron_stat.st_mtime));
if (old_db->mtime == new_mtime) {
Debug(DLOAD, ("[%ld] spool dir mtime unch, no load needed.\n",
(long)getpid()));
return;
}
/* something's different. make a new database, moving unchanged
* elements from the old database, reloading elements that have
* actually changed. Whatever is left in the old database when
* we're done is chaff -- crontabs that disappeared.
*/
new_db.mtime = new_mtime;
new_db.head = new_db.tail = NULL;
if (syscron_stat.st_mtime)
process_crontab("root", NULL, SYSCRONTAB, &syscron_stat,
&new_db, old_db);
if (crond_stat.st_mtime)
process_dir(CROND_DIR, &crond_stat, 1, &new_db, old_db);
process_dir(SPOOL_DIR, &spool_stat, 0, &new_db, old_db);
/* if we don't do this, then when our children eventually call
* getpwnam() in do_command.c's child_process to verify MAILTO=,
* they will screw us up (and v-v).
*/
endpwent();
/* whatever's left in the old database is now junk.
*/
Debug(DLOAD, ("unlinking old database:\n"));
for (u = old_db->head; u != NULL; u = nu) {
Debug(DLOAD, ("\t%s\n", u->name));
nu = u->next;
unlink_user(old_db, u);
free_user(u);
}
/* overwrite the database control block with the new one.
*/
*old_db = new_db;
Debug(DLOAD, ("load_database is done\n"));
}
static int iterate_over_clients(struct config *conf, strlist_t **grouplist, int gcount, const char *ext, unsigned int maxlinks)
{
int ret=0;
DIR *dirp=NULL;
struct dirent *dirinfo=NULL;
signal(SIGABRT, &sighandler);
signal(SIGTERM, &sighandler);
signal(SIGINT, &sighandler);
if(!(dirp=opendir(conf->clientconfdir)))
{
logp("Could not opendir '%s': %s\n",
conf->clientconfdir, strerror(errno));
return 0;
}
while((dirinfo=readdir(dirp)))
{
char *lockfile=NULL;
char *lockfilebase=NULL;
if(!strcmp(dirinfo->d_name, ".")
|| !strcmp(dirinfo->d_name, "..")
|| looks_like_vim_tmpfile(dirinfo->d_name)
|| !is_regular_file(conf->clientconfdir, dirinfo->d_name))
continue;
if(gcount)
{
int ig=0;
if((ig=in_group(conf->clientconfdir,
dirinfo->d_name, grouplist, gcount, conf))<0)
{
ret=-1;
break;
}
if(!ig) continue;
}
if(!(lockfilebase=prepend(conf->client_lockdir,
dirinfo->d_name, "/"))
|| !(lockfile=prepend(lockfilebase,
BEDUP_LOCKFILE_NAME, "/")))
{
if(lockfilebase) free(lockfilebase);
if(lockfile) free(lockfile);
ret=-1;
break;
}
free(lockfilebase);
if(get_lock(lockfile))
{
logp("Could not get %s\n", lockfile);
free(lockfile);
continue;
}
// Remember that we got that lock.
if(strlist_add(&locklist, &lockcount, lockfile, 1))
{
free(lockfile);
lockcount=0;
break;
}
logp("Got %s\n", lockfile);
if(process_dir(conf->directory, dirinfo->d_name,
ext, maxlinks, 1 /* burp mode */, 0 /* level */))
{
ret=-1;
break;
}
ccount++;
}
closedir(dirp);
remove_locks();
return ret;
}
int main(int argc, char *argv[])
{
int i=1;
int ret=0;
int option=0;
int nonburp=0;
unsigned int maxlinks=DEF_MAX_LINKS;
char *groups=NULL;
char ext[16]="";
int givenconfigfile=0;
prog=basename(argv[0]);
init_log(prog);
const char *configfile=NULL;
configfile=get_config_path();
snprintf(ext, sizeof(ext), ".bedup.%d", getpid());
while((option=getopt(argc, argv, "c:g:hlmnv?"))!=-1)
{
switch(option)
{
case 'c':
configfile=optarg;
givenconfigfile=1;
break;
case 'g':
groups=optarg;
break;
case 'l':
makelinks=1;
break;
case 'm':
maxlinks=atoi(optarg);
break;
case 'n':
nonburp=1;
break;
case 'v':
printf("%s-%s\n", prog, VERSION);
return 0;
case 'h':
case '?':
return usage();
}
}
if(nonburp && givenconfigfile)
{
logp("-n and -c options are mutually exclusive\n");
return 1;
}
if(nonburp && groups)
{
logp("-n and -g options are mutually exclusive\n");
return 1;
}
if(!nonburp && maxlinks!=DEF_MAX_LINKS)
{
logp("-m option is specified via the configuration file in burp mode (max_hardlinks=)\n");
return 1;
}
if(optind>=argc)
{
if(nonburp)
{
logp("No directories found after options\n");
return 1;
}
}
else
{
if(!nonburp)
{
logp("Do not specify extra arguments.\n");
return 1;
}
}
if(maxlinks<2)
{
logp("The argument to -m needs to be greater than 1.\n");
return 1;
}
if(nonburp)
{
// Read directories from command line.
for(i=optind; i<argc; i++)
{
// Strip trailing slashes, for tidiness.
if(argv[i][strlen(argv[i])-1]=='/')
argv[i][strlen(argv[i])-1]='\0';
if(process_dir("", argv[i], ext, maxlinks,
0 /* not burp mode */, 0 /* level */))
{
ret=1;
break;
}
}
}
else
{
int gcount=0;
struct config conf;
char *globallock=NULL;
struct strlist **grouplist=NULL;
if(groups)
{
char *tok=NULL;
if((tok=strtok(groups, ",\n")))
{
do
{
if(strlist_add(&grouplist, &gcount,
tok, 1))
{
logp("out of memory\n");
return -1;
}
} while((tok=strtok(NULL, ",\n")));
}
if(!gcount)
{
logp("unable to read list of groups\n");
return -1;
}
}
// Read directories from config files, and get locks.
init_config(&conf);
if(load_config(configfile, &conf, 1)) return 1;
if(conf.mode!=MODE_SERVER)
{
logp("%s is not a server config file\n", configfile);
free_config(&conf);
return 1;
}
logp("Dedup clients from %s\n", conf.clientconfdir);
maxlinks=conf.max_hardlinks;
if(gcount)
{
logp("in dedup groups:\n");
for(i=0; i<gcount; i++)
logp("%s\n", grouplist[i]->path);
}
else
{
// Only get the global lock when doing a global run.
// If you are doing individual groups, you are likely
// to want to do many different dedup jobs and a
// global lock would get in the way.
if(!(globallock=prepend(conf.lockfile, ".bedup", "")))
return 1;
if(get_lock(globallock))
{
logp("Could not get %s\n", globallock);
return 1;
}
logp("Got %s\n", globallock);
}
ret=iterate_over_clients(&conf, grouplist, gcount,
ext, maxlinks);
free_config(&conf);
if(globallock)
{
unlink(globallock);
free(globallock);
}
}
if(!nonburp)
{
logp("%d client storages scanned\n", ccount);
}
logp("%llu duplicate %s found\n",
count, count==1?"file":"files");
logp("%llu bytes %s%s\n",
savedbytes, makelinks?"saved":"saveable",
bytes_to_human(savedbytes));
return ret;
}
