void ExpireDir(char *dir, dirstat_t *dirstat, uint64_t maxsize, uint64_t maxlife, uint32_t runtime ) { FTS *fts; FTSENT *ftsent; uint64_t sizelimit, num_expired; int done, size_done, lifetime_done, dir_files; char *const path[] = { dir, NULL }; char *expire_timelimit = NULL; time_t now = time(NULL); dir_files = 0; if ( dirstat->low_water == 0 ) dirstat->low_water = 95; if ( runtime ) { SetupSignalHandler(); alarm(runtime); } if ( maxlife ) { // build an appropriate string for comparing time_t t_expire = now - maxlife; time_t t_watermark = now - (time_t)((maxlife * dirstat->low_water)/100); // printf("Expire files before %s", ctime(&t_expire)); expire_timelimit = strdup(UNIX2ISO(t_watermark)); // printf("down to %s", ctime(&t_watermark)); // printf("Diff: %i\n", t_watermark - t_expire ); if ( dirstat->last < t_expire && (isatty(STDIN_FILENO) ) ) { // this means all files will get expired - are you sure ? char *s, s1[32], s2[32]; time_t t; struct tm *when; t = t_expire; when = localtime(&t); strftime(s1, 31, "%Y-%m-%d %H:%M:%S", when); s1[31] = '\0'; t = dirstat->last; when = localtime(&t); strftime(s2, 31, "%Y-%m-%d %H:%M:%S", when); s2[31] = '\0'; printf("Your max lifetime of %s will expire all file before %s\n", ScaleTime(maxlife), s1); printf("Your latest files are dated %s. This means all files will be deleted!\n", s2); printf("Are you sure? yes/[no] "); s = fgets(s1, 31, stdin); s1[31] = '\0'; if ( s && strncasecmp(s1, "yes\n", 31) == 0 ) { printf("Ok - you've beeen warned!\n"); } else { printf("Expire canceled!\n"); return; } } } done = 0; size_done = maxsize == 0 || dirstat->filesize < maxsize; lifetime_done = maxlife == 0 || ( now - dirstat->first ) < maxlife; sizelimit = (dirstat->low_water * maxsize)/100; num_expired = 0; fts = fts_open(path, FTS_LOGICAL, compare); while ( !done && ((ftsent = fts_read(fts)) != NULL) ) { if ( ftsent->fts_info == FTS_F ) { dir_files++; // count files in directories if ( ftsent->fts_namelen == 19 && strncmp(ftsent->fts_name, "nfcapd.", 7) == 0 ) { // nfcapd.200604301200 strlen = 19 char *s, *p = &(ftsent->fts_name[7]); // process only nfcapd. files // make sure it's really an nfcapd. file and we have // only digits in the rest of the file name s = p; while ( *s ) { if ( *s < '0' || *s > '9' ) break; s++; } // otherwise skip if ( *s ) continue; // expire size-wise if needed if ( !size_done ) { if ( dirstat->filesize > sizelimit ) { if ( unlink(ftsent->fts_path) == 0 ) { dirstat->filesize -= 512 * ftsent->fts_statp->st_blocks; num_expired++; dir_files--; } else { LogError( "unlink() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); } continue; // next file if file was unlinked } else { dirstat->first = ISO2UNIX(p); // time of first file not expired size_done = 1; } } // expire time-wise if needed // this part of the code is executed only when size-wise is fullfilled if ( !lifetime_done ) { if ( expire_timelimit && strcmp(p, expire_timelimit) < 0 ) { if ( unlink(ftsent->fts_path) == 0 ) { dirstat->filesize -= 512 * ftsent->fts_statp->st_blocks; num_expired++; dir_files--; } else { LogError( "unlink() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); } lifetime_done = 0; } else { dirstat->first = ISO2UNIX(p); // time of first file not expired lifetime_done = 1; } } done = (size_done && lifetime_done) || timeout; } } else { switch (ftsent->fts_info) { case FTS_D: // set pre-order flag dir_files = 0; // skip all '.' entries as well as hidden directories if ( ftsent->fts_level > 0 && ftsent->fts_name[0] == '.' ) fts_set(fts, ftsent, FTS_SKIP); // any valid directory needs to start with a digit ( %Y -> year ) if ( ftsent->fts_level > 0 && !isdigit(ftsent->fts_name[0]) ) fts_set(fts, ftsent, FTS_SKIP); break; case FTS_DP: // do not delete base data directory ( level == 0 ) if ( dir_files == 0 && ftsent->fts_level > 0 ) { // directory is empty and can be deleted // printf("Will remove directory %s\n", ftsent->fts_path); if ( rmdir(ftsent->fts_path) != 0 ) { LogError( "rmdir() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); } } break; } } } fts_close(fts); if ( !done ) { // all files expired and limits not reached // this may be possible, when files get time-wise expired and // the time limit is shorter than the latest file dirstat->first = dirstat->last; } if ( runtime ) alarm(0); if ( timeout ) { LogError( "Maximum execution time reached! Interrupt expire.\n"); } if ( num_expired > dirstat->numfiles ) { LogError( "Error updating stat record: Number of files inconsistent!\n"); LogError( "Will automatically rebuild this directory next time\n"); dirstat->numfiles = 0; dirstat->status = FORCE_REBUILD; } else { dirstat->numfiles -= num_expired; } if ( dirstat->numfiles == 0 ) { dirstat->first = dirstat->last = time(NULL); dirstat->status = FORCE_REBUILD; } free(expire_timelimit); } // End of ExpireDir
static void do_expire(char *datadir) { bookkeeper_t *books; dirstat_t *dirstat, oldstat; int ret, bookkeeper_stat, do_rescan; syslog(LOG_INFO, "Run expire on '%s'", datadir); do_rescan = 0; ret = ReadStatInfo(datadir, &dirstat, CREATE_AND_LOCK); switch (ret) { case STATFILE_OK: break; case ERR_NOSTATFILE: dirstat->low_water = 95; case FORCE_REBUILD: syslog(LOG_INFO, "Force rebuild stat record"); do_rescan = 1; break; case ERR_FAIL: syslog(LOG_ERR, "expire failed: can't read stat record"); return; /* not reached */ break; default: syslog(LOG_ERR, "expire failed: unexpected return code %i reading stat record", ret); return; /* not reached */ } bookkeeper_stat = AccessBookkeeper(&books, datadir); if ( do_rescan ) { RescanDir(datadir, dirstat); if ( bookkeeper_stat == BOOKKEEPER_OK ) { ClearBooks(books, NULL); // release the books below } } if ( bookkeeper_stat == BOOKKEEPER_OK ) { bookkeeper_t tmp_books; ClearBooks(books, &tmp_books); UpdateBookStat(dirstat, &tmp_books); ReleaseBookkeeper(books, DETACH_ONLY); } else { syslog(LOG_ERR, "Error %i: can't access book keeping records", ret); } syslog(LOG_INFO, "Limits: Filesize %s, Lifetime %s, Watermark: %llu%%\n", dirstat->max_size ? ScaleValue(dirstat->max_size) : "<none>", dirstat->max_lifetime ? ScaleTime(dirstat->max_lifetime) : "<none>", (unsigned long long)dirstat->low_water); syslog(LOG_INFO, "Current size: %s, Current lifetime: %s, Number of files: %llu", ScaleValue(dirstat->filesize), ScaleTime(dirstat->last - dirstat->first), (unsigned long long)dirstat->numfiles); oldstat = *dirstat; if ( dirstat->max_size || dirstat->max_lifetime ) ExpireDir(datadir, dirstat, dirstat->max_size, dirstat->max_lifetime, 0); WriteStatInfo(dirstat); if ( (oldstat.numfiles - dirstat->numfiles) > 0 ) { syslog(LOG_INFO, "expire completed"); syslog(LOG_INFO, " expired files: %llu", (unsigned long long)(oldstat.numfiles - dirstat->numfiles)); syslog(LOG_INFO, " expired time slot: %s", ScaleTime(dirstat->first - oldstat.first)); syslog(LOG_INFO, " expired file size: %s", ScaleValue(oldstat.filesize - dirstat->filesize)); syslog(LOG_INFO, "New size: %s, New lifetime: %s, Number of files: %llu", ScaleValue(dirstat->filesize), ScaleTime(dirstat->last - dirstat->first), (unsigned long long)dirstat->numfiles); } else { syslog(LOG_INFO, "expire completed - nothing to expire."); } ReleaseStatInfo(dirstat); } // End of do_expire
int main( int argc, char **argv ) { struct stat fstat; int c, err, maxsize_set, maxlife_set; int do_rescan, do_expire, do_list, print_stat, do_update_param, print_books, is_profile, nfsen_format; char *maxsize_string, *lifetime_string, *datadir; uint64_t maxsize, lifetime, low_water; uint32_t runtime; channel_t *channel, *current_channel; maxsize_string = lifetime_string = NULL; datadir = NULL; maxsize = lifetime = 0; do_rescan = 0; do_expire = 0; do_list = 0; do_update_param = 0; is_profile = 0; print_stat = 0; print_books = 0; maxsize_set = 0; maxlife_set = 0; low_water = 0; nfsen_format = 0; runtime = 0; while ((c = getopt(argc, argv, "e:hl:L:T:Ypr:s:t:u:w:")) != EOF) { switch (c) { case 'h': usage(argv[0]); exit(0); break; case 'l': CheckDataDir(datadir); datadir = optarg; do_list = 1; print_stat = 1; break; case 'L': CheckDataDir(datadir); datadir = optarg; print_stat = 1; print_books = 1; break; case 'p': is_profile = 1; break; case 'r': CheckDataDir(datadir); do_rescan = 1; print_stat = 1; datadir = optarg; break; case 'e': CheckDataDir(datadir); datadir = optarg; do_expire = 1; print_stat = 1; break; case 's': if ( ParseSizeDef(optarg, &maxsize ) == 0 ) exit(250); maxsize_set = 1; break; case 't': if ( ParseTimeDef(optarg, &lifetime ) == 0 ) exit(250); maxlife_set = 1; break; case 'u': CheckDataDir(datadir); datadir = optarg; do_update_param = 1; break; case 'w': low_water = strtoll(optarg, NULL, 10); if ( low_water > 100 ) { fprintf(stderr, "Water mark > 100%%\n"); exit(250); } if ( low_water == 0 ) low_water = 100; break; case 'T': runtime = strtoll(optarg, NULL, 10); if ( runtime > 3600 ) { fprintf(stderr, "Runtime > 3600 (1h)\n"); exit(250); } break; case 'Y': nfsen_format = 1; break; default: usage(argv[0]); exit(250); } } datadir = AbsolutePath(datadir); if ( !datadir ) { fprintf(stderr, "Missing data directory\n"); usage(argv[0]); exit(250); } err = stat(datadir, &fstat); if ( !(fstat.st_mode & S_IFDIR) ) { fprintf(stderr, "No such directory: %s\n", datadir); exit(250); } channel = GetChannelList(datadir, is_profile, do_rescan); // GetChannelList(datadir, is_profile, do_rescan); if ( !channel ) { exit(250); } // printf("Size: %llu, time: %llu\n", maxsize, lifetime); // update water mark only, when not listing if ( !is_profile && !do_list && low_water ) channel->dirstat->low_water = low_water; /* process do_list first: if the UpdateBookStat results in a FORCE_REBUILD, * this will immediately done afterwards * do_expire will need accurate books as well, so update the books here as well */ if ( do_list || do_expire ) { current_channel = channel; while ( current_channel ) { if ( current_channel->books_stat == BOOKKEEPER_OK ) { bookkeeper_t tmp_books; printf("Include nfcapd bookeeping record in %s\n", current_channel->datadir); ClearBooks(current_channel->books, &tmp_books); UpdateBookStat(current_channel->dirstat, &tmp_books); if ( current_channel->dirstat->status == FORCE_REBUILD ) current_channel->do_rescan = 1; } current_channel = current_channel->next; } } // process do_rescan: make sure stats are up to date, if required current_channel = channel; while ( current_channel ) { if ( current_channel->do_rescan ) { int i; uint64_t last_sequence; /* detect new files: If nfcapd adds a new file while we are rescanning the directory * this results in inconsistent data for the rescan. Therefore check at the begin and end * of the rescan for the sequence number, which reflects the accesss/change to the bookkeeping record * It's assumed, that such an event does not occure more than once. However, loop 3 times max */ for ( i=0; i<3; i++ ) { last_sequence = BookSequence(current_channel->books); printf("Scanning files in %s .. ", current_channel->datadir); RescanDir(current_channel->datadir, current_channel->dirstat); if ( current_channel->dirstat->numfiles == 0 ) { //nothing found current_channel->status = NOFILES; } if ( BookSequence(current_channel->books) == last_sequence ) break; printf("Rescan again, due to file changes in directory!\n"); } if ( BookSequence(current_channel->books) != last_sequence ) { fprintf(stderr, "Could not savely rescan the directory. Data is not consistent.\n"); ReleaseBookkeeper(current_channel->books, DETACH_ONLY); if ( current_channel->status == OK ) WriteStatInfo(current_channel->dirstat); exit(250); } printf("done.\n"); if ( current_channel->books_stat == BOOKKEEPER_OK ) { printf("Updating nfcapd bookeeping records\n"); ClearBooks(channel->books, NULL); } } current_channel = current_channel->next; } // now process do_expire if required if ( do_expire ) { dirstat_t old_stat, current_stat; if ( is_profile ) { current_stat.status = 0; current_stat.max_lifetime = lifetime; current_stat.max_size = maxsize; current_stat.low_water = low_water ? low_water : 98; // sum up all channels in the profile current_channel = channel; current_stat.numfiles = current_channel->dirstat->numfiles; current_stat.filesize = current_channel->dirstat->filesize; current_stat.first = current_channel->dirstat->first; current_stat.last = current_channel->dirstat->last; current_channel = current_channel->next; while ( current_channel ) { current_stat.numfiles += current_channel->dirstat->numfiles; current_stat.filesize += current_channel->dirstat->filesize; if ( current_channel->dirstat->first && (current_channel->dirstat->first < current_stat.first) ) current_stat.first = current_channel->dirstat->first; if ( current_channel->dirstat->last > current_stat.last ) current_stat.last = current_channel->dirstat->last; current_channel = current_channel->next; } old_stat = current_stat; ExpireProfile(channel, ¤t_stat, maxsize, lifetime, runtime); } else { // cmd args override dirstat values if ( maxsize_set ) channel->dirstat->max_size = maxsize; else maxsize = channel->dirstat->max_size; if ( maxlife_set ) channel->dirstat->max_lifetime = lifetime; else lifetime = channel->dirstat->max_lifetime; old_stat = *(channel->dirstat); ExpireDir(channel->datadir, channel->dirstat, maxsize, lifetime, runtime); current_stat = *(channel->dirstat); } // Report, what we have done printf("Expired files: %llu\n", (unsigned long long)(old_stat.numfiles - current_stat.numfiles)); printf("Expired file size: %s\n", ScaleValue(old_stat.filesize - current_stat.filesize)); printf("Expired time range: %s\n\n", ScaleTime(current_stat.first - old_stat.first)); } if ( !is_profile && do_update_param ) { switch (channel->books_stat) { case BOOKKEEPER_OK: if ( maxsize_set ) channel->dirstat->max_size = maxsize; else maxsize = channel->dirstat->max_size; if ( maxlife_set ) channel->dirstat->max_lifetime = lifetime; else lifetime = channel->dirstat->max_lifetime; printf("Update collector process running for directory: '%s'\n", datadir); UpdateBooksParam(channel->books, (time_t)lifetime, maxsize); print_stat = 1; break; case ERR_NOTEXISTS: if ( maxsize_set ) channel->dirstat->max_size = maxsize; if ( maxlife_set ) channel->dirstat->max_lifetime = lifetime; print_stat = 1; break; default: // should never be reached as already cought earlier printf("Error %i while connecting to collector\n", channel->books_stat); } if ( channel->status == OK || channel->status == NOFILES ) WriteStatInfo(channel->dirstat); } if ( !is_profile && print_books ) { switch (channel->books_stat) { case BOOKKEEPER_OK: PrintBooks(channel->books); break; case ERR_NOTEXISTS: printf("No collector process running for directory: '%s'\n", channel->datadir); break; default: // should never be reached as already cought earlier printf("Error %i while connecting to collector\n", channel->books_stat); } } if ( print_stat ) { if ( is_profile ) { dirstat_t current_stat; current_stat.status = 0; current_stat.max_lifetime = lifetime; current_stat.max_size = maxsize; current_stat.low_water = low_water ? low_water : 98; // sum up all channels in the profile current_channel = channel; current_stat.numfiles = current_channel->dirstat->numfiles; current_stat.filesize = current_channel->dirstat->filesize; current_stat.first = current_channel->dirstat->first; current_stat.last = current_channel->dirstat->last; current_channel = current_channel->next; while ( current_channel ) { current_stat.numfiles += current_channel->dirstat->numfiles; current_stat.filesize += current_channel->dirstat->filesize; if ( current_channel->dirstat->first && (current_channel->dirstat->first < current_stat.first) ) current_stat.first = current_channel->dirstat->first; if ( current_channel->dirstat->last > current_stat.last ) current_stat.last = current_channel->dirstat->last; current_channel = current_channel->next; } if ( nfsen_format ) { printf("Stat|%llu|%llu|%llu\n", (unsigned long long)current_stat.filesize, (unsigned long long)current_stat.first, (unsigned long long)current_stat.last); } else PrintDirStat(¤t_stat); } else if ( nfsen_format ) printf("Stat|%llu|%llu|%llu\n", (unsigned long long)channel->dirstat->filesize, (unsigned long long)channel->dirstat->first, (unsigned long long)channel->dirstat->last ); else PrintDirStat(channel->dirstat); } current_channel = channel; while ( current_channel ) { ReleaseBookkeeper(current_channel->books, DETACH_ONLY); if ( current_channel->status == OK ) WriteStatInfo(current_channel->dirstat); current_channel = current_channel->next; } return 0; }