int extfs_get_reqmntopt(char *partition, cstrlist *reqopt, cstrlist *badopt)
{
blk_t use_superblock=0;
int use_blocksize=0;
u32 defmntoptmask;
ext2_filsys fs;
if (!reqopt || !badopt)
return -1;
// check the "default mount options"
if (ext2fs_open(partition, EXT2_FLAG_JOURNAL_DEV_OK | EXT2_FLAG_SOFTSUPP_FEATURES, use_superblock, use_blocksize, unix_io_manager, &fs)!=0)
return -1;
defmntoptmask=fs->super->s_default_mount_opts;
if (!(defmntoptmask&EXT2_DEFM_XATTR_USER))
strlist_add(reqopt, "user_xattr");
if (!(defmntoptmask&EXT2_DEFM_ACL))
strlist_add(reqopt, "acl");
strlist_add(badopt, "nouser_xattr");
strlist_add(badopt, "noacl");
ext2fs_close(fs);
return 0;
}
static void expand_jars(char *path, struct strlist *dst)
{
struct dirent *entry;
DIR* dir;
size_t path_len = strlen(path);
if ((path_len < 2) || (path[path_len - 2] != '/') || (path[path_len - 1] != '*')) {
strlist_add(dst, path);
strlist_add(dst, ":");
return;
}
path[path_len - 1] = '\0'; /* Cut off trailing star */
dir = opendir(path);
if (dir == NULL) {
path[path_len - 1] = '*'; /* Restore original value */
return;
}
while ((entry = readdir(dir)) != NULL) {
#ifdef _DIRENT_HAVE_D_TYPE
if (entry->d_type != DT_REG)
continue;
#endif
if (is_jarfile(entry->d_name)) {
strlist_add(dst, path);
strlist_add(dst, entry->d_name);
strlist_add(dst, ":");
}
}
closedir(dir);
path[path_len - 1] = '*'; /* Restore original value */
}
static void split(char *str, char ch, struct strlist *dst)
{
char *p = str;
while ((p = strchr(str, ch)) != NULL) {
*p = '\0';
strlist_add(dst, str);
*p = ch;
str = p + 1;
}
strlist_add(dst, str);
}
static void add_slnk(int find, const char *path, const char *link)
{
char *tmp;
fail_unless((tmp=prepend_s(fullpath, path))!=NULL);
fail_unless(!symlink(link, tmp));
if(find==FOUND)
{
fail_unless(!strlist_add(&expected, tmp, (long)FT_LNK_S));
fail_unless(!strlist_add(&expected, link, 0));
}
free_w(&tmp);
}
static void add_hlnk(int find, const char *path, const char *link)
{
char *src;
char *dst;
fail_unless((src=prepend_s(fullpath, path))!=NULL);
fail_unless((dst=prepend_s(fullpath, link))!=NULL);
fail_unless(!do_link(dst, src, NULL, NULL, 0));
if(find==FOUND)
{
fail_unless(!strlist_add(&expected, src, (long)FT_LNK_H));
fail_unless(!strlist_add(&expected, dst, 0));
}
free_w(&src);
free_w(&dst);
}
strlist *readfile(char *filename) {
int fd, n, i;
char line[MAXLEN];
strlist *data = strlist_new();
strlist *ret = data;
INFO("Opening file %s\n", filename);
fd = open(filename, O_RDONLY);
if (fd < 0) {
ERROR("ERROR: could not open %s\n", filename);
IFERROR(perror(""));
return NULL;
}
i = 0; // TODO make sure this doesn't overflow line
while ((n = read(fd, &line[i], 1)) > 0) {
if (line[i] == '\n') {
line[i+1] = '\0';
data = strlist_add(data, line);
i = 0;
continue;
}
i++;
}
close(fd);
return ret;
}
static void add_nostat(int find, const char *path)
{
char *tmp;
fail_unless((tmp=prepend_s(fullpath, path))!=NULL);
if(find==FOUND)
fail_unless(!strlist_add(&expected, tmp, (long)FT_NOSTAT));
free_w(&tmp);
}
int btrfs_get_reqmntopt(char *partition, cstrlist *reqopt, cstrlist *badopt)
{
if (!reqopt || !badopt)
return -1;
strlist_add(badopt, "noacl");
return 0;
}
static void add_file(int find, const char *path, size_t s)
{
char *tmp;
fail_unless((tmp=prepend_s(fullpath, path))!=NULL);
create_file(tmp, s);
if(find==FOUND)
fail_unless(!strlist_add(&expected, tmp, (long)FT_REG));
free_w(&tmp);
}
static void add_node(int find, const char *path, mode_t mode, long ftype)
{
char *tmp;
fail_unless((tmp=prepend_s(fullpath, path))!=NULL);
fail_unless(!mknod(tmp, mode, 100));
if(find==FOUND)
fail_unless(!strlist_add(&expected, tmp, ftype));
free_w(&tmp);
}
static char* create_class_path_option(char *cp) {
struct strlist parts;
char *result;
strlist_init(&parts, 128);
strlist_add(&parts, "-Djava.class.path=");
expand_class_path(cp, &parts);
if (parts.size > 1) {
strlist_remove_last(&parts); /* last entry is a ':' */
}
result = strlist_concat(&parts);
strlist_destroy(&parts);
return result;
}
static retvalue add_override_field(struct overridedata *data, const char *secondpart, const char *thirdpart, bool source) {
retvalue r;
char *p;
if (forbidden_field_name(source, secondpart)) {
fprintf(stderr,
"Error: field '%s' not allowed in override files.\n",
secondpart);
return RET_ERROR;
}
if (secondpart[0] == '$') {
if (strcasecmp(secondpart, "$Delete") == 0) {
if (forbidden_field_name(source, thirdpart)) {
fprintf(stderr,
"Error: field '%s' not allowed in override files (not even as to be deleted).\n",
thirdpart);
return RET_ERROR;
}
} else if (strcasecmp(secondpart, "$Component") != 0) {
fprintf(stderr,
"Warning: special override field '%s' unknown and will be ignored\n",
secondpart);
}
}
p = strdup(secondpart);
if (FAILEDTOALLOC(p))
return RET_ERROR_OOM;
r = strlist_add(&data->fields, p);
if (RET_WAS_ERROR(r))
return r;
p = strdup(thirdpart);
if (FAILEDTOALLOC(p))
return RET_ERROR_OOM;
r = strlist_add(&data->fields, p);
return r;
}
retvalue chunk_getextralinelist(const char *chunk, const char *name, struct strlist *strlist) {
retvalue r;
const char *f, *b, *e;
char *v;
f = chunk_getfield(name, chunk);
if (f == NULL)
return RET_NOTHING;
strlist_init(strlist);
/* walk over the first line */
while (*f != '\0' && *f != '\n')
f++;
/* nothing there is an empty list */
if (*f == '\0')
return RET_OK;
f++;
/* while lines begin with ' ' or '\t', add them */
while (*f == ' ' || *f == '\t') {
while (*f != '\0' && xisblank(*f))
f++;
b = f;
while (*f != '\0' && *f != '\n')
f++;
e = f;
while (e > b && *e != '\0' && xisspace(*e))
e--;
if (!xisspace(*e))
v = strndup(b, e - b + 1);
else
v = strdup("");
if (FAILEDTOALLOC(v)) {
strlist_done(strlist);
return RET_ERROR_OOM;
}
r = strlist_add(strlist, v);
if (!RET_IS_OK(r)) {
strlist_done(strlist);
return r;
}
if (*f == '\0')
return RET_OK;
f++;
}
return RET_OK;
}
static int get_conf_val_args(const char *field, const char *value, const char *opt, struct strlist ***args, int *got_args, int *count, struct strlist ***list, int include)
{
char *tmp=NULL;
if(get_conf_val(field, value, opt, &tmp)) return -1;
if(tmp)
{
if(got_args && *got_args && args)
{
strlists_free(*args, *count);
*got_args=0;
*args=NULL;
*count=0;
}
if(strlist_add(list, count, tmp, include)) return -1;
free(tmp); tmp=NULL;
}
return 0;
}
retvalue chunk_getwordlist(const char *chunk, const char *name, struct strlist *strlist) {
retvalue r;
const char *f, *b;
char *v;
f = chunk_getfield(name, chunk);
if (f == NULL)
return RET_NOTHING;
strlist_init(strlist);
while (*f != '\0') {
/* walk over spaces */
while (*f != '\0' && xisspace(*f)) {
if (*f == '\n') {
f++;
if (*f != ' ' && *f != '\t')
return RET_OK;
} else
f++;
}
if (*f == '\0')
return RET_OK;
b = f;
/* search for end of word */
while (*f != '\0' && !xisspace(*f))
f++;
v = strndup(b, f - b);
if (FAILEDTOALLOC(v)) {
strlist_done(strlist);
return RET_ERROR_OOM;
}
r = strlist_add(strlist, v);
if (!RET_IS_OK(r)) {
strlist_done(strlist);
return r;
}
}
return RET_OK;
}
static void ctw_add_header(void *x, int argc, t_atom *argv) {
struct _ctw *common = x;
char *val;
char temp[MAXPDSTRING];
size_t header_len = 0;
size_t val_len;
if (argc < 1) {
pd_error(x, "You need to add some data to set headers");
return;
}
for (int i = 0; i < argc; i++) {
atom_string(argv + i, temp, MAXPDSTRING);
header_len += strlen(temp) + 1;
}
val = string_create(&(val_len), header_len);
for (int i = 0; i < argc; i++) {
atom_string(argv + i, temp, MAXPDSTRING);
strcat(val, temp);
if (i < argc - 1) {
strcat(val, " ");
}
}
common->http_headers = strlist_add(common->http_headers, val, val_len);
}
int strlist_split(cstrlist *l, char *text, char sep)
{
char *textcopy;
char delims[4];
char *saveptr;
char *result;
int len;
if (!l || !text)
{ errprintf("invalid param\n");
return -1;
}
// init
len=strlen(text);
snprintf(delims, sizeof(delims), "%c", sep);
strlist_empty(l);
if ((textcopy=malloc(len+1))==NULL)
{ errprintf("malloc(%d) failed\n", len+1);
return -1;
}
memcpy(textcopy, text, len+1);
for (result=strtok_r(textcopy, delims, &saveptr); result!=NULL; result=strtok_r(NULL, delims, &saveptr))
{
if (strlist_add(l, result)!=0)
{ errprintf("strlist_add(l, [%s]) failed\n", result);
free(textcopy);
return -1;
}
}
free(textcopy);
return 0;
}
int archwriter_create(carchwriter *ai)
{
//char testpath[PATH_MAX];
//struct statfs svfs;
//int tempfd;
struct stat64 st;
long archflags=0;
long archperm;
int res;
assert(ai);
// init
memset(&st, 0, sizeof(st));
archflags=O_RDWR|O_CREAT|O_TRUNC|O_LARGEFILE;
archperm=S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH;
// if the archive already exists and is a not regular file
res=stat64(ai->volpath, &st);
if (res==0 && !S_ISREG(st.st_mode))
{ errprintf("%s already exists, and is not a regular file.\n", ai->basepath);
return -1;
}
else if ((g_options.overwrite==0) && (res==0) && S_ISREG(st.st_mode)) // archive exists and is a regular file
{ errprintf("%s already exists, please remove it first.\n", ai->basepath);
return -1;
}
// check if it's a network filesystem
/*snprintf(testpath, sizeof(testpath), "%s.test", ai->volpath);
if (((tempfd=open64(testpath, basicflags, archperm))<0) ||
(fstatfs(tempfd, &svfs)!=0) ||
(close(tempfd)!=0) ||
(unlink(testpath)!=0))
{ errprintf("Cannot check the filesystem type on file %s\n", testpath);
return -1;
}
if (svfs.f_type==FSA_CIFS_MAGIC_NUMBER || svfs.f_type==FSA_SMB_SUPER_MAGIC)
{ sysprintf ("writing an archive on a smbfs/cifs filesystem is "
"not allowed, since it can produce corrupt archives.\n");
return -1;
}*/
ai->archfd=open64(ai->volpath, archflags, archperm);
if (ai->archfd < 0)
{ sysprintf ("cannot create archive %s\n", ai->volpath);
return -1;
}
ai->newarch=true;
strlist_add(&ai->vollist, ai->volpath);
/* lockf is causing corruption when the archive is written on a smbfs/cifs filesystem */
/*if (lockf(ai->archfd, F_LOCK, 0)!=0)
{ sysprintf("Cannot lock archive file: %s\n", ai->volpath);
close(ai->archfd);
return -1;
}*/
return 0;
}
static int fit_config_get_hash_list(void *fit, int conf_noffset,
int sig_offset, struct strlist *node_inc)
{
int allow_missing;
const char *prop, *iname, *end;
const char *conf_name, *sig_name;
char name[200], path[200];
int image_count;
int ret, len;
conf_name = fit_get_name(fit, conf_noffset, NULL);
sig_name = fit_get_name(fit, sig_offset, NULL);
/*
* Build a list of nodes we need to hash. We always need the root
* node and the configuration.
*/
strlist_init(node_inc);
snprintf(name, sizeof(name), "%s/%s", FIT_CONFS_PATH, conf_name);
if (strlist_add(node_inc, "/") ||
strlist_add(node_inc, name))
goto err_mem;
/* Get a list of images that we intend to sign */
prop = fit_config_get_image_list(fit, sig_offset, &len,
&allow_missing);
if (!prop)
return 0;
/* Locate the images */
end = prop + len;
image_count = 0;
for (iname = prop; iname < end; iname += strlen(iname) + 1) {
int noffset;
int image_noffset;
int hash_count;
image_noffset = fit_conf_get_prop_node(fit, conf_noffset,
iname);
if (image_noffset < 0) {
printf("Failed to find image '%s' in configuration '%s/%s'\n",
iname, conf_name, sig_name);
if (allow_missing)
continue;
return -ENOENT;
}
ret = fdt_get_path(fit, image_noffset, path, sizeof(path));
if (ret < 0)
goto err_path;
if (strlist_add(node_inc, path))
goto err_mem;
snprintf(name, sizeof(name), "%s/%s", FIT_CONFS_PATH,
conf_name);
/* Add all this image's hashes */
hash_count = 0;
for (noffset = fdt_first_subnode(fit, image_noffset);
noffset >= 0;
noffset = fdt_next_subnode(fit, noffset)) {
const char *name = fit_get_name(fit, noffset, NULL);
if (strncmp(name, FIT_HASH_NODENAME,
strlen(FIT_HASH_NODENAME)))
continue;
ret = fdt_get_path(fit, noffset, path, sizeof(path));
if (ret < 0)
goto err_path;
if (strlist_add(node_inc, path))
goto err_mem;
hash_count++;
}
if (!hash_count) {
printf("Failed to find any hash nodes in configuration '%s/%s' image '%s' - without these it is not possible to verify this image\n",
conf_name, sig_name, iname);
return -ENOMSG;
}
image_count++;
}
if (!image_count) {
printf("Failed to find any images for configuration '%s/%s'\n",
conf_name, sig_name);
return -ENOMSG;
}
return 0;
err_mem:
printf("Out of memory processing configuration '%s/%s'\n", conf_name,
sig_name);
return -ENOMEM;
err_path:
printf("Failed to get path for image '%s' in configuration '%s/%s': %s\n",
iname, conf_name, sig_name, fdt_strerror(ret));
return -ENOENT;
}
int run_bedup(int argc, char *argv[])
{
int i=1;
int ret=0;
int option=0;
int nonburp=0;
unsigned int maxlinks=DEF_MAX_LINKS;
char *groups=NULL;
char ext[16]="";
int givenconfigfile=0;
const char *configfile=NULL;
configfile=get_config_path();
snprintf(ext, sizeof(ext), ".bedup.%d", getpid());
while((option=getopt(argc, argv, "c:dg:hlm:nvV?"))!=-1)
{
switch(option)
{
case 'c':
configfile=optarg;
givenconfigfile=1;
break;
case 'd':
deletedups=1;
break;
case 'g':
groups=optarg;
break;
case 'l':
makelinks=1;
break;
case 'm':
maxlinks=atoi(optarg);
break;
case 'n':
nonburp=1;
break;
case 'V':
printf("%s-%s\n", prog, VERSION);
return 0;
case 'v':
verbose=1;
break;
case 'h':
case '?':
return usage();
}
}
if(nonburp && givenconfigfile)
{
logp("-n and -c options are mutually exclusive\n");
return 1;
}
if(nonburp && groups)
{
logp("-n and -g options are mutually exclusive\n");
return 1;
}
if(!nonburp && maxlinks!=DEF_MAX_LINKS)
{
logp("-m option is specified via the configuration file in burp mode (max_hardlinks=)\n");
return 1;
}
if(deletedups && makelinks)
{
logp("-d and -l options are mutually exclusive\n");
return 1;
}
if(deletedups && !nonburp)
{
logp("-d option requires -n option\n");
return 1;
}
if(optind>=argc)
{
if(nonburp)
{
logp("No directories found after options\n");
return 1;
}
}
else
{
if(!nonburp)
{
logp("Do not specify extra arguments.\n");
return 1;
}
}
if(maxlinks<2)
{
logp("The argument to -m needs to be greater than 1.\n");
return 1;
}
if(nonburp)
{
// Read directories from command line.
for(i=optind; i<argc; i++)
{
// Strip trailing slashes, for tidiness.
if(argv[i][strlen(argv[i])-1]=='/')
argv[i][strlen(argv[i])-1]='\0';
if(process_dir("", argv[i], ext, maxlinks,
0 /* not burp mode */, 0 /* level */))
{
ret=1;
break;
}
}
}
else
{
struct conf **globalcs=NULL;
struct strlist *grouplist=NULL;
struct lock *globallock=NULL;
if(groups)
{
char *tok=NULL;
if((tok=strtok(groups, ",\n")))
{
do
{
if(strlist_add(&grouplist, tok, 1))
{
log_out_of_memory(__func__);
return -1;
}
} while((tok=strtok(NULL, ",\n")));
}
if(!grouplist)
{
logp("unable to read list of groups\n");
return -1;
}
}
// Read directories from config files, and get locks.
if(!(globalcs=confs_alloc())) return -1;
if(confs_init(globalcs)) return -1;
if(conf_load_global_only(configfile, globalcs)) return 1;
if(get_e_burp_mode(globalcs[OPT_BURP_MODE])!=BURP_MODE_SERVER)
{
logp("%s is not a server config file\n", configfile);
confs_free(&globalcs);
return 1;
}
logp("Dedup clients from %s\n",
get_string(globalcs[OPT_CLIENTCONFDIR]));
maxlinks=get_int(globalcs[OPT_MAX_HARDLINKS]);
if(grouplist)
{
struct strlist *g=NULL;
logp("in dedup groups:\n");
for(g=grouplist; g; g=g->next)
logp("%s\n", g->path);
}
else
{
char *lockpath=NULL;
// Only get the global lock when doing a global run.
// If you are doing individual groups, you are likely
// to want to do many different dedup jobs and a
// global lock would get in the way.
if(!(lockpath=prepend(
get_string(globalcs[OPT_LOCKFILE]),
".bedup", ""))
|| !(globallock=lock_alloc_and_init(lockpath)))
return 1;
lock_get(globallock);
if(globallock->status!=GET_LOCK_GOT)
{
logp("Could not get lock %s (%d)\n", lockpath,
globallock->status);
free_w(&lockpath);
return 1;
}
logp("Got %s\n", lockpath);
}
ret=iterate_over_clients(globalcs, grouplist, ext, maxlinks);
confs_free(&globalcs);
lock_release(globallock);
lock_free(&globallock);
strlists_free(&grouplist);
}
if(!nonburp)
{
logp("%d client storages scanned\n", ccount);
}
logp("%llu duplicate %s found\n",
count, count==1?"file":"files");
logp("%llu bytes %s%s\n",
savedbytes, (makelinks || deletedups)?"saved":"saveable",
bytes_to_human(savedbytes));
return ret;
}
static inline retvalue parse_data(const char *name, const char *version, const char *data, size_t datalen, /*@out@*/struct trackedpackage **pkg) {
struct trackedpackage *p;
int i;
p = zNEW(struct trackedpackage);
if (FAILEDTOALLOC(p))
return RET_ERROR_OOM;
p->sourcename = strdup(name);
p->sourceversion = strdup(version);
if (FAILEDTOALLOC(p->sourcename)
|| FAILEDTOALLOC(p->sourceversion)
/* || FAILEDTOALLOC(p->sourcedir) */) {
trackedpackage_free(p);
return RET_ERROR_OOM;
}
while (datalen > 0 && *data != '\0') {
char *filekey;
const char *separator;
size_t filekeylen;
retvalue r;
if (((p->filekeys.count)&31) == 0) {
enum filetype *n = realloc(p->filetypes,
(p->filekeys.count+32)*sizeof(enum filetype));
if (FAILEDTOALLOC(n)) {
trackedpackage_free(p);
return RET_ERROR_OOM;
}
p->filetypes = n;
}
p->filetypes[p->filekeys.count] = *data;
data++; datalen--;
separator = memchr(data, '\0', datalen);
if (separator == NULL) {
fprintf(stderr,
"Internal Error: Corrupt tracking data for %s %s\n",
name, version);
trackedpackage_free(p);
return RET_ERROR;
}
filekeylen = separator - data;
filekey = strndup(data, filekeylen);
if (FAILEDTOALLOC(filekey)) {
trackedpackage_free(p);
return RET_ERROR_OOM;
}
r = strlist_add(&p->filekeys, filekey);
if (RET_WAS_ERROR(r)) {
trackedpackage_free(p);
return r;
}
data += filekeylen + 1;
datalen -= filekeylen + 1;
}
data++; datalen--;
p->refcounts = nzNEW(p->filekeys.count, int);
if (FAILEDTOALLOC(p->refcounts)) {
trackedpackage_free(p);
return RET_ERROR_OOM;
}
for (i = 0 ; i < p->filekeys.count ; i++) {
if ((p->refcounts[i] = parsenumber(&data, &datalen)) < 0) {
fprintf(stderr,
"Internal Error: Corrupt tracking data for %s %s\n",
name, version);
trackedpackage_free(p);
return RET_ERROR;
}
}
if (datalen > 0) {
fprintf(stderr,
"Internal Error: Trailing garbage in tracking data for %s %s\n (%ld bytes)",
name, version, (long)datalen);
trackedpackage_free(p);
return RET_ERROR;
}
p->flags.isnew = false;
p->flags.deleted = false;
*pkg = p;
return RET_OK;
}
retvalue trackedpackage_addfilekey(trackingdb tracks, struct trackedpackage *pkg, enum filetype filetype, char *filekey, bool used) {
char *id;
enum filetype ft = filetypechar(filetype);
int i, *newrefcounts;
enum filetype *newfiletypes;
retvalue r;
if (FAILEDTOALLOC(filekey))
return RET_ERROR_OOM;
for (i = 0 ; i < pkg->filekeys.count ; i++) {
if (strcmp(pkg->filekeys.values[i], filekey) == 0) {
if (pkg->filetypes[i] != ft) {
/* if old file has refcount 0, just repair: */
if (pkg->refcounts[i] <= 0) {
free(filekey);
pkg->filetypes[i] = ft;
if (used)
pkg->refcounts[i] = 1;
return RET_OK;
}
fprintf(stderr,
"Filekey '%s' already registered for '%s_%s' as type '%c' is tried to be reregistered as type '%c'!\n",
filekey, pkg->sourcename,
pkg->sourceversion,
pkg->filetypes[i], ft);
free(filekey);
return RET_ERROR;
}
free(filekey);
if (used)
pkg->refcounts[i]++;
return RET_OK;
}
}
newrefcounts = realloc(pkg->refcounts,
(pkg->filekeys.count + 1) * sizeof(int));
if (FAILEDTOALLOC(newrefcounts)) {
free(filekey);
return RET_ERROR_OOM;
}
if (used)
newrefcounts[pkg->filekeys.count]=1;
else
newrefcounts[pkg->filekeys.count]=0;
pkg->refcounts = newrefcounts;
newfiletypes = realloc(pkg->filetypes,
(pkg->filekeys.count + 1) * sizeof(enum filetype));
if (FAILEDTOALLOC(newfiletypes)) {
free(filekey);
return RET_ERROR_OOM;
}
newfiletypes[pkg->filekeys.count] = filetype;
pkg->filetypes = newfiletypes;
r = strlist_add(&pkg->filekeys, filekey);
if (RET_WAS_ERROR(r))
return r;
id = calc_trackreferee(tracks->codename,
pkg->sourcename, pkg->sourceversion);
if (FAILEDTOALLOC(id))
return RET_ERROR_OOM;
r = references_increment(filekey, id);
free(id);
return r;
}
int extfs_mkfs(cdico *d, char *partition, int extfstype, char *fsoptions)
{
cstrlist strfeatures;
u64 features_tab[3];
u64 fsextrevision;
int origextfstype;
char buffer[2048];
char command[2048];
char options[2048];
char temp[1024];
char progname[64];
u64 e2fstoolsver;
int compat_type;
u64 temp64;
int exitst;
int ret=0;
int res;
int i;
// init
memset(options, 0, sizeof(options));
snprintf(progname, sizeof(progname), "mke2fs");
strlist_init(&strfeatures);
// ---- check that mkfs is installed and get its version
if (exec_command(command, sizeof(command), NULL, NULL, 0, NULL, 0, "%s -V", progname)!=0)
{ errprintf("%s not found. please install a recent e2fsprogs on your system or check the PATH.\n", progname);
ret=-1;
goto extfs_mkfs_cleanup;
}
e2fstoolsver=check_prog_version(progname);
// ---- filesystem revision (good-old-rev or dynamic)
if (dico_get_u64(d, 0, FSYSHEADKEY_FSEXTREVISION, &fsextrevision)!=0)
fsextrevision=EXT2_DYNAMIC_REV; // don't fail (case of fs conversion to extfs)
// "mke2fs -q" prevents problems in exec_command when too many output details printed
strlcatf(options, sizeof(options), " -q ");
// filesystem revision: good-old-rev or dynamic
strlcatf(options, sizeof(options), " -r %d ", (int)fsextrevision);
strlcatf(options, sizeof(options), " %s ", fsoptions);
// ---- set the advanced filesystem settings from the dico
if (dico_get_string(d, 0, FSYSHEADKEY_FSLABEL, buffer, sizeof(buffer))==0 && strlen(buffer)>0)
strlcatf(options, sizeof(options), " -L '%.16s' ", buffer);
if (dico_get_u64(d, 0, FSYSHEADKEY_FSEXTBLOCKSIZE, &temp64)==0)
strlcatf(options, sizeof(options), " -b %ld ", (long)temp64);
if (dico_get_u64(d, 0, FSYSHEADKEY_FSINODESIZE, &temp64)==0)
strlcatf(options, sizeof(options), " -I %ld ", (long)temp64);
// ---- get original filesystem features (if the original filesystem was an ext{2,3,4})
if (dico_get_u64(d, 0, FSYSHEADKEY_FSEXTFEATURECOMPAT, &features_tab[E2P_FEATURE_COMPAT])!=0 ||
dico_get_u64(d, 0, FSYSHEADKEY_FSEXTFEATUREINCOMPAT, &features_tab[E2P_FEATURE_INCOMPAT])!=0 ||
dico_get_u64(d, 0, FSYSHEADKEY_FSEXTFEATUREROCOMPAT, &features_tab[E2P_FEATURE_RO_INCOMPAT])!=0)
{ // dont fail the original filesystem may not be ext{2,3,4}. in that case set defaults features
features_tab[E2P_FEATURE_COMPAT]=EXT2_FEATURE_COMPAT_RESIZE_INODE|EXT2_FEATURE_COMPAT_DIR_INDEX;
features_tab[E2P_FEATURE_INCOMPAT]=EXT2_FEATURE_INCOMPAT_FILETYPE;
features_tab[E2P_FEATURE_RO_INCOMPAT]=EXT2_FEATURE_RO_COMPAT_SPARSE_SUPER;
}
// ---- check that fsarchiver is aware of all the filesystem features used on that filesystem
if (extfs_check_compatibility(features_tab[E2P_FEATURE_COMPAT], features_tab[E2P_FEATURE_INCOMPAT], features_tab[E2P_FEATURE_RO_INCOMPAT])!=0)
{ errprintf("this filesystem has ext{2,3,4} features which are not supported by this fsarchiver version.\n");
return -1;
}
// ---- get original filesystem type
origextfstype=extfs_get_fstype_from_compat_flags(features_tab[E2P_FEATURE_COMPAT],
features_tab[E2P_FEATURE_INCOMPAT], features_tab[E2P_FEATURE_RO_INCOMPAT]);
msgprintf(MSG_VERB2, "the filesystem type determined by the original filesystem features is [%s]\n", format_fstype(origextfstype));
// remove all the features not supported by the filesystem to create (conversion = downgrade fs)
for (i=0; mkfeatures[i].name; i++)
{
compat_type=mkfeatures[i].compat;
if (mkfeatures[i].firstfs > extfstype)
features_tab[compat_type] &= ~mkfeatures[i].mask;
}
// add new features if the filesystem to create is newer than the filesystem type that was backed up
// eg: user did a "savefs" of an ext3 and does a "restfs mkfs=ext4" --> add features to force ext4
// it's a bit more difficult because we only want to add such a feature if no feature of the new
// filesystem is currently enabled.
msgprintf(MSG_VERB2, "the filesystem type to create considering the command options is [%s]\n", format_fstype(extfstype));
if (origextfstype==EXTFSTYPE_EXT2 && extfstype>EXTFSTYPE_EXT2) // upgrade ext2 to ext{3,4}
{ fsextrevision=EXT2_DYNAMIC_REV;
features_tab[E2P_FEATURE_COMPAT]|=EXT3_FEATURE_COMPAT_HAS_JOURNAL;
}
if (origextfstype<EXTFSTYPE_EXT4 && extfstype>=EXTFSTYPE_EXT4) // upgrade ext{2,3} to ext4
{ fsextrevision=EXT2_DYNAMIC_REV;
features_tab[E2P_FEATURE_INCOMPAT]|=EXT3_FEATURE_INCOMPAT_EXTENTS;
}
// convert int features to string to be passed to mkfs
for (i=0; mkfeatures[i].name; i++)
{
if (mkfeatures[i].firste2p<=e2fstoolsver) // don't pass an option to a program that does not support it
{
compat_type=mkfeatures[i].compat;
if (features_tab[compat_type] & mkfeatures[i].mask)
{ msgprintf(MSG_VERB2, "--> feature [%s]=YES\n", mkfeatures[i].name);
strlist_add(&strfeatures, mkfeatures[i].name);
}
else
{ msgprintf(MSG_VERB2, "--> feature [%s]=NO\n", mkfeatures[i].name);
snprintf(temp, sizeof(temp), "^%s", mkfeatures[i].name); // exclude feature
strlist_add(&strfeatures, temp);
}
}
}
// if extfs revision is dynamic and there are features in the list
if (fsextrevision!=EXT2_GOOD_OLD_REV && strlist_count(&strfeatures)>0)
{ strlist_merge(&strfeatures, temp, sizeof(temp), ',');
strlcatf(options, sizeof(options), " -O %s ", temp);
msgprintf(MSG_VERB2, "features: mkfs_options+=[-O %s]\n", temp);
}
// ---- check mke2fs version requirement
msgprintf(MSG_VERB2, "mke2fs version detected: %s\n", format_prog_version(e2fstoolsver, temp, sizeof(temp)));
msgprintf(MSG_VERB2, "mke2fs version required: %s\n", format_prog_version(e2fsprogs_minver[extfstype], temp, sizeof(temp)));
if (e2fstoolsver < e2fsprogs_minver[extfstype])
{ errprintf("mke2fs was found but is too old, please upgrade to a version %s or more recent.\n",
format_prog_version(e2fsprogs_minver[extfstype], temp, sizeof(temp)));
ret=-1;
goto extfs_mkfs_cleanup;
}
// ---- extended options
if (dico_get_u64(d, 0, FSYSHEADKEY_FSEXTEOPTRAIDSTRIDE, &temp64)==0)
strlcatf(options, sizeof(options), " -E stride=%ld ", (long)temp64);
if ((dico_get_u64(d, 0, FSYSHEADKEY_FSEXTEOPTRAIDSTRIPEWIDTH, &temp64)==0) && e2fstoolsver>=PROGVER(1,40,7))
strlcatf(options, sizeof(options), " -E stripe-width=%ld ", (long)temp64);
// ---- execute mke2fs
msgprintf(MSG_VERB2, "exec: %s\n", command);
if (exec_command(command, sizeof(command), &exitst, NULL, 0, NULL, 0, "%s %s %s", progname, partition, options)!=0 || exitst!=0)
{ errprintf("command [%s] failed with return status=%d\n", command, exitst);
ret=-1;
goto extfs_mkfs_cleanup;
}
// ---- use tune2fs to set the other advanced options
memset(options, 0, sizeof(options));
if (dico_get_string(d, 0, FSYSHEADKEY_FSUUID, buffer, sizeof(buffer))==0 && strlen(buffer)==36)
strlcatf(options, sizeof(options), " -U %s ", buffer);
if (dico_get_string(d, 0, FSYSHEADKEY_FSEXTDEFMNTOPT, buffer, sizeof(buffer))==0 && strlen(buffer)>0)
strlcatf(options, sizeof(options), " -o %s ", buffer);
if (dico_get_u64(d, 0, FSYSHEADKEY_FSEXTFSCKMAXMNTCOUNT, &temp64)==0)
strlcatf(options, sizeof(options), " -c %ld ", (long)temp64);
if (dico_get_u64(d, 0, FSYSHEADKEY_FSEXTFSCKCHECKINTERVAL, &temp64)==0)
strlcatf(options, sizeof(options), " -i %ldd ", (long)(temp64/86400L));
if (options[0])
{
if (exec_command(command, sizeof(command), &exitst, NULL, 0, NULL, 0, "tune2fs %s %s", partition, options)!=0 || exitst!=0)
{ errprintf("command [%s] failed with return status=%d\n", command, exitst);
ret=-1;
goto extfs_mkfs_cleanup;
}
// run e2fsck to workaround an tune2fs bug in e2fsprogs < 1.41.4 on ext4
// http://article.gmane.org/gmane.comp.file-systems.ext4/11181
if (extfstype==EXTFSTYPE_EXT4 && e2fstoolsver<PROGVER(1,41,4))
{
if ( ((res=exec_command(command, sizeof(command), &exitst, NULL, 0, NULL, 0, "e2fsck -fy %s", partition))!=0) || ((exitst!=0) && (exitst!=1)) )
{ errprintf("command [%s] failed with return status=%d\n", command, exitst);
ret=-1;
goto extfs_mkfs_cleanup;
}
}
}
extfs_mkfs_cleanup:
strlist_destroy(&strfeatures);
return ret;
}
// TODO: check for scripts in confdir early...
retvalue exportmode_set(struct exportmode *mode, struct configiterator *iter) {
retvalue r;
char *word;
r = config_getword(iter, &word);
if (RET_WAS_ERROR(r))
return r;
if (r == RET_NOTHING) {
fprintf(stderr,
"Error parsing %s, line %u, column %u: Unexpected end of field!\n"
"Filename to use for index files (Packages, Sources, ...) missing.\n",
config_filename(iter),
config_markerline(iter), config_markercolumn(iter));
return RET_ERROR_MISSING;
}
assert (word[0] != '\0');
if (word[0] == '.') {
free(word);
fprintf(stderr,
"Error parsing %s, line %u, column %u: filename for index files expected!\n",
config_filename(iter),
config_markerline(iter), config_markercolumn(iter));
return RET_ERROR;
}
free(mode->filename);
mode->filename = word;
r = config_getword(iter, &word);
if (RET_WAS_ERROR(r))
return r;
if (r == RET_NOTHING)
word = NULL;
if (r != RET_NOTHING && word[0] != '.') {
assert (word[0] != '\0');
free(mode->release);
mode->release = word;
r = config_getword(iter, &word);
if (RET_WAS_ERROR(r))
return r;
}
if (r == RET_NOTHING) {
fprintf(stderr,
"Error parsing %s, line %u, column %u: Unexpected end of field!\n"
"Compression identifiers ('.', '.gz' or '.bz2') missing.\n",
config_filename(iter),
config_markerline(iter), config_markercolumn(iter));
return RET_ERROR;
}
if (word[0] != '.') {
fprintf(stderr,
"Error parsing %s, line %u, column %u:\n"
"Compression extension ('.', '.gz' or '.bz2') expected.\n",
config_filename(iter),
config_markerline(iter), config_markercolumn(iter));
free(word);
return RET_ERROR;
}
mode->compressions = 0;
while (r != RET_NOTHING && word[0] == '.') {
if (word[1] == '\0')
mode->compressions |= IC_FLAG(ic_uncompressed);
else if (word[1] == 'g' && word[2] == 'z' &&
word[3] == '\0')
mode->compressions |= IC_FLAG(ic_gzip);
#ifdef HAVE_LIBBZ2
else if (word[1] == 'b' && word[2] == 'z' && word[3] == '2' &&
word[4] == '\0')
mode->compressions |= IC_FLAG(ic_bzip2);
#endif
else {
fprintf(stderr,
"Error parsing %s, line %u, column %u:\n"
"Unsupported compression extension '%s'!\n",
config_filename(iter),
config_markerline(iter),
config_markercolumn(iter),
word);
free(word);
return RET_ERROR;
}
free(word);
r = config_getword(iter, &word);
if (RET_WAS_ERROR(r))
return r;
}
while (r != RET_NOTHING) {
if (word[0] == '.') {
fprintf(stderr,
"Error parsing %s, line %u, column %u:\n"
"Scripts starting with dot are forbidden to avoid ambiguity ('%s')!\n"
"Try to put all compressions first and then all scripts to avoid this.\n",
config_filename(iter),
config_markerline(iter),
config_markercolumn(iter),
word);
free(word);
return RET_ERROR;
} else {
char *fullfilename = configfile_expandname(word, word);
if (FAILEDTOALLOC(fullfilename))
return RET_ERROR_OOM;
r = strlist_add(&mode->hooks, fullfilename);
if (RET_WAS_ERROR(r))
return r;
}
r = config_getword(iter, &word);
if (RET_WAS_ERROR(r))
return r;
}
return RET_OK;
}
int main(int argc, char *argv[])
{
int i=1;
int ret=0;
int option=0;
int nonburp=0;
unsigned int maxlinks=DEF_MAX_LINKS;
char *groups=NULL;
char ext[16]="";
int givenconfigfile=0;
prog=basename(argv[0]);
init_log(prog);
const char *configfile=NULL;
configfile=get_config_path();
snprintf(ext, sizeof(ext), ".bedup.%d", getpid());
while((option=getopt(argc, argv, "c:g:hlmnv?"))!=-1)
{
switch(option)
{
case 'c':
configfile=optarg;
givenconfigfile=1;
break;
case 'g':
groups=optarg;
break;
case 'l':
makelinks=1;
break;
case 'm':
maxlinks=atoi(optarg);
break;
case 'n':
nonburp=1;
break;
case 'v':
printf("%s-%s\n", prog, VERSION);
return 0;
case 'h':
case '?':
return usage();
}
}
if(nonburp && givenconfigfile)
{
logp("-n and -c options are mutually exclusive\n");
return 1;
}
if(nonburp && groups)
{
logp("-n and -g options are mutually exclusive\n");
return 1;
}
if(!nonburp && maxlinks!=DEF_MAX_LINKS)
{
logp("-m option is specified via the configuration file in burp mode (max_hardlinks=)\n");
return 1;
}
if(optind>=argc)
{
if(nonburp)
{
logp("No directories found after options\n");
return 1;
}
}
else
{
if(!nonburp)
{
logp("Do not specify extra arguments.\n");
return 1;
}
}
if(maxlinks<2)
{
logp("The argument to -m needs to be greater than 1.\n");
return 1;
}
if(nonburp)
{
// Read directories from command line.
for(i=optind; i<argc; i++)
{
// Strip trailing slashes, for tidiness.
if(argv[i][strlen(argv[i])-1]=='/')
argv[i][strlen(argv[i])-1]='\0';
if(process_dir("", argv[i], ext, maxlinks,
0 /* not burp mode */, 0 /* level */))
{
ret=1;
break;
}
}
}
else
{
int gcount=0;
struct config conf;
char *globallock=NULL;
struct strlist **grouplist=NULL;
if(groups)
{
char *tok=NULL;
if((tok=strtok(groups, ",\n")))
{
do
{
if(strlist_add(&grouplist, &gcount,
tok, 1))
{
logp("out of memory\n");
return -1;
}
} while((tok=strtok(NULL, ",\n")));
}
if(!gcount)
{
logp("unable to read list of groups\n");
return -1;
}
}
// Read directories from config files, and get locks.
init_config(&conf);
if(load_config(configfile, &conf, 1)) return 1;
if(conf.mode!=MODE_SERVER)
{
logp("%s is not a server config file\n", configfile);
free_config(&conf);
return 1;
}
logp("Dedup clients from %s\n", conf.clientconfdir);
maxlinks=conf.max_hardlinks;
if(gcount)
{
logp("in dedup groups:\n");
for(i=0; i<gcount; i++)
logp("%s\n", grouplist[i]->path);
}
else
{
// Only get the global lock when doing a global run.
// If you are doing individual groups, you are likely
// to want to do many different dedup jobs and a
// global lock would get in the way.
if(!(globallock=prepend(conf.lockfile, ".bedup", "")))
return 1;
if(get_lock(globallock))
{
logp("Could not get %s\n", globallock);
return 1;
}
logp("Got %s\n", globallock);
}
ret=iterate_over_clients(&conf, grouplist, gcount,
ext, maxlinks);
free_config(&conf);
if(globallock)
{
unlink(globallock);
free(globallock);
}
}
if(!nonburp)
{
logp("%d client storages scanned\n", ccount);
}
logp("%llu duplicate %s found\n",
count, count==1?"file":"files");
logp("%llu bytes %s%s\n",
savedbytes, makelinks?"saved":"saveable",
bytes_to_human(savedbytes));
return ret;
}
static int iterate_over_clients(struct config *conf, strlist_t **grouplist, int gcount, const char *ext, unsigned int maxlinks)
{
int ret=0;
DIR *dirp=NULL;
struct dirent *dirinfo=NULL;
signal(SIGABRT, &sighandler);
signal(SIGTERM, &sighandler);
signal(SIGINT, &sighandler);
if(!(dirp=opendir(conf->clientconfdir)))
{
logp("Could not opendir '%s': %s\n",
conf->clientconfdir, strerror(errno));
return 0;
}
while((dirinfo=readdir(dirp)))
{
char *lockfile=NULL;
char *lockfilebase=NULL;
if(!strcmp(dirinfo->d_name, ".")
|| !strcmp(dirinfo->d_name, "..")
|| looks_like_vim_tmpfile(dirinfo->d_name)
|| !is_regular_file(conf->clientconfdir, dirinfo->d_name))
continue;
if(gcount)
{
int ig=0;
if((ig=in_group(conf->clientconfdir,
dirinfo->d_name, grouplist, gcount, conf))<0)
{
ret=-1;
break;
}
if(!ig) continue;
}
if(!(lockfilebase=prepend(conf->client_lockdir,
dirinfo->d_name, "/"))
|| !(lockfile=prepend(lockfilebase,
BEDUP_LOCKFILE_NAME, "/")))
{
if(lockfilebase) free(lockfilebase);
if(lockfile) free(lockfile);
ret=-1;
break;
}
free(lockfilebase);
if(get_lock(lockfile))
{
logp("Could not get %s\n", lockfile);
free(lockfile);
continue;
}
// Remember that we got that lock.
if(strlist_add(&locklist, &lockcount, lockfile, 1))
{
free(lockfile);
lockcount=0;
break;
}
logp("Got %s\n", lockfile);
if(process_dir(conf->directory, dirinfo->d_name,
ext, maxlinks, 1 /* burp mode */, 0 /* level */))
{
ret=-1;
break;
}
ccount++;
}
closedir(dirp);
remove_locks();
return ret;
}