/* Returns 0 == success, -1 == failure */
static int write_manifest_tar(struct manifest *manifest)
{
char *conf = config_output_dir();
char *tarcmd = NULL;
int ret = -1;
if (conf == NULL) {
assert(0);
}
/* now, tar the thing up for efficient full file download */
/* and put the signature of the plain manifest into the archive, too */
if (enable_signing) {
string_or_die(&tarcmd, TAR_COMMAND " --directory=%s/%i " TAR_PERM_ATTR_ARGS " -Jcf "
"%s/%i/Manifest.%s.tar Manifest.%s Manifest.%s.signed",
conf, manifest->version, conf, manifest->version, manifest->component,
manifest->component, manifest->component);
} else {
string_or_die(&tarcmd, TAR_COMMAND " --directory=%s/%i " TAR_PERM_ATTR_ARGS " -Jcf "
"%s/%i/Manifest.%s.tar Manifest.%s",
conf, manifest->version, conf, manifest->version, manifest->component,
manifest->component);
}
if (system(tarcmd) != 0) {
fprintf(stderr, "Creation of Manifest.tar failed\n");
goto exit;
}
ret = 0;
exit:
free(tarcmd);
free(conf);
return ret;
}
void prepare_delta_dir(struct manifest *manifest)
{
char *path;
char *conf;
printf("Preparing delta directory \n");
conf = config_output_dir();
string_or_die(&path, "%s/%i", conf, manifest->version);
if (mkdir(path, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) != 0) {
if (errno != EEXIST) {
LOG(NULL, "Failed to create directory ", "%s", path);
return;
}
}
free(path);
string_or_die(&path, "%s/%i/delta/", conf, manifest->version);
if (mkdir(path, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) != 0) {
if (errno != EEXIST) {
LOG(NULL, "Failed to create directory ", "%s", path);
return;
}
}
free(path);
free(conf);
}
bool read_configuration_file(char *filename)
{
GError *error = NULL;
keyfile = g_key_file_new();
if (!g_key_file_load_from_file(keyfile, filename, G_KEY_FILE_NONE, &error)) {
printf("Failed to Load configuration file: %s (%s)\n", filename, error->message);
g_error_free(error);
return false;
}
#if 0
char *c;
printf("Configuration settings:\n");
c = config_image_base();
printf(" image base path : %s\n", c);
free(c);
c = config_output_dir();
printf(" output directory : %s\n", c);
free(c);
c = config_empty_dir();
printf(" empty directory : %s\n", c);
free(c);
printf("\n");
#endif
return true;
}
void create_fullfiles(struct manifest *manifest)
{
GList *deduped_file_list;
char *path;
char *conf, *empty;
empty = config_empty_dir();
char *const rmcmd[] = { "rm", "-rf", empty, NULL };
if (system_argv(rmcmd) != 0) {
assert(0);
}
g_mkdir_with_parents(empty, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
free(empty);
conf = config_output_dir();
string_or_die(&path, "%s/%i", conf, manifest->version);
if (mkdir(path, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) != 0) {
if (errno != EEXIST) {
LOG(NULL, "Failed to create directory ", "%s", path);
return;
}
}
free(path);
string_or_die(&path, "%s/%i/files/", conf, manifest->version);
if (mkdir(path, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) != 0) {
if (errno != EEXIST) {
LOG(NULL, "Failed to create directory ", "%s", path);
return;
}
}
free(path);
free(conf);
/* De-duplicate the list of fullfiles needing created to avoid races */
deduped_file_list = get_deduplicated_fullfile_list(manifest);
/* Submit tasks to create full files */
submit_fullfile_tasks(deduped_file_list);
g_list_free(deduped_file_list);
}
/* Returns 0 == success, -1 == failure */
static int write_manifest_signature(struct manifest *manifest, const char *suffix)
{
char *conf = config_output_dir();
char *filename = NULL;
int ret = -1;
if (conf == NULL) {
assert(0);
}
string_or_die(&filename, "%s/%i/Manifest.%s%s", conf, manifest->version,
manifest->component, suffix);
if (!signature_sign(filename)) {
fprintf(stderr, "Creating signature for '%s' failed\n", filename);
goto exit;
}
ret = 0;
exit:
free(filename);
free(conf);
return ret;
}
void __create_delta(struct file *file, int from_version)
{
char *original, *newfile, *outfile, *dotfile, *testnewfile, *sanitycheck;
char *conf;
int ret;
if (file->is_link) {
return;
}
if (file->is_deleted) {
return; /* file got deleted -> by definition we cannot make a delta */
}
if (file->is_dir) {
return; /* cannot do this for directories yet */
}
conf = config_image_base();
string_or_die(&newfile, "%s/%i/full/%s", conf, file->last_change, file->filename);
string_or_die(&original, "%s/%i/full/%s", conf, from_version, file->peer->filename);
free(conf);
conf = config_output_dir();
string_or_die(&outfile, "%s/%i/delta/%i-%i-%s", conf, file->last_change, from_version, file->last_change, file->hash);
string_or_die(&dotfile, "%s/%i/delta/.%i-%i-%s", conf, file->last_change, from_version, file->last_change, file->hash);
string_or_die(&testnewfile, "%s/%i/delta/.%i-%i-%s.testnewfile", conf, file->last_change, from_version, file->last_change, file->hash);
string_or_die(&sanitycheck, "cmp -s \"%s\" \"%s\"", newfile, testnewfile);
LOG(file, "Making delta", "%s->%s", original, newfile);
ret = xattrs_compare(original, newfile);
if (ret != 0) {
LOG(NULL, "xattrs have changed, don't create diff ", "%s", newfile);
goto out;
}
ret = make_bsdiff_delta(original, newfile, dotfile, 0);
if (ret < 0) {
LOG(file, "Delta creation failed", "%s->%s ret is %i", original, newfile, ret);
goto out;
}
if (ret == 1) {
LOG(file, "...delta larger than newfile: FULLDL", "%s", newfile);
unlink(dotfile);
goto out;
}
/* does delta properly recreate expected content? */
ret = apply_bsdiff_delta(original, testnewfile, dotfile);
if (ret != 0) {
printf("Delta application failed.\n");
printf("Attempted %s->%s via diff %s\n", original, testnewfile, dotfile);
LOG(file, "Delta application failed.", "Attempted %s->%s via diff %s", original, testnewfile, dotfile);
#warning the above is racy..tolerate it temporarily
// ok fine
//unlink(testnewfile);
//unlink(dotfile);
ret = 0;
goto out;
}
xattrs_copy(original, newfile);
/* does xattrs have been correctly copied?*/
if (xattrs_compare(original, testnewfile) != 0) {
printf("Delta application resulted in xattrs mismatch.\n");
printf("%s->%s via diff %s yielded %s\n", original, newfile, dotfile, testnewfile);
LOG(file, "Delta xattrs mismatch:", "%s->%s via diff %s yielded %s", original, newfile, dotfile, testnewfile);
assert(0);
goto out;
}
ret = system(sanitycheck);
if (ret == -1 || !WIFEXITED(ret) || WEXITSTATUS(ret) == 2) {
printf("Sanity check system command failed %i. \n", ret);
printf("%s->%s via diff %s yielded %s\n", original, newfile, dotfile, testnewfile);
assert(0);
goto out;
} else if (WEXITSTATUS(ret) == 1) {
printf("Delta application resulted in file mismatch %i. \n", ret);
printf("%s->%s via diff %s yielded %s\n", original, newfile, dotfile, testnewfile);
LOG(file, "Delta mismatch:", "%s->%s via diff %s yielded %s", original, newfile, dotfile, testnewfile);
#warning this too will have failures due to races
//unlink(testnewfile);
//unlink(dotfile);
ret = 0;
goto out;
}
unlink(testnewfile);
if (rename(dotfile, outfile) != 0) {
if (errno == ENOENT) {
LOG(NULL, "dotfile:", " %s does not exist", dotfile);
}
LOG(NULL, "Failed to rename", "");
}
out: free(sanitycheck);
free(testnewfile);
free(conf);
free(newfile);
free(original);
free(outfile);
free(dotfile);
}
void write_cookiecrumbs_to_download_area(int version)
{
char *conf;
char *filename, *strformat;
FILE *versionfd, *formatfd;
conf = config_output_dir();
if (conf == NULL) {
assert(0);
}
string_or_die(&filename, "%s/%i/swupd-server-src-version", conf, version);
versionfd = fopen(filename, "w");
if (versionfd == NULL) {
assert(0);
}
if (fwrite(VERSION, 1, strlen(VERSION), versionfd) != strlen(VERSION)) {
assert(0);
}
fclose(versionfd);
free(filename);
string_or_die(&filename, "%s/%i/format", conf, version);
string_or_die(&strformat, "%llu", format);
formatfd = fopen(filename, "w");
if (formatfd == NULL) {
assert(0);
}
if (fwrite(strformat, 1, strlen(strformat), formatfd) != strlen(strformat)) {
assert(0);
}
fclose(formatfd);
free(filename);
free(strformat);
/* Updating the WEBDIR/version/formatN/latest file is an operation very closely tied to a DevOps
* workflow and not something swupd should make a decision about.
*
* The supported format values (replacing the "N" in the path above) are:
*
* - "staging": intended for testing only
* - [1-+inf] (positive integer): the number that a corresponding version of swupd-client
* understands to run software updates. For example, if a given swupd-client release understands
* format 1, then WEBDIR/version/format1/latest contains the value of most recent release that
* that swupd-client can update to.
*
* Adapt this step appropriately for your DevOps flow.
*/
#if 0
FILE *file;
char *fullfile = NULL;
char *param;
string_or_die($param, "%s/version/formatstaging/", conf);
char *const mkdircmd[] = { "mkdir", "-p", param, NULL };
if (system_argv(mkdircmd) != 0) {
assert(0);
}
free(param);
string_or_die(&fullfile, "%s/version/formatstaging/latest", conf);
file = fopen(fullfile, "w");
if (!file) {
fprintf(stderr, "Cannot write new version to download area, failed to open %s (%s)\n", fullfile, strerror(errno));
free(fullfile);
return;
}
fprintf(file, "%i\n", version);
fclose(file);
free(fullfile);
#endif
free(conf);
}
/* output must be a file, which is a (compressed) tar file, of the file denoted by "file", without any of its
directory paths etc etc */
static void create_fullfile(struct file *file)
{
char *origin;
char *tarname = NULL;
char *rename_source = NULL;
char *rename_target = NULL;
char *rename_tmpdir = NULL;
int ret;
struct stat sbuf;
char *empty, *indir, *outdir;
char *param1, *param2;
if (file->is_deleted) {
return; /* file got deleted -> by definition we cannot tar it up */
}
empty = config_empty_dir();
indir = config_image_base();
outdir = config_output_dir();
string_or_die(&tarname, "%s/%i/files/%s.tar", outdir, file->last_change, file->hash);
if (access(tarname, R_OK) == 0) {
/* output file already exists...done */
free(tarname);
return;
}
free(tarname);
//printf("%s was missing\n", file->hash);
string_or_die(&origin, "%s/%i/full/%s", indir, file->last_change, file->filename);
if (lstat(origin, &sbuf) < 0) {
/* no input file: means earlier phase of update creation failed */
LOG(NULL, "Failed to stat", "%s: %s", origin, strerror(errno));
assert(0);
}
if (file->is_dir) { /* directories are easy */
char *tmp1, *tmp2, *dir, *base;
tmp1 = strdup(origin);
assert(tmp1);
base = basename(tmp1);
tmp2 = strdup(origin);
assert(tmp2);
dir = dirname(tmp2);
string_or_die(&rename_tmpdir, "%s/XXXXXX", outdir);
if (!mkdtemp(rename_tmpdir)) {
LOG(NULL, "Failed to create temporary directory for %s move", origin);
assert(0);
}
string_or_die(¶m1, "--exclude=%s/?*", base);
string_or_die(¶m2, "./%s", base);
char *const tarcfcmd[] = { TAR_COMMAND, "-C", dir, TAR_PERM_ATTR_ARGS_STRLIST, "-cf", "-", param1, param2, NULL };
char *const tarxfcmd[] = { TAR_COMMAND, "-C", rename_tmpdir, TAR_PERM_ATTR_ARGS_STRLIST, "-xf", "-", NULL };
int tarcmdresult = system_argv_pipe(tarcfcmd, tarxfcmd);
if (tarcmdresult != 0) {
LOG(NULL, "Tar command for copying directory full file failed with code %d", tarcmdresult);
assert(0);
}
free(param1);
free(param2);
string_or_die(&rename_source, "%s/%s", rename_tmpdir, base);
string_or_die(&rename_target, "%s/%s", rename_tmpdir, file->hash);
if (rename(rename_source, rename_target)) {
LOG(NULL, "rename failed for %s to %s", rename_source, rename_target);
assert(0);
}
free(rename_source);
/* for a directory file, tar up simply with gzip */
string_or_die(¶m1, "%s/%i/files/%s.tar", outdir, file->last_change, file->hash);
char *const tarcmd[] = { TAR_COMMAND, "-C", rename_tmpdir, TAR_PERM_ATTR_ARGS_STRLIST, "-zcf", param1, file->hash, NULL };
if (system_argv(tarcmd) != 0) {
assert(0);
}
free(param1);
if (rmdir(rename_target)) {
LOG(NULL, "rmdir failed for %s", rename_target);
}
free(rename_target);
if (rmdir(rename_tmpdir)) {
LOG(NULL, "rmdir failed for %s", rename_tmpdir);
}
free(rename_tmpdir);
free(tmp1);
free(tmp2);
} else { /* files are more complex */
char *gzfile = NULL, *bzfile = NULL, *xzfile = NULL;
char *tempfile;
uint64_t gz_size = LONG_MAX, bz_size = LONG_MAX, xz_size = LONG_MAX;
/* step 1: hardlink the guy to an empty directory with the hash as the filename */
string_or_die(&tempfile, "%s/%s", empty, file->hash);
if (link(origin, tempfile) < 0) {
LOG(NULL, "hardlink failed", "%s due to %s (%s -> %s)", file->filename, strerror(errno), origin, tempfile);
char *const argv[] = { "cp", "-a", origin, tempfile, NULL };
if (system_argv(argv) != 0) {
assert(0);
}
}
/* step 2a: tar it with each compression type */
// lzma
string_or_die(¶m1, "--directory=%s", empty);
string_or_die(¶m2, "%s/%i/files/%s.tar.xz", outdir, file->last_change, file->hash);
char *const tarlzmacmd[] = { TAR_COMMAND, param1, TAR_PERM_ATTR_ARGS_STRLIST, "-Jcf", param2, file->hash, NULL };
if (system_argv(tarlzmacmd) != 0) {
assert(0);
}
free(param1);
free(param2);
// gzip
string_or_die(¶m1, "--directory=%s", empty);
string_or_die(¶m2, "%s/%i/files/%s.tar.gz", outdir, file->last_change, file->hash);
char *const targzipcmd[] = { TAR_COMMAND, param1, TAR_PERM_ATTR_ARGS_STRLIST, "-zcf", param2, file->hash, NULL };
if (system_argv(targzipcmd) != 0) {
assert(0);
}
free(param1);
free(param2);
#ifdef SWUPD_WITH_BZIP2
string_or_die(¶m1, "--directory=%s", empty);
string_or_die(¶m2, "%s/%i/files/%s.tar.bz2", outdir, file->last_change, file->hash);
char *const tarbzip2cmd[] = { TAR_COMMAND, param1, TAR_PERM_ATTR_ARGS_STRLIST, "-jcf", param2, file->hash, NULL };
if (system_argv(tarbzip2cmd) != 0) {
assert(0);
}
free(param1);
free(param2);
#endif
/* step 2b: pick the smallest of the three compression formats */
string_or_die(&gzfile, "%s/%i/files/%s.tar.gz", outdir, file->last_change, file->hash);
if (stat(gzfile, &sbuf) == 0) {
gz_size = sbuf.st_size;
}
string_or_die(&bzfile, "%s/%i/files/%s.tar.bz2", outdir, file->last_change, file->hash);
if (stat(bzfile, &sbuf) == 0) {
bz_size = sbuf.st_size;
}
string_or_die(&xzfile, "%s/%i/files/%s.tar.xz", outdir, file->last_change, file->hash);
if (stat(xzfile, &sbuf) == 0) {
xz_size = sbuf.st_size;
}
string_or_die(&tarname, "%s/%i/files/%s.tar", outdir, file->last_change, file->hash);
if (gz_size <= xz_size && gz_size <= bz_size) {
ret = rename(gzfile, tarname);
} else if (xz_size <= bz_size) {
ret = rename(xzfile, tarname);
} else {
ret = rename(bzfile, tarname);
}
if (ret != 0) {
LOG(file, "post-tar rename failed", "ret=%d", ret);
}
unlink(bzfile);
unlink(xzfile);
unlink(gzfile);
free(bzfile);
free(xzfile);
free(gzfile);
free(tarname);
/* step 3: remove the hardlink */
unlink(tempfile);
free(tempfile);
}
free(indir);
free(outdir);
free(empty);
free(origin);
}
/* Returns 0 == success, -1 == failure */
static int write_manifest_plain(struct manifest *manifest)
{
GList *includes;
GList *list;
struct file *file;
FILE *out = NULL;
char *base = NULL, *dir;
char *conf = config_output_dir();
char *filename = NULL;
char *submanifest_filename = NULL;
int ret = -1;
if (conf == NULL) {
assert(0);
}
string_or_die(&filename, "%s/%i/Manifest.%s", conf, manifest->version, manifest->component);
base = strdup(filename);
if (base == NULL) {
assert(0);
}
dir = dirname(base);
if (g_mkdir_with_parents(dir, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) != 0) {
assert(0);
}
out = fopen(filename, "w");
if (out == NULL) {
printf("Failed to open %s for write\n", filename);
goto exit;
}
fprintf(out, "MANIFEST\t%llu\n", format);
fprintf(out, "version:\t%i\n", manifest->version);
fprintf(out, "previous:\t%i\n", manifest->prevversion);
fprintf(out, "filecount:\t%i\n", manifest->count);
fprintf(out, "timestamp:\t%i\n", (int)time(NULL));
compute_content_size(manifest);
fprintf(out, "contentsize:\t%llu\n", (long long unsigned int)manifest->contentsize);
includes = manifest->includes;
while (includes) {
struct manifest *sub = includes->data;
includes = g_list_next(includes);
fprintf(out, "includes:\t%s\n", sub->component);
}
fprintf(out, "\n");
list = g_list_first(manifest->files);
while (list) {
file = list->data;
list = g_list_next(list);
fprintf(out, "%s\t%s\t%i\t%s\n", file_type_to_string(file), file->hash, file->last_change, file->filename);
}
list = g_list_first(manifest->manifests);
while (list) {
file = list->data;
list = g_list_next(list);
string_or_die(&submanifest_filename, "%s/%i/Manifest.%s", conf, file->last_change, file->filename);
populate_file_struct(file, submanifest_filename);
ret = compute_hash(file, submanifest_filename);
if (ret != 0) {
printf("Hash computation failed\n");
assert(0);
}
fprintf(out, "%s\t%s\t%i\t%s\n", file_type_to_string(file), file->hash, file->last_change, file->filename);
free(submanifest_filename);
}
ret = 0;
exit:
if (out) {
fclose(out);
}
free(conf);
free(base);
free(filename);
return ret;
}
struct manifest *manifest_from_file(int version, char *component)
{
FILE *infile;
GList *includes = NULL;
char line[8192], *c, *c2;
int count = 0;
struct manifest *manifest;
char *filename, *conf;
int previous = 0;
unsigned long long int format_number;
conf = config_output_dir();
if (conf == NULL) {
assert(0);
}
string_or_die(&filename, "%s/%i/Manifest.%s", conf, version, component);
free(conf);
LOG(NULL, "Reading manifest", "%s", filename);
infile = fopen(filename, "rb");
if (infile == NULL) {
LOG(NULL, "Cannot read manifest", "%s (%s)\n", filename, strerror(errno));
free(filename);
return alloc_manifest(version, component);
}
/* line 1: MANIFEST\t<version> */
line[0] = 0;
if (fgets(line, 8191, infile) == NULL) {
fclose(infile);
return NULL;
}
if (strncmp(line, "MANIFEST\t", 9) != 0) {
printf("Invalid file format: MANIFEST line\n");
fclose(infile);
return NULL;
}
c = &line[9];
format_number = strtoull(c, NULL, 10);
if ((errno < 0) || (format_number == 0)) {
//format string shall be a positive integer
printf("Unknown file format version in MANIFEST line: %s\n", c);
fclose(infile);
return NULL;
}
line[0] = 0;
while (strcmp(line, "\n") != 0) {
/* read the header */
line[0] = 0;
if (fgets(line, 8191, infile) == NULL) {
break;
}
c = strchr(line, '\n');
if (c) {
*c = 0;
}
if (strlen(line) == 0) {
break;
}
c = strchr(line, '\t');
/* Make sure we're not at the end of the array before incrementing */
if (c && c <= &line[strlen(line)]) {
c++;
} else {
printf("Manifest is corrupt\n");
assert(0);
}
if (strncmp(line, "version:", 8) == 0) {
version = strtoull(c, NULL, 10);
}
if (strncmp(line, "previous:", 9) == 0) {
previous = strtoull(c, NULL, 10);
}
if (strncmp(line, "includes:", 9) == 0) {
includes = g_list_prepend(includes, strdup(c));
if (!includes->data) {
abort();
}
}
}
manifest = alloc_manifest(version, component);
manifest->format = format_number;
manifest->prevversion = previous;
manifest->includes = includes;
/* empty line */
while (!feof(infile)) {
struct file *file;
line[0] = 0;
if (fgets(line, 8191, infile) == NULL) {
break;
}
c = strchr(line, '\n');
if (c) {
*c = 0;
}
if (strlen(line) == 0) {
break;
}
file = calloc(1, sizeof(struct file));
if (file == NULL) {
assert(0);
}
c = line;
c2 = strchr(c, '\t');
if (c2) {
*c2 = 0;
c2++;
}
if (c[0] == 'F') {
file->is_file = 1;
} else if (c[0] == 'D') {
file->is_dir = 1;
} else if (c[0] == 'L') {
file->is_link = 1;
} else if (c[0] == 'M') {
LOG(NULL, "Found a manifest!", "%s", c);
file->is_manifest = 1;
} else if (c[0] != '.') {
assert(0); /* unknown file type */
}
if (c[1] == 'd') {
file->is_deleted = 1;
} else if (c[1] != '.') {
assert(0); /* unknown deleted status */
}
if (c[2] == 'C') {
file->is_config = 1;
} else if (c[2] == 's') {
file->is_state = 1;
} else if (c[2] == 'b') {
file->is_boot = 1;
} else if (c[2] != '.') {
assert(0); /* unknown modifier status */
}
if (c[3] == 'r') {
file->is_rename = 1;
} else if (c[3] != '.') {
; /* field 4: ignore unknown letters */
}
c = c2;
if (!c) {
free(file);
continue;
}
c2 = strchr(c, '\t');
if (c2) {
*c2 = 0;
c2++;
}
hash_assign(c, file->hash);
c = c2;
if (!c) {
free(file);
continue;
}
c2 = strchr(c, '\t');
if (c2) {
*c2 = 0;
c2++;
}
file->last_change = strtoull(c, NULL, 10);
c = c2;
if (!c) {
free(file);
continue;
}
file->filename = strdup(c);
if (file->is_manifest) {
nest_manifest_file(manifest, file);
} else {
manifest->files = g_list_prepend(manifest->files, file);
}
manifest->count++;
count++;
}
manifest->files = g_list_sort(manifest->files, file_sort_filename);
fclose(infile);
LOG(NULL, "Manifest info", "Manifest for version %i/%s contains %i files", version, component, count);
free(filename);
return manifest;
}
