//!
//! downloads a digest and returns it as a new string (or NULL if error)
//! that the caller must free
//!
//! @param[in] url
//!
//! @return the digested string.
//!
//! @see file2strn()
//!
//! @note the caller must free the returned memory when done.
//!
char *walrus_get_digest(const char *url)
{
char *digest_str = NULL;
char *digest_path = strdup("/tmp/walrus-digest-XXXXXX");
if (!digest_path) {
logprintfl(EUCAERROR, "out of memory (failed to allocate digest path)\n");
return digest_path;
}
int tmp_fd = safe_mkstemp(digest_path);
if (tmp_fd < 0) {
logprintfl(EUCAERROR, "failed to create a digest file %s\n", digest_path);
} else {
close(tmp_fd); // walrus_ routine will reopen the file
// download a fresh digest
if (walrus_object_by_url(url, digest_path, 0) != 0) {
logprintfl(EUCAERROR, "failed to download digest to %s\n", digest_path);
} else {
digest_str = file2strn(digest_path, 2000000);
}
unlink(digest_path);
}
EUCA_FREE(digest_path);
return digest_str;
}
/* downloads a digest of an image and compare it to file at digest_path
* returns 0 if same, -N if different, N if error */
int walrus_verify_digest (const char * url, const char * old_digest)
{
int e = ERROR;
char * new_digest = strdup ("/tmp/walrus-digest-XXXXXX");
int tmp_fd = mkstemp (new_digest);
if (tmp_fd<0) {
logprintfl (EUCAERROR, "error: failed to create a digest file %s\n", new_digest);
} else {
close (tmp_fd); /* walrus routine will reopen the file */
/* download a fresh digest */
if ( (e=walrus_object_by_url (url, new_digest, 0)) != 0 ) {
logprintfl (EUCAERROR, "error: failed to download digest to %s\n", new_digest);
} else {
/* compare the two */
e = diff (new_digest, old_digest);
}
}
unlink (new_digest);
free (new_digest);
return e;
}
/* downloads a Walrus object from the default Walrus endpoint,
* so only the path is needed; saves object to outfile */
int walrus_object_by_path (const char * path, const char * outfile, const int do_compress)
{
char url [STRSIZE];
snprintf (url, STRSIZE, "http://%s%s/%s", DEFAULT_HOST_PORT, WALRUS_ENDPOINT, path);
return walrus_object_by_url (url, outfile, do_compress);
}
/* returns size of the file in bytes if OK, otherwise a negative error */
static long long get_cached_file (const char * user_id, const char * url, const char * file_id, const char * instance_id, const char * file_name, char * file_path, sem * s, int convert_to_disk, long long limit_mb)
{
char tmp_digest_path [BUFSIZE];
char cached_dir [BUFSIZE];
char cached_path [BUFSIZE];
char staging_path [BUFSIZE];
char digest_path [BUFSIZE];
snprintf (file_path, BUFSIZE, "%s/%s/%s/%s", sc_instance_path, user_id, instance_id, file_name);
snprintf (tmp_digest_path, BUFSIZE, "%s-digest", file_path);
snprintf (cached_dir, BUFSIZE, "%s/%s/cache/%s", sc_instance_path, EUCALYPTUS_ADMIN, file_id); /* cache is in admin's directory */
snprintf (cached_path, BUFSIZE, "%s/%s", cached_dir, file_name);
snprintf (staging_path, BUFSIZE, "%s-staging", cached_path);
snprintf (digest_path, BUFSIZE, "%s-digest", cached_path);
retry:
/* under a lock, figure out the state of the file */
sem_p (sc_sem); /***** acquire lock *****/
ensure_subdirectory_exists (file_path); /* creates missing directories */
struct stat mystat;
int cached_exists = ! stat (cached_path, &mystat);
int staging_exists = ! stat (staging_path, &mystat);
int e = ERROR;
int action;
enum { ABORT, VERIFY, WAIT, STAGE };
if ( staging_exists ) {
action = WAIT;
} else {
if ( cached_exists ) {
action = VERIFY;
} else {
action = STAGE;
}
}
/* we return the sum of these */
long long file_size_b = 0;
long long digest_size_b = 0;
/* while still under lock, decide whether to cache */
int should_cache = 0;
if (action==STAGE) {
e = walrus_object_by_url (url, tmp_digest_path, 0); /* get the digest to see how big the file is */
if (e==OK && stat (tmp_digest_path, &mystat)) {
digest_size_b = (long long)mystat.st_size;
}
if (e==OK) {
/* pull the size out of the digest */
char * xml_file = file2str (tmp_digest_path);
if (xml_file) {
file_size_b = str2longlong (xml_file, "<size>", "</size>");
free (xml_file);
}
if (file_size_b > 0) {
long long full_size_b = file_size_b+digest_size_b;
if (convert_to_disk) {
full_size_b += swap_size_mb*MEGABYTE + MEGABYTE; /* TODO: take into account extra padding required for disks (over partitions) */
}
if ( full_size_b/MEGABYTE + 1 > limit_mb ) {
logprintfl (EUCAFATAL, "error: insufficient disk capacity remaining (%lldMB) in VM Type of instance %s for component %s\n", limit_mb, instance_id, file_name);
action = ABORT;
} else if ( ok_to_cache (cached_path, full_size_b) ) { /* will invalidate the cache, if needed */
ensure_path_exists (cached_dir); /* creates missing directories */
should_cache = 1;
if ( touch (staging_path) ) { /* indicate that we'll be caching it */
logprintfl (EUCAERROR, "error: failed to create staging file %s\n", staging_path);
action = ABORT;
}
}
} else {
logprintfl (EUCAERROR, "error: failed to obtain file size from digest %s\n", url);
action = ABORT;
}
} else {
logprintfl (EUCAERROR, "error: failed to obtain digest from %s\n", url);
action = ABORT;
}
}
sem_v (sc_sem); /***** release lock *****/
switch (action) {
case STAGE:
logprintfl (EUCAINFO, "downloding image into %s...\n", file_path);
e = walrus_image_by_manifest_url (url, file_path, 1);
/* for KVM, convert partition into disk */
if (e==OK && convert_to_disk) {
sem_p (s);
/* for the cached disk swap==0 and ephemeral==0 as we'll append them below */
if ((e=vrun("%s %s %d %d", disk_convert_command_path, file_path, 0, 0))!=0) {
logprintfl (EUCAERROR, "error: partition-to-disk image conversion command failed\n");
}
sem_v (s);
/* recalculate file size now that it was converted */
if ( stat (file_path, &mystat ) != 0 ) {
logprintfl (EUCAERROR, "error: file %s not found\n", file_path);
} else if (mystat.st_size < 1) {
logprintfl (EUCAERROR, "error: file %s has the size of 0\n", file_path);
} else {
file_size_b = (long long)mystat.st_size;
}
}
/* cache the partition or disk, if possible */
if ( e==OK && should_cache ) {
if ( (e=vrun ("cp -a %s %s", file_path, cached_path)) != 0) {
logprintfl (EUCAERROR, "failed to copy file %s into cache at %s\n", file_path, cached_path);
}
if ( e==OK && (e=vrun ("cp -a %s %s", tmp_digest_path, digest_path)) != 0) {
logprintfl (EUCAERROR, "failed to copy digest file %s into cache at %s\n", tmp_digest_path, digest_path);
}
}
sem_p (sc_sem);
if (should_cache) {
unlink (staging_path);
}
if ( e ) {
logprintfl (EUCAERROR, "error: failed to download file from Walrus into %s\n", file_path);
unlink (file_path);
unlink (tmp_digest_path);
if (should_cache) {
unlink (cached_path);
unlink (digest_path);
if ( rmdir(cached_dir) ) {
logprintfl (EUCAWARN, "warning: failed to remove cache directory %s\n", cached_dir);
}
}
}
sem_v (sc_sem);
break;
case WAIT:
logprintfl (EUCAINFO, "waiting for disapperance of %s...\n", staging_path);
/* wait for staging_path to disappear, which means both either the
* download succeeded or it failed */
if ( (e=wait_for_file (NULL, staging_path, 180, "cached image")) )
return 0L;
/* yes, it is OK to fall through */
case VERIFY:
logprintfl (EUCAINFO, "verifying cached file in %s...\n", cached_path);
sem_p (sc_sem); /***** acquire lock *****/
e = ERROR;
if ( stat (cached_path, &mystat ) != 0 ) {
logprintfl (EUCAERROR, "error: file %s not found\n", cached_path);
} else if (mystat.st_size < 1) {
logprintfl (EUCAERROR, "error: file %s has the size of 0\n", cached_path);
} else if ((e=walrus_verify_digest (url, digest_path))<0) {
/* negative status => digest changed */
unlink (cached_path);
unlink (staging_path); /* TODO: needed? */
unlink (digest_path);
if ( rmdir (cached_dir) ) {
logprintfl (EUCAWARN, "warning: failed to remove cache directory %s\n", cached_dir);
} else {
logprintfl (EUCAINFO, "due to failure, removed cache directory %s\n", cached_dir);
}
} else {
file_size_b = mystat.st_size;
/* touch the digest so cache can use mtime for invalidation */
if ( touch (digest_path) ) {
logprintfl (EUCAERROR, "error: failed to touch digest file %s\n", digest_path);
} else if ( stat (digest_path, &mystat) ) {
logprintfl (EUCAERROR, "error: digest file %s not found\n", digest_path);
} else {
digest_size_b = (long long)mystat.st_size;
}
}
sem_v (sc_sem); /***** release lock *****/
if (e<0) { /* digest changed */
if (action==VERIFY) { /* i.e. we did not download/waited for this file */
/* try downloading anew */
goto retry;
} else {
logprintfl (EUCAERROR, "error: digest mismatch, giving up\n");
return 0L;
}
} else if (e>0) { /* problem with file or digest */
return 0L;
} else { /* all good - copy it, finally */
ensure_subdirectory_exists (file_path); /* creates missing directories */
if ( (e=vrun ("cp -a %s %s", cached_path, file_path)) != 0) {
logprintfl (EUCAERROR, "failed to copy file %s from cache at %s\n", file_path, cached_path);
return 0L;
}
}
break;
case ABORT:
logprintfl (EUCAERROR, "get_cached_file() failed (errno=%d)\n", e);
e = ERROR;
}
if (e==OK && file_size_b > 0 && convert_to_disk ) { // if all went well above
long long ephemeral_mb = limit_mb - swap_size_mb - (file_size_b+digest_size_b)/MEGABYTE;
if ( swap_size_mb>0L || ephemeral_mb>0L ) {
sem_p (s);
if ((e=vrun("%s %s %lld %lld", disk_convert_command_path, file_path, swap_size_mb, ephemeral_mb))!=0) {
logprintfl (EUCAERROR, "error: failed to add swap or ephemeral to the disk image\n");
}
sem_v (s);
/* recalculate file size (again!) now that it was converted */
if ( stat (file_path, &mystat ) != 0 ) {
logprintfl (EUCAERROR, "error: file %s not found\n", file_path);
} else if (mystat.st_size < 1) {
logprintfl (EUCAERROR, "error: file %s has the size of 0\n", file_path);
} else {
file_size_b = (long long)mystat.st_size;
}
}
}
if (e==OK && action!=ABORT)
return file_size_b + digest_size_b;
return 0L;
}
//!
//! Main entry point of the application
//!
//! @param[in] argc the number of parameter passed on the command line
//! @param[in] argv the list of arguments
//!
//! @return EUCA_OK
//!
int main(int argc, char *argv[])
{
int ch = 0;
int result = 0;
int tmp_fd = -1;
char *tmp_name = NULL;
char *command = DEFAULT_COMMAND;
char *hostport = NULL;
char *manifest = NULL;
char *file_name = NULL;
char *url = NULL;
char *login = NULL;
char *password = NULL;
char request[STRSIZE] = { 0 };
boolean do_compress = FALSE;
boolean do_get = FALSE;
while ((ch = getopt(argc, argv, "dh:m:f:zu:l:p:")) != -1) {
switch (ch) {
case 'h':
hostport = optarg;
break;
case 'm':
manifest = optarg;
break;
case 'd':
debug = TRUE;
break;
case 'f':
file_name = optarg;
break;
case 'u':
url = optarg;
break;
case 'l':
login = optarg;
break;
case 'p':
password = optarg;
break;
case 'z':
do_compress = TRUE;
break;
case '?':
default:
USAGE();
break;
}
}
argc -= optind;
argv += optind;
if (argc > 0) {
command = argv[0];
}
if (strcmp(command, "GetDecryptedImage") == 0 || strcmp(command, "GetObject") == 0) {
if (manifest == NULL) {
fprintf(stderr, "Error: manifest must be specified\n");
USAGE();
}
do_get = TRUE;
} else if (strcmp(command, "HttpPut") == 0) {
if (url == NULL || file_name == NULL) {
fprintf(stderr, "Error: URL and input file must be specified\n");
USAGE();
}
do_get = FALSE;
} else {
fprintf(stderr, "Error: unknown command [%s]\n", command);
USAGE();
}
if (do_get) {
/* use a temporary file for network data */
tmp_name = strdup("walrus-download-XXXXXX");
tmp_fd = safe_mkstemp(tmp_name);
if (tmp_fd < 0) {
fprintf(stderr, "Error: failed to create a temporary file\n");
USAGE();
}
close(tmp_fd);
if (hostport) {
snprintf(request, STRSIZE, "http://%s%s/%s", hostport, WALRUS_ENDPOINT, manifest);
if (strcmp(command, "GetObject") == 0) {
result = walrus_object_by_url(request, tmp_name, do_compress);
} else {
result = walrus_image_by_manifest_url(request, tmp_name, do_compress);
}
} else {
euca_strncpy(request, manifest, STRSIZE);
if (strcmp(command, "GetObject") == 0) {
result = walrus_object_by_path(request, tmp_name, do_compress);
} else {
result = walrus_image_by_manifest_path(request, tmp_name, do_compress);
}
}
if (result) {
/* error has occured */
cat(tmp_name);
fprintf(stderr, "\n"); /* in case error doesn't end with a newline */
remove(tmp_name);
} else {
/* all's well */
if (file_name) {
rename(tmp_name, file_name);
} else {
fprintf(stderr, "Saved output in %s\n", tmp_name);
}
}
EUCA_FREE(tmp_name);
} else { // HttpPut
result = http_put(file_name, url, login, password);
}
return (EUCA_OK);
}
