static batch_job_id_t batch_job_amazon_batch_submit(struct batch_queue* q, const char* cmd, const char* extra_input_files, const char* extra_output_files, struct jx* envlist, const struct rmsummary* resources){
struct internal_amazon_batch_amazon_ids amazon_ids = initialize(q);
char* env_var = amazon_ids.master_env_prefix;
//so, we have the access keys, now we need to either set up the queues and exec environments, or add them.
unsigned int jobid = gen_guid();
char* job_name = string_format("%s_%u",queue_name,jobid);
//makeflow specifics
struct batch_job_info *info = malloc(sizeof(*info));
memset(info, 0, sizeof(*info));
//specs
int cpus=1;
long int mem=1000;
char* img = hash_table_lookup(q->options,"amazon-batch-img");
int disk = 1000;
if(resources){
cpus = resources->cores;
mem = resources->memory;
disk = resources->disk;
cpus = cpus > 1? cpus:1;
mem = mem > 1000? mem:1000;
disk = disk > 1000 ? disk : 1000;
}
//upload files to S3
upload_input_files_to_s3((char*)extra_input_files,job_name);
upload_cmd_file(bucket_name,(char*)extra_input_files,(char*)extra_output_files,(char*)cmd,jobid);
//create the fmd string to give to the command
char* fmt_cmd = string_format("%s aws s3 cp s3://%s/COMAND_FILE_%u.sh ./ && sh ./COMAND_FILE_%u.sh",env_var,bucket_name,jobid,jobid);
//combine all properties together
char* properties_string = string_format("{ \\\"image\\\": \\\"%s\\\", \\\"vcpus\\\": %i, \\\"memory\\\": %li, \\\"privileged\\\":true ,\\\"command\\\": [\\\"sh\\\",\\\"-c\\\",\\\"%s\\\"], \\\"environment\\\":[{\\\"name\\\":\\\"AWS_ACCESS_KEY_ID\\\",\\\"value\\\":\\\"%s\\\"},{\\\"name\\\":\\\"AWS_SECRET_ACCESS_KEY\\\",\\\"value\\\":\\\"%s\\\"},{\\\"name\\\":\\\"REGION\\\",\\\"value\\\":\\\"%s\\\"}] }", img,cpus,mem,fmt_cmd,amazon_ids.aws_access_key_id,amazon_ids.aws_secret_access_key,amazon_ids.aws_region);
char* jaid = aws_submit_job(job_name,properties_string);
itable_insert(amazon_job_ids,jobid,jaid);
debug(D_BATCH,"Job %u has amazon id: %s",jobid,jaid);
itable_insert(done_files,jobid,string_format("%s",extra_output_files));
debug(D_BATCH,"Job %u successfully Submitted",jobid);
//let makeflow know
info->submitted = time(0);
info->started = time(0);
itable_insert(q->job_table, jobid, info);
//cleanup
free(job_name);
free(fmt_cmd);
return jobid;
}
static void upload_cmd_file(char* bucket_name, char* input_files, char* output_files, char* cmd, unsigned int jobid){
char* env_var = initialized_data.master_env_prefix;
//Create command to pull files from s3 and into local space to work on
char* bucket = string_format("s3://%s",bucket_name);
char* cpy_in = generate_s3_cp_cmds(input_files,bucket,"./");
char* chmod = chmod_all(input_files);
//run the actual command
char* cmd_tmp = string_format("%s\n%s\n%s\n",cpy_in,chmod,cmd);
free(cpy_in);
//copy out any external files
char* cpy_out = generate_s3_cp_cmds(output_files,"./",bucket);
cmd_tmp = string_format("%s\n%s\n",cmd_tmp,cpy_out);
//add headder
char* final_cmd = string_format("#!/bin/sh\n%s",cmd_tmp);
free(cmd_tmp);
//write out to file
unsigned int tempuid = gen_guid();
char* tmpfile_string = string_format("TEMPFILE-%u.sh",tempuid);
FILE* tmpfile = fopen(tmpfile_string,"w+");
fwrite(final_cmd,sizeof(char),strlen(final_cmd),tmpfile);
fclose(tmpfile);
free(final_cmd);
//make executable and put into s3
cmd_tmp = string_format("chmod +x %s",tmpfile_string);
sh_system(cmd_tmp);
free(cmd_tmp);
cmd_tmp = string_format("%s aws s3 cp %s s3://%s/COMAND_FILE_%u.sh",env_var,tmpfile_string,bucket_name,jobid);
sh_system(cmd_tmp);
free(cmd_tmp);
remove(tmpfile_string);
free(tmpfile_string);
}
int load_map(struct _LwVM *LwVM, char *map)
{
char *line_end;
char *line = map;
unsigned int line_sz;
unsigned int line_num = 1;
bool part_desc_open = false;
int current_partition;
for(line_end = map; (line_end = strchr(line, '\n')) != NULL;)
{
line_sz = line_end - line;
if (!strncmp(line, "//", 2));
else if (!strncmp(line, "part", 4))
{
if (!strncmp(line + 4, "\n", 1))
{
printf("Fatal syntax error at line %u: partition number not specified, exiting...", line_num);
return 1;
}
part_desc_open = true;
current_partition = atoi(line + 5);
if (current_partition > 12)
{
printf("Fatal error at line %u: partition number too big.\n", line_num);
return 1;
}
if (*(&LwVM->numPartitions) < current_partition)
{
*(&LwVM->numPartitions) = current_partition;
void *new_guid = gen_guid();
memcpy(&LwVM->partitions[current_partition - 1].guid[0], &new_guid[0], 16);
free(new_guid);
}
}
else if (!strncmp(line, "endpart", line_sz))
{
part_desc_open = false;
}
else if (!strncmp(line, "wipe", line_sz))
{
*(&LwVM->numPartitions) = 0;
memset(&LwVM->partitions[0], 0, sizeof(LwVMPartitionRecord) * 12);
}
else if (part_desc_open)
{
for(; line_sz != 0 && (!strncmp(line, " ", 1) || !strncmp(line, "\t", 1)); line++, line_sz--);
if (!strncmp(line, "type", 4))
{
if (!strncmp(line + 5, "hfs+", line_sz - 5) || !strncmp(line + 5, "HFS+", line_sz - 5))
{
memcpy(&LwVM->partitions[current_partition - 1].type, LwVMPartitionTypeHFSPlus, sizeof(*LwVMPartitionTypeHFSPlus));
}
else if (!strncmp(line + 5, "linuxdata", line_sz - 5) || !strncmp(line + 5, "LinuxData", line_sz - 5))
{
memcpy(&LwVM->partitions[current_partition - 1].type, LwVMPartitionTypeLinuxData, sizeof(*LwVMPartitionTypeLinuxData));
}
else
{
printf("Error at line %u: unknown type, exiting...\n", line_num);
return 1;
/*if (ignore_errors)
{
printf("contiuing anyway...\n");
}
else
{
printf("exiting...\n");
return 1;
}*/
}
}
else if (!strncmp(line, "name", 4))
{
if (line_sz < 6)
{
printf("Syntax error at line %u: no paramter value, exiting...\n", line_num);
return 1;
}
*(&line[line_sz]) = 0x00;
char *lwvm_name = str_to_lwvm_name(line + 5);
memcpy(&LwVM->partitions[current_partition - 1].partitionName, lwvm_name, 0x48);
free(lwvm_name);
}
else if (!strncmp(line, "end ", 4))
{
if (line_sz < 5)
{
printf("Syntax error at line %u: no paramter value, exiting...\n", line_num);
return 1;
}
line[line_sz] = 0;
*(&LwVM->partitions[current_partition - 1].end) = parse_size_str(line + 4);
}
else if (!strncmp(line, "begin", 5))
{
if (line_sz < 6)
{
printf("Syntax error at line %u: no paramter value, exiting...\n", line_num);
return 1;
}
line[line_sz] = 0;
*(&LwVM->partitions[current_partition - 1].begin) = parse_size_str(line + 5);
}
else
{
printf("Map syntax error at line %u: unknown partition parameter, ", line_num);
if (ignore_errors)
{
printf("ignoring...\n");
}
else
{
printf("exiting...\n");
return 1;
}
}
}
else
{
printf("Map syntax error at line %u: wrong operation or partition parameter outside any partition descriptor, ", line_num);
if (ignore_errors)
{
printf("ignoring...\n");
}
else
{
printf("exiting...\n");
return 1;
}
}
line = ++line_end;
line_num++;
}
return 0;
}
int edit_pt(struct _LwVM *LwVM, bool pt_no_crc)
{
char input[CLI_INPUT_BUFF_SZ];
while(1)
{
printf("Command (? for help): ");
fgets(&input[0], CLI_INPUT_BUFF_SZ, stdin);
if (!strcmp(input, "?\n") || !strcmp(input, "help\n") || !strcmp(input, "h\n"))
{
edit_help();
}
else if (!strcmp(input, "print\n") || !strcmp(input, "p\n"))
{
print_pt(LwVM, pt_no_crc);
}
else if (!strcmp(input, "quit\n") || !strcmp(input, "q\n"))
{
break;
}
else if (!strcmp(input, "write\n") || !strcmp(input, "w\n"))
{
damage_warning();
return SAVE_CHANGES;
}
else if (!strcmp(input, "add\n") || !strcmp(input, "a\n"))
{
if (*(&LwVM->numPartitions) < 12)
{
memset(&LwVM->partitions[*(&LwVM->numPartitions)], 0, sizeof(LwVMPartitionRecord));
/*uint64_t new_guid[2];
FILE *random_f = fopen("/dev/random", "r");
while(fread(&new_guid, 1, 16, random_f) != 16);
fclose(random_f);*/
void *new_guid = gen_guid();
memcpy(&LwVM->partitions[*(&LwVM->numPartitions)].guid[0], &new_guid[0], 16);
*(&LwVM->numPartitions) = *(&LwVM->numPartitions) + 1;
free(new_guid);
printf("Done.\n");
}
else printf("Can't create more than 12 partitions.\n");
}
else if (!strcmp(input, "rm\n") || !strcmp(input, "r\n"))
{
if (*(&LwVM->numPartitions) == 0)
{
printf("Nothing to delete.\n\n");
continue;
}
printf("Type the number of the partition you want to remove [1-%u]: ", *(&LwVM->numPartitions));
fgets(&input[0], CLI_INPUT_BUFF_SZ, stdin);
int part_num = atoi(input);
if (pt_splice(LwVM, part_num) == E_NOPART)
{
printf("No such partition.\n");
}
else
{
printf("Done.\n");
}
}
else if (!strcmp(input, "edit\n") || !strcmp(input, "e\n"))
{
printf("Type the number of the partition you want to edit [1-%u]: ", *(&LwVM->numPartitions));
fgets(&input[0], CLI_INPUT_BUFF_SZ, stdin);
int part_to_edit = atoi(input);
part_to_edit--;
if (part_to_edit < 12 && part_to_edit >= 0 && part_to_edit < *(&LwVM->numPartitions))
{
printf("What parameter do you want to edit? [begin/end/type/name] ");
fgets(&input[0], CLI_INPUT_BUFF_SZ, stdin);
if (!strcmp(input, "begin\n"))
{
printf("Begin [0-%llu(MB/GB)]: ", *(&LwVM->mediaSize));
uint64_t new_begin = get_param_input();
if (new_begin > *(&LwVM->mediaSize))
{
printf("Can't create a partition beyond the disk.\n");
}
else
{
*(&LwVM->partitions[part_to_edit].begin) = new_begin;
}
}
else if (!strcmp(input, "end\n"))
{
printf("End [0-%llu(MB/GB)]: ", *(&LwVM->mediaSize));
uint64_t new_end = get_param_input();
if (new_end > *(&LwVM->mediaSize))
{
printf("Can't set partition begin beyond the disk.\n");
}
else
{
*(&LwVM->partitions[part_to_edit].end) = new_end;
}
}
else if (!strcmp(input, "type\n"))
{
while(1)
{
printf("Enter partition type you want (? for help): ");
fgets(&input[0], CLI_INPUT_BUFF_SZ, stdin);
if (!strcmp(input, "?\n"))
{
available_part_types();
}
else if (!strcmp(input, "HFS+\n") || !strcmp(input, "hfs+\n"))
{
memcpy(&LwVM->partitions[part_to_edit].type, LwVMPartitionTypeHFSPlus, sizeof(*LwVMPartitionTypeHFSPlus));
break;
}
else if (!strcmp(input, "LinuxData\n") || !strcmp(input, "linuxdata\n"))
{
memcpy(&LwVM->partitions[part_to_edit].type, LwVMPartitionTypeLinuxData, sizeof(*LwVMPartitionTypeLinuxData));
break;
}
else if (!strcmp(input, "disabled\n") || !strcmp(input, "Disabled\n"))
{
memcpy(&LwVM->partitions[part_to_edit].type, LwVMPartitionTypeDisabled, sizeof(*LwVMPartitionTypeDisabled));
break;
}
else
{
printf("Unknown type: %s\n", input);
available_part_types();
}
}
}
else if (!strcmp(input, "name\n"))
{
printf("Type new name of the partition (max of 36 characters): ");
fgets(&input[0], CLI_INPUT_BUFF_SZ, stdin);
input[strlen(input) - 1] = 0; //Remove the \n byte.
char *lwvm_name = str_to_lwvm_name(input);
memcpy(&LwVM->partitions[part_to_edit].partitionName, lwvm_name, 0x48);
free(lwvm_name);
}
else
{
printf("Invalid parameter name.\n");
}
}
else
{
printf("Invalid number.\n");
}
}
else if (strcmp(input, " \n") && strcmp(input, "\n"))
{
edit_help();
}
printf("\n");
}
return DISCARD_CHANGES;
}
