extern int job_submit(struct job_descriptor *job_desc, uint32_t submit_uid)
{
char *std_out, *tok;
uint32_t my_job_id;
my_job_id = get_next_job_id(true);
_xlate_dependency(job_desc, submit_uid, my_job_id);
if (job_desc->account)
_add_env2(job_desc, "PBS_ACCOUNT", job_desc->account);
if (job_desc->script) {
/* Setting PBS_ENVIRONMENT causes Intel MPI to believe that
* it is running on a PBS system, which isn't the case here. */
/* _add_env(job_desc, "PBS_ENVIRONMENT=PBS_BATCH"); */
} else {
/* Interactive jobs lack an environment in the job submit
* RPC, so it needs to be handled by a SPANK plugin */
/* _add_env(job_desc, "PBS_ENVIRONMENT=PBS_INTERACTIVE"); */
}
if (job_desc->partition)
_add_env2(job_desc, "PBS_QUEUE", job_desc->partition);
if (job_desc->std_out)
std_out = job_desc->std_out;
else
std_out = "slurm-%j.out";
if (job_desc->comment)
xstrcat(job_desc->comment, ",");
xstrcat(job_desc->comment, "stdout=");
if (std_out && (std_out[0] != '/') && job_desc->work_dir) {
xstrcat(job_desc->comment, job_desc->work_dir);
xstrcat(job_desc->comment, "/");
}
tok = strstr(std_out, "%j");
if (tok) {
char buf[16], *tok2;
char *tmp = xstrdup(std_out);
tok2 = strstr(tmp, "%j");
tok2[0] = '\0';
snprintf(buf, sizeof(buf), "%u", my_job_id);
xstrcat(tmp, buf);
xstrcat(tmp, tok + 2);
xstrcat(job_desc->comment, tmp);
xfree(tmp);
} else {
xstrcat(job_desc->comment, std_out);
}
return SLURM_SUCCESS;
}
int backup_server(char *path) {
Jcr *jcr = new_write_jcr();
strcpy(jcr->backup_path, path);
if (access(jcr->backup_path, 4) != 0) {
free(jcr);
puts("This path does not exist or can not be read!");
return -1;
}
if (index_init() == FALSE) {
return -1;
}
jcr->job_id = get_next_job_id();
if (is_job_existed(jcr->job_id)) {
printf("job existed!\n");
free(jcr);
return FAILURE;
}
if (jcr->job_id == 0) {
destor_stat->simulation_level = simulation_level;
} else {
if (simulation_level <= SIMULATION_RECOVERY
&& destor_stat->simulation_level >= SIMULATION_APPEND
|| simulation_level >= SIMULATION_APPEND
&& destor_stat->simulation_level <= SIMULATION_RECOVERY) {
dprint(
"the current simulation level is not matched with the destor stat!");
return FAILURE;
}
}
jcr->job_volume = create_job_volume(jcr->job_id);
puts("==== backup begin ====");
puts("==== transfering begin ====");
struct timeval begin, end;
gettimeofday(&begin, 0);
if (backup(jcr) != 0) {
free(jcr);
return FAILURE;
}
gettimeofday(&end, 0);
index_destroy();
jcr->time = end.tv_sec - begin.tv_sec
+ (double) (end.tv_usec - begin.tv_usec) / (1000 * 1000);
puts("==== transferring end ====");
printf("job id: %d\n", jcr->job_id);
printf("backup path: %s\n", jcr->backup_path);
printf("number of files: %d\n", jcr->file_num);
//printf("number of chunks: %d\n", jcr->chunk_num);
printf("number of dup chunks: %d\n", jcr->number_of_dup_chunks);
printf("total size: %ld\n", jcr->job_size);
printf("dedup size: %ld\n", jcr->dedup_size);
printf("dedup efficiency: %.4f, %.4f\n",
jcr->job_size != 0 ?
(double) (jcr->dedup_size) / (double) (jcr->job_size) : 0,
jcr->job_size / (double) (jcr->job_size - jcr->dedup_size));
printf("elapsed time: %.3fs\n", jcr->time);
printf("throughput: %.2fMB/s\n",
(double) jcr->job_size / (1024 * 1024 * jcr->time));
printf("zero chunk count: %d\n", jcr->zero_chunk_count);
printf("zero_chunk_amount: %ld\n", jcr->zero_chunk_amount);
printf("rewritten_chunk_count: %d\n", jcr->rewritten_chunk_count);
printf("rewritten_chunk_amount: %ld\n", jcr->rewritten_chunk_amount);
printf("rewritten rate in amount: %.3f\n",
jcr->rewritten_chunk_amount / (double) jcr->job_size);
printf("rewritten rate in count: %.3f\n",
jcr->rewritten_chunk_count / (double) jcr->chunk_num);
destor_stat->data_amount += jcr->job_size;
destor_stat->consumed_capacity += jcr->job_size - jcr->dedup_size;
destor_stat->saved_capacity += jcr->dedup_size;
destor_stat->number_of_chunks += jcr->chunk_num;
destor_stat->number_of_dup_chunks += jcr->number_of_dup_chunks;
destor_stat->zero_chunk_count += jcr->zero_chunk_count;
destor_stat->zero_chunk_amount += jcr->zero_chunk_amount;
destor_stat->rewritten_chunk_count += jcr->rewritten_chunk_count;
destor_stat->rewritten_chunk_amount += jcr->rewritten_chunk_amount;
destor_stat->sparse_chunk_count += sparse_chunk_count;
destor_stat->sparse_chunk_amount += sparse_chunk_amount;
destor_stat->container_num = container_volume.container_num;
destor_stat->index_memory_overhead = index_memory_overhead;
printf("read_time : %.3fs, %.2fMB/s\n", jcr->read_time / 1000000,
jcr->job_size * 1000000 / jcr->read_time / 1024 / 1024);
printf("chunk_time : %.3fs, %.2fMB/s\n", jcr->chunk_time / 1000000,
jcr->job_size * 1000000 / jcr->chunk_time / 1024 / 1024);
printf("name_time : %.3fs, %.2fMB/s\n", jcr->name_time / 1000000,
jcr->job_size * 1000000 / jcr->name_time / 1024 / 1024);
printf("filter_time : %.3fs, %.2fMB/s\n", jcr->filter_time / 1000000,
jcr->job_size * 1000000 / jcr->filter_time / 1024 / 1024);
printf("write_time : %.3fs, %.2fMB/s\n", jcr->write_time / 1000000,
jcr->job_size * 1000000 / jcr->write_time / 1024 / 1024);
printf("index_search_time : %.3fs, %.2fMB/s\n", search_time / 1000000,
jcr->job_size * 1000000 / search_time / 1024 / 1024);
printf("index_update_time : %.3fs, %.2fMB/s\n", update_time / 1000000,
jcr->job_size * 1000000 / update_time / 1024 / 1024);
puts("==== backup end ====");
jcr->job_volume->job.job_id = jcr->job_id;
jcr->job_volume->job.is_del = FALSE;
jcr->job_volume->job.file_num = jcr->file_num;
jcr->job_volume->job.chunk_num = jcr->chunk_num;
strcpy(jcr->job_volume->job.backup_path, jcr->backup_path);
update_job_volume_des(jcr->job_volume);
close_job_volume(jcr->job_volume);
jcr->job_volume = 0;
double seek_time = 0.005; //5ms
double bandwidth = 120 * 1024 * 1024; //120MB/s
double index_lookup_throughput = jcr->job_size
/ (index_read_times * seek_time
+ index_read_entry_counter * 24 / bandwidth) / 1024 / 1024;
double write_data_throughput = 1.0 * jcr->job_size * bandwidth
/ (jcr->job_size - jcr->dedup_size) / 1024 / 1024;
double index_read_throughput = 1.0 * jcr->job_size / 1024 / 1024
/ (index_read_times * seek_time
+ index_read_entry_counter * 24 / bandwidth);
double index_write_throughput = 1.0 * jcr->job_size / 1024 / 1024
/ (index_write_times * seek_time
+ index_write_entry_counter * 24 / bandwidth);
double estimated_throughput = write_data_throughput;
if (estimated_throughput > index_read_throughput)
estimated_throughput = index_read_throughput;
/*if (estimated_throughput > index_write_throughput)
estimated_throughput = index_write_throughput;*/
char logfile[] = "backup.log";
int fd = open(logfile, O_WRONLY | O_CREAT, S_IRWXU);
lseek(fd, 0, SEEK_END);
char buf[512];
/*
* job id,
* chunk number,
* accumulative consumed capacity,
* deduplication ratio,
* rewritten ratio,
* total container number,
* sparse container number,
* inherited container number,
* throughput,
* index memory overhead,
* index lookups,
* index updates,
*/
sprintf(buf, "%d %d %ld %.4f %.4f %d %d %d %.2f %ld %ld %ld\n", jcr->job_id,
jcr->chunk_num, destor_stat->consumed_capacity,
jcr->job_size != 0 ?
(double) (jcr->dedup_size) / (double) (jcr->job_size) : 0,
jcr->job_size != 0 ?
(double) (jcr->rewritten_chunk_amount)
/ (double) (jcr->job_size) :
0, jcr->total_container_num, jcr->sparse_container_num,
jcr->inherited_sparse_num,
(double) jcr->job_size / (1024 * 1024 * jcr->time),
index_memory_overhead, index_read_times, index_write_times);
if (write(fd, buf, strlen(buf)) != strlen(buf)) {
}
close(fd);
free_jcr(jcr);
return SUCCESS;
}
