dict_t dict_remove(dict_t dict, word_t word) { assert((dict != NULL) && (word != NULL) && (dict_exists(dict, word))); index_t index_word = index_from_string(word); list_t new_list = list_remove(dict->list, index_word); index_word = index_destroy(index_word); dict->list = new_list; dict->length--; return (dict); }
def_t dict_search(dict_t dict, word_t word) { assert(dict != NULL && word != NULL && dict_exists(dict, word)); index_t palabra = NULL; palabra = index_from_string(word); data_t definition = list_search(dict->list, palabra); def_t result = data_to_string(definition); palabra = index_destroy(palabra); assert(result != NULL); return (result); }
bool dict_exists(dict_t dict, word_t word) { assert(dict != NULL && word != NULL); bool result = false; index_t palabra = index_from_string(word); data_t definition = list_search(dict->list, palabra); if (definition != NULL) result = true; palabra = index_destroy(palabra); return (result); }
void hfile_destroy(hfile_t *fp) { meta_t *o; if (fp == NULL) return; hinfo_destroy(fp->fileinfo); htail_destroy(fp->trailer); for (o = index_head(fp->metas); o != NULL; o = meta_next(o)) block_destroy(fp->blocks[o->id]); index_destroy(fp->metas); bloom_destroy(fp->bloom); free(fp->blocks); hfile_free(fp); }
bool dict_exists(dict_t dict, word_t word) { /*Precondition verification*/ assert(dict != NULL); assert(word != NULL); index_t index = index_from_string(word); data_t data = bst_search(dict->data, index); index_destroy(index); return (data != NULL); }
dict_t dict_remove(dict_t dict, word_t word) { /*Precondition verification*/ assert(word != NULL); assert(dict != NULL); assert(dict_exists(dict, word)); index_t index = index_from_string(word); dict->data = bst_remove(dict->data, index); dict->length -= 1; index_destroy(index); return (dict); }
def_t dict_search(dict_t dict, word_t word) { /*Precondition verification*/ assert(dict != NULL); assert(word != NULL); assert( dict_exists(dict, word) ); index_t index = index_from_string(word); data_t data = bst_search(dict->data, index); def_t definicion = data_to_string(data); index_destroy(index); /*Postcondition verification*/ assert(definicion != NULL); return (definicion); }
static void write_index(const char *filename) { struct index_node *index; char *line; FILE *cfile; cfile = fopen(filename, "w"); if (!cfile) fatal("Could not open %s for writing: %s\n", filename, strerror(errno)); index = index_create(); while((line = getline_wrapped(stdin, NULL))) { index_insert(index, line); free(line); } index_write(index, cfile); index_destroy(index); fclose(cfile); }
SEXP insnp_new(const SEXP Filenames, const SEXP Sample_id, const SEXP Snp_id, const SEXP Diploid, const SEXP Fields, const SEXP Codes, const SEXP Threshold, const SEXP Lower, const SEXP Sep, const SEXP Comment, const SEXP Skip, const SEXP Simplify, const SEXP Verbose, const SEXP In_order, const SEXP Every){ /* Process arguments */ if (TYPEOF(Verbose)!=LGLSXP) error("Argument type error: Verbose"); if (length(Verbose)>1) warning("Only first element of argument used: Verbose"); int verbose = *INTEGER(Verbose); if (TYPEOF(In_order)!=LGLSXP) error("Argument type error: In_order"); if (length(In_order)>1) warning("Only first element of argument used: In_order"); int in_order = *INTEGER(In_order); if (TYPEOF(Filenames)!=STRSXP) error("Argument type error: Filenames"); int Nfile = length(Filenames); int Nsample=0; if (TYPEOF(Sample_id)==STRSXP) Nsample = length(Sample_id); else if (TYPEOF(Sample_id)!=NILSXP) error("Argument type error: Sample_id"); int Nsnp=0; if (TYPEOF(Snp_id)==STRSXP) Nsnp = length(Snp_id); else if (TYPEOF(Snp_id)!=NILSXP) error("Argument type error: Snp_id"); /* file is 1 = sample, 2 = snp, 0 = irrelevant */ int file_is = 0; if (!Nsample) { if (Nsnp) { Nsample = Nfile; Rprintf("Each file is assumed to concern a single sample\n"); Rprintf("(Sample IDs are assumed to be included in filenames)\n"); file_is = 1; } else error("No sample or SNP IDs specified"); } else if (!Nsnp) { Nsnp = Nfile; Rprintf("Each file is assumed to concern a single SNP\n"); Rprintf("(SNP IDs are assumed to be included in filenames)\n"); file_is = 2; } /* If not in order, set up hash tables */ index_db sample_index = NULL; index_db snp_index = NULL; if (!in_order) { if (file_is != 1) sample_index = create_name_index(Sample_id); if (file_is != 2) snp_index = create_name_index(Snp_id); } int *diploid=NULL; if (TYPEOF(Diploid)==LGLSXP) { if (length(Diploid)!=Nsample) error("Argument length error: diploid argument"); diploid = LOGICAL(Diploid); } else if (TYPEOF(Diploid)!=NILSXP) error("Argument type error: diploid argument"); if (TYPEOF(Fields)!=INTSXP) error("Argument type error: Fields"); int *fields = INTEGER(Fields); int fsamp=0, fsnp=0, fgt=0, fa1=0, fa2=0, fconf=0; SEXP Fnames = getAttrib(Fields, R_NamesSymbol); if (TYPEOF(Fnames)==NILSXP) error("Argument error: Fields argument has no names"); int fmax = 0; int Nfield = length(Fields); for (int i=0; i<Nfield; i++) { const char *fname = CHAR(STRING_ELT(Fnames, i)); int fi = fields[i]; if (!strcmp(fname, "sample")) fsamp = fi; else if (!strcmp(fname, "snp")) fsnp = fi; else if (!strcmp(fname, "genotype")) fgt = fi; else if (!strcmp(fname, "allele1")) fa1 = fi; else if (!strcmp(fname, "allele2")) fa2 = fi; else if (!strcmp(fname, "confidence")) fconf = fi; else error("Unrecognized input field name: %s", fname); if (fi>fmax) fmax = fi; } if (verbose) { Rprintf("Reading one call per input line\n"); if (fsamp) { Rprintf(" Sample id is in field %d", fsamp); if (file_is==1) Rprintf(" (ignored)\n"); else Rprintf("\n"); } if (fsnp) { Rprintf(" SNP id is in field %d", fsnp); if (file_is==2) Rprintf(" (ignored)\n"); else Rprintf("\n"); } if (fgt) Rprintf(" Genotype is in field %d\n", fgt); if (fa1) Rprintf(" Allele 1 is in field %d\n", fa1); if (fa2) Rprintf(" Allele 2 is in field %d\n", fa2); if (fconf) Rprintf(" Confidence score is in field %d\n", fconf); } if (file_is==1) fsamp = 0; else if (file_is==2) fsnp = 0; /* Allele or genotype coding? */ int gcoding; if (fgt) { if (fa1 || fa2) error("Coding must be by genotype OR allele, not both"); gcoding = 1; } else { if (!(fa1 && fa2)) error("No genotype or allele field(s) specified"); if (!(fa1 && fa2)) error("Field positions for both alleles must be specified"); gcoding = 0; } int nuc = 0; if (TYPEOF(Codes)!=STRSXP) error("Argument type error: Codes"); if (length(Codes)==1) { SEXP Code = STRING_ELT(Codes, 0); const char *code = CHAR(Code); if (!strcmp(code, "nucleotide")) nuc = 1; else error("Unrecognized coding: %s", code); } else { int ncode = length(Codes); if (gcoding) { if (diploid) { if (ncode!=5) { if (ncode==3) warning("Genotype coding for X: haploid genotypes are assumed to be coded as homozygous"); else error("Genotype coding for X.snp: three or five genotype codes must be specified"); } } else { if (ncode!=3) error("Genotype coding: three genotype codes must be specified"); } } else { if (ncode!=2) error("Allele coding: two allele codes must be specified"); } } if (TYPEOF(Threshold)==NILSXP && !fconf) error("Argument type error: no threshold argument"); if (TYPEOF(Threshold)!=REALSXP) error("Argument type error: Threshold"); double threshold = *REAL(Threshold); if (fconf && threshold==NA_REAL) error("Confidence score is read but no threshold is set"); if (TYPEOF(Lower)!=LGLSXP) error("Argument type error: Lower"); if (length(Lower)>1) warning("Only first element of argument used: Lower"); int lower = *INTEGER(Lower); char sep = ' '; if (TYPEOF(Sep)==STRSXP) { if (length(Sep)>1) warning("Only first element of argument used: Sep"); const char *c = CHAR(STRING_ELT(Sep, 0)); if (strlen(c)>1) warning("Only first character used: Sep"); sep = c[0]; } else if (TYPEOF(Sep)!=NILSXP) error("Argument type error: Sep"); char comment = (char) 0; if (TYPEOF(Comment)==STRSXP) { if (length(Sep)>1) warning("Only first element of argument used: Comment"); const char *c = CHAR(STRING_ELT(Comment, 0)); if (strlen(c)>1) warning("Only first character used: Comment"); comment = c[0]; } else if (TYPEOF(Comment)!=NILSXP) error("Argument type error: Comment"); int skip = 0; if (TYPEOF(Skip)==INTSXP) { if (length(Skip)>1) warning("Only first element used: Skip"); skip = INTEGER(Skip)[0]; } else if (TYPEOF(Skip)!=NILSXP) error("Argument type error: Skip"); if (TYPEOF(Simplify)!=LGLSXP) error("Argument type error: Simplify"); if (length(Simplify)>2) error("Argument length error: Simplify"); int *simplify = INTEGER(Simplify); if (length(Simplify)==1) simplify[1] = simplify[0]; int every=0; if (TYPEOF(Every)==INTSXP) { if (length(Every)>1) warning("Only first element used: Every"); every = INTEGER(Every)[0]; } else if (TYPEOF(Every)!=NILSXP) error("Argument type error: Every"); /* Create output object and initialise to zero */ if (verbose) { if (diploid) Rprintf("Reading XSnpMatrix with %d rows and %d columns\n", Nsample, Nsnp); else Rprintf("Reading SnpMatrix with %d rows and %d columns\n", Nsample, Nsnp); } SEXP Result, Dimnames, Package, Class; PROTECT(Result = allocMatrix(RAWSXP, Nsample, Nsnp)); PROTECT(Dimnames = allocVector(VECSXP, 2)); if (simplify[0]) { SET_VECTOR_ELT(Dimnames, 0, simplify_names(file_is==1? Filenames: Sample_id)); } else { SET_VECTOR_ELT(Dimnames, 0, duplicate(file_is==1? Filenames: Sample_id)); } if (simplify[1]) { SET_VECTOR_ELT(Dimnames, 1, simplify_names(file_is==2? Filenames: Snp_id)); } else { SET_VECTOR_ELT(Dimnames, 1, duplicate(file_is==2? Filenames: Snp_id)); } setAttrib(Result, R_DimNamesSymbol, Dimnames); /* Class */ PROTECT(Class = allocVector(STRSXP, 1)); if (diploid) { R_do_slot_assign(Result, mkString("diploid"), Diploid); SET_STRING_ELT(Class, 0, mkChar("XSnpMatrix")); } else { SET_STRING_ELT(Class, 0, mkChar("SnpMatrix")); } PROTECT(Package = allocVector(STRSXP, 1)); SET_STRING_ELT(Package, 0, mkChar("snpStats")); setAttrib(Class, install("package"), Package); classgets(Result, Class); SET_S4_OBJECT(Result); unsigned char *result = RAW(Result); memset(result, 0x00, Nsample*Nsnp); /* Read in data */ char field[MAX_FLD]; int Naccept = 0, Nreject = 0, Nocall = 0, Nskipped = 0, Nxerror = 0; int i_this = 0, j_this = 0; const char *this_sample=NULL, *this_snp=NULL; if (fsamp) { this_sample = CHAR(STRING_ELT(Sample_id, 0)); } if (fsnp) { this_snp = CHAR(STRING_ELT(Snp_id, 0)); } if (verbose) { Rprintf(" Cumulative totals\n"); Rprintf(" -----------------------------------\n"); Rprintf(" File Line Accepted Rejected No call Skipped File name\n"); } /* slowest varying 0 = don't know, 1 = sample, 2 = snp */ int slowest = file_is, last=Nsample*Nsnp-1; int advance = 0, finished=0; for (int f=0; f<Nfile; f++) { /* Open input file */ const char *filename = CHAR(STRING_ELT(Filenames, f)); if (verbose) { int lfn = strlen(filename); if (lfn > 20) { Rprintf("%59s...%-17s\r", "", filename+lfn-17); } else Rprintf("%59s%-20s\r", "", filename); } gzFile infile = gzopen(filename, "rb"); if (!infile) { warning("Failure to open input file: %s", filename); continue; } int fterm = 2, line = 0, found_in_file = 0; /* Skip any header lines */ for (int i=0; i<skip; i++) { line++; if (skip_to_eol(infile)==3) error("End-of-file reached on line %d", line); } Nskipped += skip; /* Read data lines */ while (fterm!=3) { /* Read a line */ line++; if (verbose && every && !(line % every)) Rprintf("%8d %8d %10d %8d %8d %8d\r", f+1, line, Naccept, Nreject, Nocall, Nskipped); int genotype=0, allele1=0, allele2=0; /* wanted is coded as: 1 if this call is to be accepted 0 if it is not wanted (or a comment line) -1 if rejected due to insufficient confidence -2 coded as no-call */ int wanted = 1; char sampid[MAX_FLD], snpid[MAX_FLD]; char gtype1[MAX_FLD], gtype2[MAX_FLD]; char cscore[MAX_FLD]; sampid[0] = snpid[0] = cscore[0] = (char) 0; for (int r=1; (r<=fmax); r++) { fterm = next_field(infile, sep, comment, '_', field, MAX_FLD); if (!fterm) error("Field overflow: line %d, field %d", line, r); if ((fterm>1) && (r<fmax)) { if (r==1) { if(!field[0]) { /* Empty line or comment line */ wanted = 0; if (fterm==2) { Nskipped++; continue; } else break; } } error("Incomplete line: %d (last field read: %d = %s)", line, r, field); } /* Save fields */ if (r==fsamp) { strncpy(sampid, field, MAX_FLD-1); } else if (r==fsnp) { strncpy(snpid, field, MAX_FLD-1); } else if (r==fconf) { strncpy(cscore, field, MAX_FLD-1); } else if (r==fgt) strncpy(gtype1, field, MAX_FLD-1); else if (r==fa1) strncpy(gtype1, field, MAX_FLD-1); else if (r==fa2) strncpy(gtype2, field, MAX_FLD-1); else { /* skip field */ } } /* Matches: for (int r=1; (r<=fmax); r++) { */ if (!wanted) continue; /* Discard any further fields */ if (fterm<2) { fterm = skip_to_eol(infile); } /* Find next target and check matches */ int match_sample, match_snp; if (in_order) { /* Advance to next target read */ if (advance) { /* If unknown, determine sort order by seeing which indicator has changed */ if (!slowest) { if (strcmp(this_sample, sampid)) slowest = 2; /* sample fastest, SNP slowest */ else if (strcmp(this_snp, snpid)) slowest = 1; /* SNP fastest, sample slowest */ else error("Error in input file sort order"); } /* Now advance fastest varying indicator */ if (slowest==1) { j_this++; if (j_this==Nsnp) { j_this = 0; if (fsamp) { i_this++; if (i_this==Nsample) { finished = 1; break; } else this_sample = CHAR(STRING_ELT(Sample_id, i_this)); } } this_snp = CHAR(STRING_ELT(Snp_id, j_this)); } else { i_this++; if (i_this==Nsample) { i_this = 0; if (fsnp) { j_this++; if (j_this==Nsnp) { finished = 1; break; } else this_snp = CHAR(STRING_ELT(Snp_id, j_this)); } } this_sample = CHAR(STRING_ELT(Sample_id, i_this)); } } /* Does current line match current target? */ match_sample = file_is==1 || !strcmp(this_sample, sampid); match_snp = file_is==2 || !strcmp(this_snp, snpid); } else { /* Not in order */ if (file_is!=1) { if (this_sample && strcmp(this_sample, sampid)) i_this = index_lookup(sample_index, sampid); if (i_this >= 0) { this_sample = CHAR(STRING_ELT(Sample_id, i_this)); match_sample = 1; } else { this_sample = NULL; match_sample = 0; } } else { match_sample = 1; i_this = f; } if (file_is!=2) { if (this_snp && strcmp(this_snp, snpid)) j_this = index_lookup(snp_index, snpid); if (j_this >= 0) { this_snp = CHAR(STRING_ELT(Snp_id, j_this)); match_snp = 1; } else { this_snp = NULL; match_snp = 0; } } else { match_snp = 1; j_this = f; } } if (match_sample && match_snp) { /* Next target read found in file(s) */ found_in_file++; int ij_this = j_this*Nsample + i_this; finished = (ij_this==last); /* Check confidence score */ if (fconf) { double conf; if (sscanf(cscore, "%lf", &conf)!=1) error("Failure to read confidence score: line %d", line); if ((lower && conf<threshold) || (!lower && conf>threshold)) { wanted = -1; Nreject++; if (finished) break; else advance = 1; continue; } } /* Decode genotype */ int which; if (gcoding) { if (nuc) { switch (strlen(gtype1)) { case 0: allele1 = allele2 = 0; break; case 1: allele1 = allele2 = nucleotide(gtype1[0]); break; case 2: allele1 = nucleotide(gtype1[0]); allele2 = nucleotide(gtype1[1]); break; default: error("Nucleotide coded genotype should be 2 character string: line %d", line); } } else { which = str_inlist(Codes, gtype1); if (!which) genotype = 0; else if (which>3) genotype = 2*which - 7; else genotype = which; } } else { if (nuc) { allele1 = nucleotide(gtype1[0]); allele2 = nucleotide(gtype2[0]); } else { allele1 = str_inlist(Codes, gtype1); allele2 = str_inlist(Codes, gtype2); } } /* Successful read, store genotype in result[i_this, j_this] */ if (nuc || !gcoding) { if (allele2 < allele1) { genotype = allele2; allele2 = allele1; allele1 = genotype; } if (allele1 && allele2) genotype = allele1 + (allele2*(allele2-1))/2; else genotype = 0; } if (genotype) { if (diploid && !diploid[i_this] && (genotype==2)) Nxerror++; else { Naccept++; result[ij_this] = (unsigned char) genotype; } } else { wanted = -2; Nocall++; } /* Flag need to advance to next target */ advance = 1; } /* matches if (match_sample && match_snp) { */ else { Nskipped++; /* Flag no advance to next target */ advance = 0; } if (finished) break; } /* matches: while (fterm!=3) { */ if (file_is==1) i_this++; if (file_is==2) j_this++; if (verbose) { Rprintf("%8d %8d %10d %8d %8d %8d\r", f+1, line, Naccept, Nreject, Nocall, Nskipped); } if(!found_in_file) warning("No calls found in file %s", filename); gzclose(infile); if (finished) break; } /* Warnings */ if (in_order && !finished) warning("End of data reached before search completed"); if (Nxerror) warning("%d haploid genotypes were coded as heterozygous; set to NA", Nxerror); if (Nskipped) warning("%d lines of input file(s) were skipped", Nskipped); /* Report */ if (verbose) Rprintf("\n"); Rprintf("%d genotypes successfully read\n", Naccept); if (Nreject) Rprintf("%d genotypes were rejected due to low confidence\n", Nreject); if (Nocall) Rprintf("%d genotypes were not called\n", Nocall); if (Nsample*Nsnp > Naccept+Nxerror+Nreject+Nocall) Rprintf("%d genotypes could not be found on input file(s)\n", Nsample*Nsnp - Naccept - Nreject - Nxerror-Nocall); if (nuc) { if (verbose) Rprintf("Recasting and checking nucleotide coding\n"); int none_snps = recode_snp(result, Nsample, Nsnp); if (none_snps) { Rprintf("%d polymorphisms were not SNPs and have been set to NA ", none_snps); Rprintf("(see warnings for details)\n"); } } UNPROTECT(4); /* Destroy hash indexes */ if (sample_index) index_destroy(sample_index); if (snp_index) index_destroy(snp_index); return Result; }
int backup_server(char *path) { Jcr *jcr = new_write_jcr(); strcpy(jcr->backup_path, path); if (access(jcr->backup_path, 4) != 0) { free(jcr); puts("This path does not exist or can not be read!"); return -1; } if (index_init() == FALSE) { return -1; } jcr->job_id = get_next_job_id(); if (is_job_existed(jcr->job_id)) { printf("job existed!\n"); free(jcr); return FAILURE; } if (jcr->job_id == 0) { destor_stat->simulation_level = simulation_level; } else { if (simulation_level <= SIMULATION_RECOVERY && destor_stat->simulation_level >= SIMULATION_APPEND || simulation_level >= SIMULATION_APPEND && destor_stat->simulation_level <= SIMULATION_RECOVERY) { dprint( "the current simulation level is not matched with the destor stat!"); return FAILURE; } } jcr->job_volume = create_job_volume(jcr->job_id); puts("==== backup begin ===="); puts("==== transfering begin ===="); struct timeval begin, end; gettimeofday(&begin, 0); if (backup(jcr) != 0) { free(jcr); return FAILURE; } gettimeofday(&end, 0); index_destroy(); jcr->time = end.tv_sec - begin.tv_sec + (double) (end.tv_usec - begin.tv_usec) / (1000 * 1000); puts("==== transferring end ===="); printf("job id: %d\n", jcr->job_id); printf("backup path: %s\n", jcr->backup_path); printf("number of files: %d\n", jcr->file_num); //printf("number of chunks: %d\n", jcr->chunk_num); printf("number of dup chunks: %d\n", jcr->number_of_dup_chunks); printf("total size: %ld\n", jcr->job_size); printf("dedup size: %ld\n", jcr->dedup_size); printf("dedup efficiency: %.4f, %.4f\n", jcr->job_size != 0 ? (double) (jcr->dedup_size) / (double) (jcr->job_size) : 0, jcr->job_size / (double) (jcr->job_size - jcr->dedup_size)); printf("elapsed time: %.3fs\n", jcr->time); printf("throughput: %.2fMB/s\n", (double) jcr->job_size / (1024 * 1024 * jcr->time)); printf("zero chunk count: %d\n", jcr->zero_chunk_count); printf("zero_chunk_amount: %ld\n", jcr->zero_chunk_amount); printf("rewritten_chunk_count: %d\n", jcr->rewritten_chunk_count); printf("rewritten_chunk_amount: %ld\n", jcr->rewritten_chunk_amount); printf("rewritten rate in amount: %.3f\n", jcr->rewritten_chunk_amount / (double) jcr->job_size); printf("rewritten rate in count: %.3f\n", jcr->rewritten_chunk_count / (double) jcr->chunk_num); destor_stat->data_amount += jcr->job_size; destor_stat->consumed_capacity += jcr->job_size - jcr->dedup_size; destor_stat->saved_capacity += jcr->dedup_size; destor_stat->number_of_chunks += jcr->chunk_num; destor_stat->number_of_dup_chunks += jcr->number_of_dup_chunks; destor_stat->zero_chunk_count += jcr->zero_chunk_count; destor_stat->zero_chunk_amount += jcr->zero_chunk_amount; destor_stat->rewritten_chunk_count += jcr->rewritten_chunk_count; destor_stat->rewritten_chunk_amount += jcr->rewritten_chunk_amount; destor_stat->sparse_chunk_count += sparse_chunk_count; destor_stat->sparse_chunk_amount += sparse_chunk_amount; destor_stat->container_num = container_volume.container_num; destor_stat->index_memory_overhead = index_memory_overhead; printf("read_time : %.3fs, %.2fMB/s\n", jcr->read_time / 1000000, jcr->job_size * 1000000 / jcr->read_time / 1024 / 1024); printf("chunk_time : %.3fs, %.2fMB/s\n", jcr->chunk_time / 1000000, jcr->job_size * 1000000 / jcr->chunk_time / 1024 / 1024); printf("name_time : %.3fs, %.2fMB/s\n", jcr->name_time / 1000000, jcr->job_size * 1000000 / jcr->name_time / 1024 / 1024); printf("filter_time : %.3fs, %.2fMB/s\n", jcr->filter_time / 1000000, jcr->job_size * 1000000 / jcr->filter_time / 1024 / 1024); printf("write_time : %.3fs, %.2fMB/s\n", jcr->write_time / 1000000, jcr->job_size * 1000000 / jcr->write_time / 1024 / 1024); printf("index_search_time : %.3fs, %.2fMB/s\n", search_time / 1000000, jcr->job_size * 1000000 / search_time / 1024 / 1024); printf("index_update_time : %.3fs, %.2fMB/s\n", update_time / 1000000, jcr->job_size * 1000000 / update_time / 1024 / 1024); puts("==== backup end ===="); jcr->job_volume->job.job_id = jcr->job_id; jcr->job_volume->job.is_del = FALSE; jcr->job_volume->job.file_num = jcr->file_num; jcr->job_volume->job.chunk_num = jcr->chunk_num; strcpy(jcr->job_volume->job.backup_path, jcr->backup_path); update_job_volume_des(jcr->job_volume); close_job_volume(jcr->job_volume); jcr->job_volume = 0; double seek_time = 0.005; //5ms double bandwidth = 120 * 1024 * 1024; //120MB/s double index_lookup_throughput = jcr->job_size / (index_read_times * seek_time + index_read_entry_counter * 24 / bandwidth) / 1024 / 1024; double write_data_throughput = 1.0 * jcr->job_size * bandwidth / (jcr->job_size - jcr->dedup_size) / 1024 / 1024; double index_read_throughput = 1.0 * jcr->job_size / 1024 / 1024 / (index_read_times * seek_time + index_read_entry_counter * 24 / bandwidth); double index_write_throughput = 1.0 * jcr->job_size / 1024 / 1024 / (index_write_times * seek_time + index_write_entry_counter * 24 / bandwidth); double estimated_throughput = write_data_throughput; if (estimated_throughput > index_read_throughput) estimated_throughput = index_read_throughput; /*if (estimated_throughput > index_write_throughput) estimated_throughput = index_write_throughput;*/ char logfile[] = "backup.log"; int fd = open(logfile, O_WRONLY | O_CREAT, S_IRWXU); lseek(fd, 0, SEEK_END); char buf[512]; /* * job id, * chunk number, * accumulative consumed capacity, * deduplication ratio, * rewritten ratio, * total container number, * sparse container number, * inherited container number, * throughput, * index memory overhead, * index lookups, * index updates, */ sprintf(buf, "%d %d %ld %.4f %.4f %d %d %d %.2f %ld %ld %ld\n", jcr->job_id, jcr->chunk_num, destor_stat->consumed_capacity, jcr->job_size != 0 ? (double) (jcr->dedup_size) / (double) (jcr->job_size) : 0, jcr->job_size != 0 ? (double) (jcr->rewritten_chunk_amount) / (double) (jcr->job_size) : 0, jcr->total_container_num, jcr->sparse_container_num, jcr->inherited_sparse_num, (double) jcr->job_size / (1024 * 1024 * jcr->time), index_memory_overhead, index_read_times, index_write_times); if (write(fd, buf, strlen(buf)) != strlen(buf)) { } close(fd); free_jcr(jcr); return SUCCESS; }
int cmd_add (const char *path, bool recursive) { char *prt_dir = NULL, *fname = NULL, *dname = NULL; struct stat64 st_buf; state_t ur = STATE_INITIALIZER; struct index index = INDEX_INITIALIZER; int status; if (lstat64 (path, &st_buf) != 0) fail("%s does not exist", path); prt_dir = (char *) malloc (strlen (path) + 4); parent_dir (path, prt_dir); if (st_buf.st_mode & S_IFREG) { fname = (char *) malloc (strlen (path) + 4); filename (path, fname); if (ur_check (prt_dir) == 0) { if (state_init (&ur, prt_dir) != 0) fail ("fail reading state of %s", prt_dir); if (index_read (&ur, &index) != 0) fail ("fail reading index of %s", prt_dir); status = index_entry_get_status (&index, fname); status |= S_IADD; if (!(status & S_IPST)) status |= S_IPST; index_entry_set_status (&index, fname, status); if (index_write (&ur, &index) != 0) fail ("fail writing index of %s", prt_dir); index_destroy (&index); state_destroy (&ur); } else fail ("%s initialization failed", prt_dir); free (fname); fname = NULL; } if (st_buf.st_mode & S_IFDIR) { dname = (char *) malloc (strlen (path) + 4); dirname (path, dname); if (ur_check (path) == 0) { if (ur_check (prt_dir) == 0) { if (state_init (&ur, prt_dir) != 0) fail ("fail reading state of %s", prt_dir); if (index_read (&ur, &index) != 0) fail ("fail reading index of %s", prt_dir); status = index_entry_get_status (&index, dname); status |= S_IADD; if (!(status & S_IPST)) status |= S_IPST; index_entry_set_status (&index, dname, status); if (index_write (&ur, &index) != 0) fail ("fail writing index of %s", prt_dir); index_destroy (&index); state_destroy (&ur); } } else { if (ur_create (path) != 0) fail ("%s initialization failed", path); output ("init: %s", path); if (ur_check (prt_dir) == 0) { if (state_init (&ur, prt_dir) != 0) fail ("fail reading state of %s", prt_dir); if (index_read (&ur, &index) != 0) fail ("fail reading index of %s", prt_dir); status = index_entry_get_status (&index, dname); status |= S_IADD; if (!(status & S_IPST)) status |= S_IPST; index_entry_set_status (&index, dname, status); if (index_write (&ur, &index) != 0) fail ("fail writing index of %s", prt_dir); index_destroy (&index); state_destroy (&ur); } } free (dname); dname = NULL; } free (prt_dir); prt_dir = NULL; /* * recursion */ if ((st_buf.st_mode & S_IFDIR) && recursive) { DIR *dp; struct dirent *ep; dp = opendir (path); if (dp != NULL) { while ((ep = readdir (dp))) { if (ep->d_name[0] != '.') { char *npath; npath = (char *) malloc (strlen (path) + strlen (ep->d_name) + 2); if (path[strlen (path) -1] == '/') sprintf (npath, "%s%s", path, ep->d_name); else sprintf (npath, "%s/%s", path, ep->d_name); cmd_add (npath, recursive); free (npath); } } (void) closedir (dp); } else fail ("could not open directory %s", path); } return 0; }
int cmd_status (const char *path, bool recursive) { state_t ur = STATE_INITIALIZER; struct index index = INDEX_INITIALIZER; struct stat64 st_buf; DIR *dp; struct dirent *ep; struct list_elem *e; char *branchname = NULL; if (lstat64 (path, &st_buf) != 0) fail("%s does not exist", path); ASSERT (st_buf.st_mode & S_IFDIR); if (ur_check (path) == 0) { if (state_init (&ur, path) != 0) fail ("fail reading state of %s", path); if (index_read (&ur, &index) != 0) fail ("fail reading index of %s", path); index_update (&ur, &index); branchname = branch_get_head_name (&ur); printf ("*** %s \n (branch: %s)\n", ur.path, branchname); free (branchname); branchname = NULL; // reading added dirty files for (e = list_begin (&index.entries); e != list_end (&index.entries); e = list_next (e)) { struct index_entry *en = list_entry (e, struct index_entry, elem); if ((en->status & S_IPST) && (en->status & S_IADD) && (en->status & S_IDRT)) { printf ("# added : %s/%s\n", path, en->name); } } // reading dirty but not added for (e = list_begin (&index.entries); e != list_end (&index.entries); e = list_next (e)) { struct index_entry *en = list_entry (e, struct index_entry, elem); if ((en->status & S_IPST) && !(en->status & S_IADD) && (en->status & S_IDRT) && (en->status & S_ITRK)) { printf ("# dirty : %s/%s\n", path, en->name); } } // reading untracked files for (e = list_begin (&index.entries); e != list_end (&index.entries); e = list_next (e)) { struct index_entry *en = list_entry (e, struct index_entry, elem); if ((en->status & S_IPST) && !(en->status & S_ITRK) && !(en->status & S_IADD)) { printf ("# untracked : %s/%s\n", path, en->name); } } index_destroy (&index); state_destroy (&ur); } else {
static db_result_t aql_execute(db_handle_t *handle, aql_adt_t *adt) { uint8_t optype; int first_rel_arg; db_result_t result; relation_t *rel; aql_attribute_t *attr; attribute_t *relattr; optype = AQL_GET_TYPE(adt); if(optype == AQL_TYPE_NONE) { /* No-ops always succeed. These can be generated by empty lines or comments in the query language. */ return DB_OK; } /* If the ASSIGN flag is set, the first relation in the array is the desired result relation. */ first_rel_arg = !!(adt->flags & AQL_FLAG_ASSIGN); if(optype != AQL_TYPE_CREATE_RELATION && optype != AQL_TYPE_REMOVE_RELATION && optype != AQL_TYPE_JOIN) { rel = relation_load(adt->relations[first_rel_arg]); if(rel == NULL) { return DB_NAME_ERROR; } } else { rel = NULL; } result = DB_RELATIONAL_ERROR; switch(optype) { case AQL_TYPE_CREATE_ATTRIBUTE: attr = &adt->attributes[0]; if(relation_attribute_add(rel, DB_STORAGE, attr->name, attr->domain, attr->element_size) != NULL) { result = DB_OK; } break; case AQL_TYPE_CREATE_INDEX: relattr = relation_attribute_get(rel, adt->attributes[0].name); if(relattr == NULL) { result = DB_NAME_ERROR; break; } result = index_create(AQL_GET_INDEX_TYPE(adt), rel, relattr); break; case AQL_TYPE_CREATE_RELATION: if(relation_create(adt->relations[0], DB_STORAGE) != NULL) { result = DB_OK; } break; case AQL_TYPE_REMOVE_ATTRIBUTE: result = relation_attribute_remove(rel, adt->attributes[0].name); break; case AQL_TYPE_REMOVE_INDEX: relattr = relation_attribute_get(rel, adt->attributes[0].name); if(relattr != NULL) { if(relattr->index != NULL) { result = index_destroy(relattr->index); } else { result = DB_OK; } } else { result = DB_NAME_ERROR; } break; case AQL_TYPE_REMOVE_RELATION: result = relation_remove(adt->relations[0], 1); break; #if DB_FEATURE_REMOVE case AQL_TYPE_REMOVE_TUPLES: /* Overwrite the attribute array with a full copy of the original relation's attributes. */ adt->attribute_count = 0; for(relattr = list_head(rel->attributes); relattr != NULL; relattr = relattr->next) { AQL_ADD_ATTRIBUTE(adt, relattr->name, DOMAIN_UNSPECIFIED, 0); } AQL_SET_FLAG(adt, AQL_FLAG_INVERSE_LOGIC); #endif /* DB_FEATURE_REMOVE */ case AQL_TYPE_SELECT: if(handle == NULL) { result = DB_ARGUMENT_ERROR; break; } result = relation_select(handle, rel, adt); break; case AQL_TYPE_INSERT: result = relation_insert(rel, adt->values); break; #if DB_FEATURE_JOIN case AQL_TYPE_JOIN: if(handle == NULL) { result = DB_ARGUMENT_ERROR; break; } handle->left_rel = relation_load(adt->relations[first_rel_arg]); if(handle->left_rel == NULL) { break; } handle->right_rel = relation_load(adt->relations[first_rel_arg + 1]); if(handle->right_rel == NULL) { relation_release(handle->left_rel); break; } result = relation_join(handle, adt); break; #endif /* DB_FEATURE_JOIN */ default: break; } if(rel != NULL) { if(handle == NULL || !(handle->flags & DB_HANDLE_FLAG_PROCESSING)) { relation_release(rel); } } return result; }