示例#1
0
dict_t dict_remove(dict_t dict, word_t word) {
    assert((dict != NULL) && (word != NULL) && (dict_exists(dict, word)));
    index_t index_word = index_from_string(word);
    list_t new_list = list_remove(dict->list, index_word);
    index_word = index_destroy(index_word);
    dict->list = new_list;
    dict->length--;
    return (dict);
}
示例#2
0
def_t dict_search(dict_t dict, word_t word) {
    assert(dict != NULL && word != NULL && dict_exists(dict, word));
    index_t palabra = NULL;
    palabra = index_from_string(word);
    data_t definition = list_search(dict->list, palabra);
    def_t result = data_to_string(definition);
    palabra = index_destroy(palabra);
    assert(result != NULL);
    return (result);
}
示例#3
0
bool dict_exists(dict_t dict, word_t word) {
    assert(dict != NULL && word != NULL);
    bool result = false;
    index_t palabra = index_from_string(word);
    data_t definition = list_search(dict->list, palabra);
    if (definition != NULL)
        result = true;
    palabra = index_destroy(palabra);
    return (result);
}
示例#4
0
文件: hfile.c 项目: pipul/lab
void hfile_destroy(hfile_t *fp)
{
	meta_t *o;
	if (fp == NULL)
		return;
	hinfo_destroy(fp->fileinfo);
	htail_destroy(fp->trailer);
	for (o = index_head(fp->metas); o != NULL; o = meta_next(o))
		block_destroy(fp->blocks[o->id]);
	index_destroy(fp->metas);
	bloom_destroy(fp->bloom);
	free(fp->blocks);
	hfile_free(fp);
}
示例#5
0
bool dict_exists(dict_t dict, word_t word) {

    /*Precondition verification*/
    assert(dict != NULL);
    assert(word != NULL);

    index_t index = index_from_string(word);
    data_t  data  = bst_search(dict->data, index);

    index_destroy(index);

    return (data != NULL);

}
示例#6
0
dict_t dict_remove(dict_t dict, word_t word) {

    /*Precondition verification*/
    assert(word != NULL);
    assert(dict != NULL);
    assert(dict_exists(dict, word));

    index_t index = index_from_string(word);
    
    dict->data      = bst_remove(dict->data, index);
    dict->length   -= 1;

    index_destroy(index);

    return (dict);
}
示例#7
0
def_t dict_search(dict_t dict, word_t word) {

    /*Precondition verification*/
    assert(dict != NULL);
    assert(word != NULL);
    assert( dict_exists(dict, word) ); 

    index_t index = index_from_string(word);
    data_t  data  = bst_search(dict->data, index);

    def_t definicion = data_to_string(data);

    index_destroy(index);

    /*Postcondition verification*/    
    assert(definicion != NULL);

    return (definicion);

}
static void write_index(const char *filename)
{
	struct index_node *index;
	char *line;
	FILE *cfile;
	
	cfile = fopen(filename, "w");
	if (!cfile)
		fatal("Could not open %s for writing: %s\n",
		      filename, strerror(errno));
	
	index = index_create();
	
	while((line = getline_wrapped(stdin, NULL))) {
		index_insert(index, line);
		free(line);
	}
	
	index_write(index, cfile);
	index_destroy(index);
	fclose(cfile);
}
示例#9
0
文件: input.c 项目: NikNakk/snpStats
SEXP insnp_new(const SEXP Filenames, const SEXP Sample_id, const SEXP Snp_id,
	       const SEXP Diploid, const SEXP Fields,  
	       const SEXP Codes, const SEXP Threshold, const SEXP Lower, 
	       const SEXP Sep, const SEXP Comment, const SEXP Skip, 
	       const SEXP Simplify, const SEXP Verbose, 
	       const SEXP In_order, const SEXP Every){

  /* Process arguments */ 
  
  if (TYPEOF(Verbose)!=LGLSXP)
    error("Argument type error: Verbose");
  if (length(Verbose)>1)
    warning("Only first element of argument used: Verbose");
  int verbose = *INTEGER(Verbose);

  if (TYPEOF(In_order)!=LGLSXP)
    error("Argument type error: In_order");
  if (length(In_order)>1)
    warning("Only first element of argument used: In_order");
  int in_order = *INTEGER(In_order);

  if (TYPEOF(Filenames)!=STRSXP)
    error("Argument type error: Filenames");
  int Nfile = length(Filenames);

  int Nsample=0;
  if (TYPEOF(Sample_id)==STRSXP)
    Nsample = length(Sample_id);
  else if (TYPEOF(Sample_id)!=NILSXP)
    error("Argument type error: Sample_id");
  
  int Nsnp=0;
  if (TYPEOF(Snp_id)==STRSXP)
    Nsnp = length(Snp_id);
  else if (TYPEOF(Snp_id)!=NILSXP)
    error("Argument type error: Snp_id");
  
  /* file is 1 = sample, 2 = snp, 0 = irrelevant */

  int file_is = 0;
  if (!Nsample) {
    if (Nsnp) {
      Nsample = Nfile;
      Rprintf("Each file is assumed to concern a single sample\n"); 
      Rprintf("(Sample IDs are assumed to be included in filenames)\n");
      file_is = 1;
    }
    else 
      error("No sample or SNP IDs specified");
  }
  else if (!Nsnp) {
    Nsnp = Nfile;
    Rprintf("Each file is assumed to concern a single SNP\n");
    Rprintf("(SNP IDs are assumed to be included in filenames)\n");
    file_is = 2;
  }

  /* If not in order, set up hash tables */

  index_db sample_index = NULL;
  index_db snp_index = NULL;
  if (!in_order) {
    if (file_is != 1) 
      sample_index = create_name_index(Sample_id);
    if (file_is != 2) 
      snp_index = create_name_index(Snp_id);
  }

  int *diploid=NULL;
  if (TYPEOF(Diploid)==LGLSXP) {
    if (length(Diploid)!=Nsample)
      error("Argument length error: diploid argument");
    diploid = LOGICAL(Diploid);
  }
  else if (TYPEOF(Diploid)!=NILSXP)
    error("Argument type error: diploid argument");
  
  if (TYPEOF(Fields)!=INTSXP) 
    error("Argument type error: Fields");
  int *fields = INTEGER(Fields);
  int fsamp=0, fsnp=0, fgt=0, fa1=0, fa2=0, fconf=0;
  SEXP Fnames = getAttrib(Fields, R_NamesSymbol);
  if (TYPEOF(Fnames)==NILSXP) 
    error("Argument error: Fields argument has no names");
  int fmax = 0;
  int Nfield = length(Fields);
  for (int i=0; i<Nfield; i++) {
    const char *fname = CHAR(STRING_ELT(Fnames, i));
    int fi = fields[i];
    if (!strcmp(fname, "sample"))
      fsamp = fi;
    else if (!strcmp(fname, "snp"))
      fsnp = fi;
    else if (!strcmp(fname, "genotype"))
      fgt = fi;
    else if (!strcmp(fname, "allele1"))
      fa1 = fi;
    else if (!strcmp(fname, "allele2"))
      fa2 = fi;
    else if (!strcmp(fname, "confidence"))
      fconf = fi;
    else
      error("Unrecognized input field name: %s", fname);
    if (fi>fmax) 
      fmax = fi;
  }
  if (verbose) {
    Rprintf("Reading one call per input line\n");
    if (fsamp) {
      Rprintf("   Sample id is in field %d", fsamp);
      if (file_is==1) 
	Rprintf(" (ignored)\n");
      else
	Rprintf("\n");
    }
    if (fsnp) {
      Rprintf("   SNP id is in field %d", fsnp);
      if (file_is==2) 
	Rprintf(" (ignored)\n");
      else
	Rprintf("\n");
    }
    if (fgt)
      Rprintf("   Genotype is in field %d\n", fgt);
    if (fa1)
      Rprintf("   Allele 1 is in field %d\n", fa1);
    if (fa2)
      Rprintf("   Allele 2 is in field %d\n", fa2);
    if (fconf)
      Rprintf("   Confidence score is in field %d\n", fconf);
  }
  
  if (file_is==1)
    fsamp = 0;
  else if (file_is==2)
    fsnp = 0;


  /* Allele or genotype coding? */

  int gcoding;
  if (fgt) {
    if (fa1 || fa2) 
      error("Coding must be by genotype OR allele, not both");
    gcoding = 1;
  }
  else {
    if (!(fa1 && fa2)) 
      error("No genotype or allele field(s) specified");
    if (!(fa1 && fa2)) 
      error("Field positions for both alleles must be specified"); 
    gcoding = 0;
  }

  int nuc = 0;
  if (TYPEOF(Codes)!=STRSXP)
    error("Argument type error: Codes");
  if (length(Codes)==1) {
    SEXP Code = STRING_ELT(Codes, 0);
    const char *code = CHAR(Code);
    if (!strcmp(code, "nucleotide"))
      nuc = 1;
    else
      error("Unrecognized coding: %s", code);
  }
  else {
    int ncode = length(Codes);
    if (gcoding) {
      if (diploid) {
	if (ncode!=5) {
	  if (ncode==3)
	    warning("Genotype coding for X: haploid genotypes are assumed to be coded as homozygous");
	  else
	    error("Genotype coding for X.snp: three or five genotype codes must be specified");
	}
      }
      else {
	if (ncode!=3)
	  error("Genotype coding: three genotype codes must be specified");
      }
    }
    else {
      if (ncode!=2) 
	error("Allele coding: two allele codes must be specified");
    }
  }

  if (TYPEOF(Threshold)==NILSXP && !fconf) 
    error("Argument type error: no threshold argument");
  if (TYPEOF(Threshold)!=REALSXP)
    error("Argument type error: Threshold");
  double threshold = *REAL(Threshold);
  if (fconf && threshold==NA_REAL)
    error("Confidence score is read but no threshold is set");

  if (TYPEOF(Lower)!=LGLSXP)
    error("Argument type error: Lower");
  if (length(Lower)>1)
    warning("Only first element of argument used: Lower");
  int lower = *INTEGER(Lower);

  char sep = ' ';
  if (TYPEOF(Sep)==STRSXP) {
    if (length(Sep)>1)
      warning("Only first element of argument used: Sep");
    const char *c = CHAR(STRING_ELT(Sep, 0));
    if (strlen(c)>1) 
      warning("Only first character used: Sep");
    sep = c[0];
  }
  else if (TYPEOF(Sep)!=NILSXP) 
    error("Argument type error: Sep");

  char comment = (char) 0;
  if (TYPEOF(Comment)==STRSXP) {
    if (length(Sep)>1)
      warning("Only first element of argument used: Comment");
    const char *c = CHAR(STRING_ELT(Comment, 0));
    if (strlen(c)>1) 
      warning("Only first character used: Comment");
    comment = c[0];
  }
  else if (TYPEOF(Comment)!=NILSXP) 
    error("Argument type error: Comment");

  int skip = 0;
  if (TYPEOF(Skip)==INTSXP) {
    if (length(Skip)>1)
      warning("Only first element used: Skip");
    skip = INTEGER(Skip)[0];
  }
  else if (TYPEOF(Skip)!=NILSXP) 
    error("Argument type error: Skip");
 
  if (TYPEOF(Simplify)!=LGLSXP)
    error("Argument type error: Simplify");
  if (length(Simplify)>2)
    error("Argument length error: Simplify");
  int *simplify = INTEGER(Simplify);
  if (length(Simplify)==1)
    simplify[1] = simplify[0];

  int every=0;
  if (TYPEOF(Every)==INTSXP) {
    if (length(Every)>1) 
      warning("Only first element used: Every");
    every = INTEGER(Every)[0];
  }
  else if (TYPEOF(Every)!=NILSXP)
    error("Argument type error: Every");
    

  /* Create output object and initialise to zero */

  if (verbose) {
    if (diploid)
      Rprintf("Reading XSnpMatrix with %d rows and %d columns\n", 
	      Nsample, Nsnp);
    else
      Rprintf("Reading SnpMatrix with %d rows and %d columns\n", 
	      Nsample, Nsnp);
  }
  SEXP Result, Dimnames, Package, Class;
  PROTECT(Result = allocMatrix(RAWSXP, Nsample, Nsnp));
  PROTECT(Dimnames = allocVector(VECSXP, 2));
  if (simplify[0]) {
    SET_VECTOR_ELT(Dimnames, 0, 
		   simplify_names(file_is==1? Filenames: Sample_id));
  }
  else {
    SET_VECTOR_ELT(Dimnames, 0, 
		   duplicate(file_is==1? Filenames: Sample_id));
  }
  if (simplify[1]) {
    SET_VECTOR_ELT(Dimnames, 1, 
		   simplify_names(file_is==2? Filenames: Snp_id));
  }
  else {
    SET_VECTOR_ELT(Dimnames, 1, 
		   duplicate(file_is==2? Filenames: Snp_id));
  }
  setAttrib(Result, R_DimNamesSymbol, Dimnames);

  /* Class */

  PROTECT(Class = allocVector(STRSXP, 1));
  if (diploid) {
    R_do_slot_assign(Result, mkString("diploid"), Diploid);
    SET_STRING_ELT(Class, 0, mkChar("XSnpMatrix"));
  }
  else {
    SET_STRING_ELT(Class, 0, mkChar("SnpMatrix"));
  }
  PROTECT(Package = allocVector(STRSXP, 1));
  SET_STRING_ELT(Package, 0, mkChar("snpStats"));
  setAttrib(Class, install("package"), Package);
  classgets(Result, Class);
  SET_S4_OBJECT(Result);
  unsigned char *result = RAW(Result);
  memset(result, 0x00, Nsample*Nsnp);

  /* Read in data */

  char field[MAX_FLD];
  int Naccept = 0, Nreject = 0, Nocall = 0, Nskipped = 0, Nxerror = 0;
  int i_this = 0, j_this = 0;
  const char *this_sample=NULL, *this_snp=NULL;
  if (fsamp) {
    this_sample = CHAR(STRING_ELT(Sample_id, 0)); 
  }
  if (fsnp) {
    this_snp = CHAR(STRING_ELT(Snp_id, 0));
  }
  if (verbose) {
    Rprintf("                             Cumulative totals\n");
    Rprintf("                    -----------------------------------\n");
    Rprintf("    File     Line   Accepted Rejected  No call  Skipped    File name\n");
  }
  
  /* slowest varying 0 = don't know, 1 = sample, 2 = snp */

  int slowest = file_is, last=Nsample*Nsnp-1;
  int advance = 0, finished=0;

  for (int f=0; f<Nfile; f++) {
    /* Open input file */
    const char *filename = CHAR(STRING_ELT(Filenames, f));
    if (verbose) {
      int lfn = strlen(filename); 
      if (lfn > 20) {
	Rprintf("%59s...%-17s\r", "", filename+lfn-17);
      }
      else
	Rprintf("%59s%-20s\r", "", filename);
    }
    gzFile infile = gzopen(filename, "rb");
    if (!infile) {
      warning("Failure to open input file: %s", filename);
      continue;
    }
    int fterm = 2, line = 0, found_in_file = 0;
    /* Skip any header lines */
    for (int i=0; i<skip; i++) {
      line++;
      if (skip_to_eol(infile)==3)
	error("End-of-file reached on line %d", line);
    }
    Nskipped += skip;
    /* Read data lines */
 
    while (fterm!=3) {

      /* Read a line */

      line++;
      if (verbose && every && !(line % every)) 
	Rprintf("%8d %8d %10d %8d %8d %8d\r", 
		f+1, line, Naccept, Nreject, Nocall, Nskipped);
      int genotype=0, allele1=0, allele2=0;
      /* wanted is coded as:
	 1  if this call is to be accepted
	 0  if it is not wanted (or a comment line)
	 -1 if rejected due to insufficient confidence
	 -2 coded as no-call
      */
      int wanted = 1; 
      char sampid[MAX_FLD], snpid[MAX_FLD];
      char gtype1[MAX_FLD], gtype2[MAX_FLD];
      char cscore[MAX_FLD];
      sampid[0] = snpid[0] = cscore[0] = (char) 0;
      for (int r=1; (r<=fmax); r++) {
	fterm = next_field(infile, sep, comment, '_', field, MAX_FLD);
	if (!fterm) 
	  error("Field overflow: line %d, field %d", line, r);
	if ((fterm>1) && (r<fmax)) {
	  if (r==1) {
	    if(!field[0]) {
	    /* Empty line or comment line */
	      wanted = 0;
	      if (fterm==2) {
		Nskipped++;
		continue;
	      }
	      else
		break;
	    }
	  }
	  error("Incomplete line: %d (last field read: %d = %s)", 
		line, r, field);
	}
	/* Save fields */
	if (r==fsamp) {
	  strncpy(sampid, field, MAX_FLD-1);
	}
	else if (r==fsnp) {
	  strncpy(snpid, field, MAX_FLD-1);
	}
	else if (r==fconf) {
	  strncpy(cscore, field, MAX_FLD-1);
	}
	else if (r==fgt) 
	  strncpy(gtype1, field, MAX_FLD-1);
        else if (r==fa1) 
	  strncpy(gtype1, field, MAX_FLD-1);
	else if (r==fa2) 
	  strncpy(gtype2, field, MAX_FLD-1);
	else {
	  /* skip field */
	}
      } /* Matches: for (int r=1; (r<=fmax); r++) { */

      if (!wanted) 
	continue;

      /* Discard any further fields */

      if (fterm<2) {
	fterm = skip_to_eol(infile);
      }

      /* Find next target and check matches */

      int match_sample, match_snp;
      if (in_order) {

	/* Advance to next target read */

	if (advance) {
	  
	  /* 
	     If unknown, determine sort order by seeing which indicator 
	     has changed
	  */

	  if (!slowest) {
	    if (strcmp(this_sample, sampid)) 
	      slowest = 2; /* sample fastest, SNP slowest */
	    else if (strcmp(this_snp, snpid))
	      slowest = 1; /* SNP fastest, sample slowest */
	    else
	      error("Error in input file sort order");
	  }
	  
	  /* Now advance fastest varying indicator */
	  
	  if (slowest==1) {
	    j_this++;
	    if (j_this==Nsnp) {
	      j_this = 0;
	      if (fsamp) { 
		i_this++;
		if (i_this==Nsample) {
		  finished = 1;
		  break;
		}
		else
		  this_sample =  CHAR(STRING_ELT(Sample_id, i_this)); 
	      }
	    }
	    this_snp =  CHAR(STRING_ELT(Snp_id, j_this));
	  }
	  else {
	    i_this++;
	    if (i_this==Nsample) {
	      i_this = 0;
	      if (fsnp) {
		j_this++;
		if (j_this==Nsnp) {
		  finished = 1;
		  break;
		}
		else 
		  this_snp =  CHAR(STRING_ELT(Snp_id, j_this));
	      }
	    }
	    this_sample =  CHAR(STRING_ELT(Sample_id, i_this));
	  }
	}
	
	/* Does current line match current target? */
	
	match_sample = file_is==1 || !strcmp(this_sample, sampid);
	match_snp = file_is==2 || !strcmp(this_snp, snpid);
      }

      else { /* Not in order */

	if (file_is!=1) {
	  if (this_sample && strcmp(this_sample, sampid))
	    i_this = index_lookup(sample_index, sampid);
	  if (i_this >= 0) {
	    this_sample =  CHAR(STRING_ELT(Sample_id, i_this)); 
	    match_sample = 1;
	  }
	  else {
	    this_sample = NULL;
	    match_sample = 0;
	  }
	}
	else {
	  match_sample = 1;
	  i_this = f;
	}
	if (file_is!=2) {
	  if (this_snp && strcmp(this_snp, snpid))
	    j_this = index_lookup(snp_index, snpid);
	  if (j_this >= 0) {
	    this_snp = CHAR(STRING_ELT(Snp_id, j_this));
	    match_snp = 1;
	  }
	  else {
	    this_snp = NULL;
	    match_snp = 0;
	  }
	}
	else {
	  match_snp = 1;
	  j_this = f;
	}
      }

      if (match_sample && match_snp) {

	/* Next target read found in file(s) */

	found_in_file++;
	int ij_this = j_this*Nsample + i_this;
	finished = (ij_this==last);
	
	/* Check confidence score */
	
	if (fconf) {
	  double conf;
	  if (sscanf(cscore, "%lf", &conf)!=1) 
	    error("Failure to read confidence score: line %d", line);
	  if ((lower && conf<threshold) || (!lower && conf>threshold)) {
	    wanted = -1;
	    Nreject++;
	    if (finished)
	      break;
	    else
	      advance = 1;
	    continue;
	  }
	}
	
	/* Decode genotype */
	
	int which;
	if (gcoding) {
	  if (nuc) {
	    switch (strlen(gtype1)) {
	    case 0:
	      allele1 = allele2 = 0;
	      break;
	    case 1:
	      allele1 = allele2 = nucleotide(gtype1[0]);
	      break;
	    case 2:
	      allele1 = nucleotide(gtype1[0]);
	      allele2 = nucleotide(gtype1[1]);
	      break;
	    default:
	      error("Nucleotide coded genotype should be 2 character string: line %d", line);
	    }
	  }
	  else {
	    which = str_inlist(Codes, gtype1);
	    if (!which)
	      genotype = 0;
	    else if (which>3) 
	      genotype = 2*which - 7;
	    else
	      genotype = which;
	  }
	}
	else {
	  if (nuc) {
	    allele1 = nucleotide(gtype1[0]);
	    allele2 = nucleotide(gtype2[0]);
	  }
	  else {
	    allele1 = str_inlist(Codes, gtype1);
	    allele2 = str_inlist(Codes, gtype2);
	  }
	}

	/* Successful read, store genotype in result[i_this, j_this] */
	
	if (nuc || !gcoding) {
	  if (allele2 < allele1) {
	    genotype = allele2;
	    allele2 = allele1;
	    allele1 = genotype;
	  }
	  if (allele1 && allele2) 
	    genotype = allele1 + (allele2*(allele2-1))/2;	
	  else
	    genotype = 0;
	}
	if (genotype) {
	  if (diploid && !diploid[i_this] && (genotype==2))
	    Nxerror++;
	  else {
	    Naccept++;
	    result[ij_this] = (unsigned char) genotype;
	  }
	}
	else {
	  wanted = -2;
	  Nocall++;
	}
	
	/* Flag need to advance to next target */

	advance = 1;
      } /* matches if (match_sample && match_snp) { */
      else {
	Nskipped++;

	/* Flag no advance to next target */

	advance = 0;	
      }
      if (finished)
	break;
    } /* matches: while (fterm!=3) { */
    if (file_is==1)  
      i_this++;
    if (file_is==2) 
      j_this++;
    if (verbose) {
      Rprintf("%8d %8d %10d %8d %8d %8d\r", f+1, line, Naccept, Nreject, Nocall, Nskipped);
    }
    if(!found_in_file)
      warning("No calls found in file %s", filename);
    gzclose(infile);
    if (finished)
      break;
  }

  /* Warnings */

  if (in_order && !finished) 
    warning("End of data reached before search completed");
  if (Nxerror) 
    warning("%d haploid genotypes were coded as heterozygous; set to NA", Nxerror);
  if (Nskipped)
    warning("%d lines of input file(s) were skipped", Nskipped);

  /* Report */

  if (verbose)
    Rprintf("\n");
  Rprintf("%d genotypes successfully read\n", Naccept);
  if (Nreject)
    Rprintf("%d genotypes were rejected due to low confidence\n", Nreject);
  if (Nocall)
    Rprintf("%d genotypes were not called\n", Nocall);
  if (Nsample*Nsnp > Naccept+Nxerror+Nreject+Nocall)
    Rprintf("%d genotypes could not be found on input file(s)\n",
	    Nsample*Nsnp - Naccept - Nreject - Nxerror-Nocall);
  if (nuc) {
    if (verbose)
      Rprintf("Recasting and checking nucleotide coding\n");
    int none_snps = recode_snp(result, Nsample, Nsnp);
    if (none_snps) {
      Rprintf("%d polymorphisms were not SNPs and have been set to NA ", 
	      none_snps);
      Rprintf("(see warnings for details)\n");
    }
  }
  UNPROTECT(4);
  
  /* Destroy hash indexes */

  if (sample_index)
    index_destroy(sample_index);
  if (snp_index)
    index_destroy(snp_index);

  return Result;
}
示例#10
0
文件: backup.c 项目: mavslh/destor
int backup_server(char *path) {
	Jcr *jcr = new_write_jcr();
	strcpy(jcr->backup_path, path);
	if (access(jcr->backup_path, 4) != 0) {
		free(jcr);
		puts("This path does not exist or can not be read!");
		return -1;
	}

	if (index_init() == FALSE) {
		return -1;
	}

	jcr->job_id = get_next_job_id();
	if (is_job_existed(jcr->job_id)) {
		printf("job existed!\n");
		free(jcr);
		return FAILURE;
	}

	if (jcr->job_id == 0) {
		destor_stat->simulation_level = simulation_level;
	} else {
		if (simulation_level <= SIMULATION_RECOVERY
				&& destor_stat->simulation_level >= SIMULATION_APPEND
				|| simulation_level >= SIMULATION_APPEND
						&& destor_stat->simulation_level <= SIMULATION_RECOVERY) {
			dprint(
					"the current simulation level is not matched with the destor stat!");
			return FAILURE;
		}
	}

	jcr->job_volume = create_job_volume(jcr->job_id);

	puts("==== backup begin ====");
	puts("==== transfering begin ====");

	struct timeval begin, end;
	gettimeofday(&begin, 0);
	if (backup(jcr) != 0) {
		free(jcr);
		return FAILURE;
	}
	gettimeofday(&end, 0);

	index_destroy();

	jcr->time = end.tv_sec - begin.tv_sec
			+ (double) (end.tv_usec - begin.tv_usec) / (1000 * 1000);
	puts("==== transferring end ====");

	printf("job id: %d\n", jcr->job_id);
	printf("backup path: %s\n", jcr->backup_path);
	printf("number of files: %d\n", jcr->file_num);
	//printf("number of chunks: %d\n", jcr->chunk_num);
	printf("number of dup chunks: %d\n", jcr->number_of_dup_chunks);
	printf("total size: %ld\n", jcr->job_size);
	printf("dedup size: %ld\n", jcr->dedup_size);
	printf("dedup efficiency: %.4f, %.4f\n",
			jcr->job_size != 0 ?
					(double) (jcr->dedup_size) / (double) (jcr->job_size) : 0,
			jcr->job_size / (double) (jcr->job_size - jcr->dedup_size));
	printf("elapsed time: %.3fs\n", jcr->time);
	printf("throughput: %.2fMB/s\n",
			(double) jcr->job_size / (1024 * 1024 * jcr->time));
	printf("zero chunk count: %d\n", jcr->zero_chunk_count);
	printf("zero_chunk_amount: %ld\n", jcr->zero_chunk_amount);
	printf("rewritten_chunk_count: %d\n", jcr->rewritten_chunk_count);
	printf("rewritten_chunk_amount: %ld\n", jcr->rewritten_chunk_amount);
	printf("rewritten rate in amount: %.3f\n",
			jcr->rewritten_chunk_amount / (double) jcr->job_size);
	printf("rewritten rate in count: %.3f\n",
			jcr->rewritten_chunk_count / (double) jcr->chunk_num);

	destor_stat->data_amount += jcr->job_size;
	destor_stat->consumed_capacity += jcr->job_size - jcr->dedup_size;
	destor_stat->saved_capacity += jcr->dedup_size;
	destor_stat->number_of_chunks += jcr->chunk_num;
	destor_stat->number_of_dup_chunks += jcr->number_of_dup_chunks;
	destor_stat->zero_chunk_count += jcr->zero_chunk_count;
	destor_stat->zero_chunk_amount += jcr->zero_chunk_amount;
	destor_stat->rewritten_chunk_count += jcr->rewritten_chunk_count;
	destor_stat->rewritten_chunk_amount += jcr->rewritten_chunk_amount;
	destor_stat->sparse_chunk_count += sparse_chunk_count;
	destor_stat->sparse_chunk_amount += sparse_chunk_amount;
	destor_stat->container_num = container_volume.container_num;
	destor_stat->index_memory_overhead = index_memory_overhead;

	printf("read_time : %.3fs, %.2fMB/s\n", jcr->read_time / 1000000,
			jcr->job_size * 1000000 / jcr->read_time / 1024 / 1024);
	printf("chunk_time : %.3fs, %.2fMB/s\n", jcr->chunk_time / 1000000,
			jcr->job_size * 1000000 / jcr->chunk_time / 1024 / 1024);
	printf("name_time : %.3fs, %.2fMB/s\n", jcr->name_time / 1000000,
			jcr->job_size * 1000000 / jcr->name_time / 1024 / 1024);
	printf("filter_time : %.3fs, %.2fMB/s\n", jcr->filter_time / 1000000,
			jcr->job_size * 1000000 / jcr->filter_time / 1024 / 1024);
	printf("write_time : %.3fs, %.2fMB/s\n", jcr->write_time / 1000000,
			jcr->job_size * 1000000 / jcr->write_time / 1024 / 1024);
	printf("index_search_time : %.3fs, %.2fMB/s\n", search_time / 1000000,
			jcr->job_size * 1000000 / search_time / 1024 / 1024);
	printf("index_update_time : %.3fs, %.2fMB/s\n", update_time / 1000000,
			jcr->job_size * 1000000 / update_time / 1024 / 1024);
	puts("==== backup end ====");

	jcr->job_volume->job.job_id = jcr->job_id;
	jcr->job_volume->job.is_del = FALSE;
	jcr->job_volume->job.file_num = jcr->file_num;
	jcr->job_volume->job.chunk_num = jcr->chunk_num;

	strcpy(jcr->job_volume->job.backup_path, jcr->backup_path);

	update_job_volume_des(jcr->job_volume);
	close_job_volume(jcr->job_volume);
	jcr->job_volume = 0;

	double seek_time = 0.005; //5ms
	double bandwidth = 120 * 1024 * 1024; //120MB/s

	double index_lookup_throughput = jcr->job_size
			/ (index_read_times * seek_time
					+ index_read_entry_counter * 24 / bandwidth) / 1024 / 1024;

	double write_data_throughput = 1.0 * jcr->job_size * bandwidth
			/ (jcr->job_size - jcr->dedup_size) / 1024 / 1024;
	double index_read_throughput = 1.0 * jcr->job_size / 1024 / 1024
			/ (index_read_times * seek_time
					+ index_read_entry_counter * 24 / bandwidth);
	double index_write_throughput = 1.0 * jcr->job_size / 1024 / 1024
			/ (index_write_times * seek_time
					+ index_write_entry_counter * 24 / bandwidth);

	double estimated_throughput = write_data_throughput;
	if (estimated_throughput > index_read_throughput)
		estimated_throughput = index_read_throughput;
	/*if (estimated_throughput > index_write_throughput)
	 estimated_throughput = index_write_throughput;*/

	char logfile[] = "backup.log";
	int fd = open(logfile, O_WRONLY | O_CREAT, S_IRWXU);
	lseek(fd, 0, SEEK_END);
	char buf[512];
	/*
	 * job id,
	 * chunk number,
	 * accumulative consumed capacity,
	 * deduplication ratio,
	 * rewritten ratio,
	 * total container number,
	 * sparse container number,
	 * inherited container number,
	 * throughput,
	 * index memory overhead,
	 * index lookups,
	 * index updates,
	 */
	sprintf(buf, "%d %d %ld %.4f %.4f %d %d %d %.2f %ld %ld %ld\n", jcr->job_id,
			jcr->chunk_num, destor_stat->consumed_capacity,
			jcr->job_size != 0 ?
					(double) (jcr->dedup_size) / (double) (jcr->job_size) : 0,
			jcr->job_size != 0 ?
					(double) (jcr->rewritten_chunk_amount)
							/ (double) (jcr->job_size) :
					0, jcr->total_container_num, jcr->sparse_container_num,
			jcr->inherited_sparse_num,
			(double) jcr->job_size / (1024 * 1024 * jcr->time),
			index_memory_overhead, index_read_times, index_write_times);
	if (write(fd, buf, strlen(buf)) != strlen(buf)) {
	}
	close(fd);
	free_jcr(jcr);
	return SUCCESS;

}
示例#11
0
文件: ur-cmd.c 项目: spolu/ur
int
cmd_add (const char *path, bool recursive)
{
  char *prt_dir = NULL, *fname = NULL, *dname = NULL;
  struct stat64 st_buf;
  state_t ur = STATE_INITIALIZER;
  struct index index = INDEX_INITIALIZER; 
  int status;

  if (lstat64 (path, &st_buf) != 0) fail("%s does not exist", path);

  prt_dir = (char *) malloc (strlen (path) + 4);
  parent_dir (path, prt_dir);

  if (st_buf.st_mode & S_IFREG) {

    fname = (char *) malloc (strlen (path) + 4);
    filename (path, fname);	

    if (ur_check (prt_dir) == 0) 
      {
	if (state_init (&ur, prt_dir) != 0) fail ("fail reading state of %s", prt_dir);
	if (index_read (&ur, &index) != 0) fail ("fail reading index of %s", prt_dir);
	
	status = index_entry_get_status (&index, fname);
	status |= S_IADD;
	if (!(status & S_IPST))
	  status |= S_IPST;
	index_entry_set_status (&index, fname, status);

	if (index_write (&ur, &index) != 0) fail ("fail writing index of %s", prt_dir);

	index_destroy (&index);
	state_destroy (&ur);
      }
    else
      fail ("%s initialization failed", prt_dir);	

    free (fname); fname = NULL;    
  }

  if (st_buf.st_mode & S_IFDIR) 
    {
      dname = (char *) malloc (strlen (path) + 4);
      dirname (path, dname);	

      if (ur_check (path) == 0) {
	if (ur_check (prt_dir) == 0) 
	  {
	    if (state_init (&ur, prt_dir) != 0) fail ("fail reading state of %s", prt_dir);
	    if (index_read (&ur, &index) != 0) fail ("fail reading index of %s", prt_dir);
	    
	    status = index_entry_get_status (&index, dname);
	    status |= S_IADD;
	    if (!(status & S_IPST))
	      status |= S_IPST;	
	    index_entry_set_status (&index, dname, status);

	    if (index_write (&ur, &index) != 0) fail ("fail writing index of %s", prt_dir);
	    index_destroy (&index);
	    state_destroy (&ur);
	}
      }
      else {

	if (ur_create (path) != 0) fail ("%s initialization failed", path);	
	output ("init: %s", path);
	
	if (ur_check (prt_dir) == 0) {
	    if (state_init (&ur, prt_dir) != 0) fail ("fail reading state of %s", prt_dir);
	    if (index_read (&ur, &index) != 0) fail ("fail reading index of %s", prt_dir);
	    
	    status = index_entry_get_status (&index, dname);
	    status |= S_IADD;
	    if (!(status & S_IPST))
	      status |= S_IPST;	
	    index_entry_set_status (&index, dname, status);

	    if (index_write (&ur, &index) != 0) fail ("fail writing index of %s", prt_dir);
	    index_destroy (&index);
	    state_destroy (&ur);
	}
      }

      free (dname); dname = NULL;    
    }
  
  free (prt_dir); prt_dir = NULL;

  /*
   * recursion
   */
  if ((st_buf.st_mode & S_IFDIR) && recursive) 
    {
      DIR *dp;
      struct dirent *ep;
      
      dp = opendir (path);
      if (dp != NULL)
	{
	  while ((ep = readdir (dp))) {
	    if (ep->d_name[0] != '.') 
	      {		
		char *npath;
		npath = (char *) malloc (strlen (path) +
					 strlen (ep->d_name) + 2);
		if (path[strlen (path) -1] == '/')
		  sprintf (npath, "%s%s", path, ep->d_name);
		else
		  sprintf (npath, "%s/%s", path, ep->d_name);
		cmd_add (npath, recursive);
		free (npath);
	      }	    
	  }
	  (void) closedir (dp);
	}
      else
	fail ("could not open directory %s", path);      
    }
  
  return 0;
}
示例#12
0
文件: ur-cmd.c 项目: spolu/ur
int
cmd_status (const char *path, bool recursive)
{
  state_t ur = STATE_INITIALIZER;
  struct index index = INDEX_INITIALIZER; 
  struct stat64 st_buf;
  DIR *dp;
  struct dirent *ep;
  struct list_elem *e;
  char *branchname = NULL;
      
  if (lstat64 (path, &st_buf) != 0) fail("%s does not exist", path);
  ASSERT (st_buf.st_mode & S_IFDIR);

  if (ur_check (path) == 0) 
    {
      
      if (state_init (&ur, path) != 0) fail ("fail reading state of %s", path);
      if (index_read (&ur, &index) != 0) fail ("fail reading index of %s", path);

      index_update (&ur, &index);

      branchname = branch_get_head_name (&ur);
      printf ("*** %s \n    (branch: %s)\n", ur.path, branchname);
      free (branchname); branchname = NULL;

      // reading added dirty files
      for (e = list_begin (&index.entries); e != list_end (&index.entries);
	   e = list_next (e))
	{
	  struct index_entry *en = list_entry (e, struct index_entry, elem);	
	  if ((en->status & S_IPST) &&
	      (en->status & S_IADD) && 
	      (en->status & S_IDRT)) {
	    printf ("#  added     : %s/%s\n", path, en->name);
	  }
	}

      // reading dirty but not added
      for (e = list_begin (&index.entries); e != list_end (&index.entries);
	   e = list_next (e))
	{
	  struct index_entry *en = list_entry (e, struct index_entry, elem);	
	  if ((en->status & S_IPST) &&
	      !(en->status & S_IADD) && 
	      (en->status & S_IDRT) && 
	      (en->status & S_ITRK)) {
	    printf ("#  dirty     : %s/%s\n", path, en->name);
	  }
	}

      // reading untracked files
      for (e = list_begin (&index.entries); e != list_end (&index.entries);
	   e = list_next (e))
	{
	  struct index_entry *en = list_entry (e, struct index_entry, elem);	
	  if ((en->status & S_IPST) &&
	      !(en->status & S_ITRK) && 
	      !(en->status & S_IADD)) {
	    printf ("#  untracked : %s/%s\n", path, en->name);
	  }
	}
      
      index_destroy (&index);
      state_destroy (&ur);    
    }
  else {
示例#13
0
文件: aql-exec.c 项目: 1uk3/contiki
static db_result_t
aql_execute(db_handle_t *handle, aql_adt_t *adt)
{
  uint8_t optype;
  int first_rel_arg;
  db_result_t result;
  relation_t *rel;
  aql_attribute_t *attr;
  attribute_t *relattr;

  optype = AQL_GET_TYPE(adt);
  if(optype == AQL_TYPE_NONE) {
    /* No-ops always succeed. These can be generated by
       empty lines or comments in the query language. */
    return DB_OK;
  }

  /* If the ASSIGN flag is set, the first relation in the array is
     the desired result relation. */
  first_rel_arg = !!(adt->flags & AQL_FLAG_ASSIGN);

  if(optype != AQL_TYPE_CREATE_RELATION &&
     optype != AQL_TYPE_REMOVE_RELATION &&
     optype != AQL_TYPE_JOIN) {
    rel = relation_load(adt->relations[first_rel_arg]);
    if(rel == NULL) {
      return DB_NAME_ERROR;
    }
  } else {
    rel = NULL;
  }

  result = DB_RELATIONAL_ERROR;
  switch(optype) {
  case AQL_TYPE_CREATE_ATTRIBUTE:
    attr = &adt->attributes[0];
    if(relation_attribute_add(rel, DB_STORAGE, attr->name, attr->domain, 
       attr->element_size) != NULL) {
      result = DB_OK;
    }
    break;
  case AQL_TYPE_CREATE_INDEX:
    relattr = relation_attribute_get(rel, adt->attributes[0].name);
    if(relattr == NULL) {
      result = DB_NAME_ERROR;
      break;
    }
    result = index_create(AQL_GET_INDEX_TYPE(adt), rel, relattr);
    break;
  case AQL_TYPE_CREATE_RELATION:
    if(relation_create(adt->relations[0], DB_STORAGE) != NULL) {
      result = DB_OK;
    }
    break;
  case AQL_TYPE_REMOVE_ATTRIBUTE:
    result = relation_attribute_remove(rel, adt->attributes[0].name);
    break;
  case AQL_TYPE_REMOVE_INDEX:
    relattr = relation_attribute_get(rel, adt->attributes[0].name);
    if(relattr != NULL) {
      if(relattr->index != NULL) {
        result = index_destroy(relattr->index);
      } else {
        result = DB_OK;
      }
    } else {
      result = DB_NAME_ERROR;
    }
    break;
  case AQL_TYPE_REMOVE_RELATION:
    result = relation_remove(adt->relations[0], 1);
    break;
#if DB_FEATURE_REMOVE
  case AQL_TYPE_REMOVE_TUPLES:
     /* Overwrite the attribute array with a full copy of the original 
        relation's attributes. */
    adt->attribute_count = 0;
    for(relattr = list_head(rel->attributes);
        relattr != NULL;
        relattr = relattr->next) {
      AQL_ADD_ATTRIBUTE(adt, relattr->name, DOMAIN_UNSPECIFIED, 0);
    }
    AQL_SET_FLAG(adt, AQL_FLAG_INVERSE_LOGIC);
#endif /* DB_FEATURE_REMOVE */
  case AQL_TYPE_SELECT:
    if(handle == NULL) {
      result = DB_ARGUMENT_ERROR;
      break;
    }
    result = relation_select(handle, rel, adt);
    break;
  case AQL_TYPE_INSERT:
    result = relation_insert(rel, adt->values);
    break;
#if DB_FEATURE_JOIN
  case AQL_TYPE_JOIN:
    if(handle == NULL) {
      result = DB_ARGUMENT_ERROR;
      break;
    }
    handle->left_rel = relation_load(adt->relations[first_rel_arg]);
    if(handle->left_rel == NULL) {
      break;
    }
    handle->right_rel = relation_load(adt->relations[first_rel_arg + 1]);
    if(handle->right_rel == NULL) {
      relation_release(handle->left_rel);
      break;
    }
    result = relation_join(handle, adt);
    break;
#endif /* DB_FEATURE_JOIN */
  default:
    break;
  }

  if(rel != NULL) {
    if(handle == NULL || !(handle->flags & DB_HANDLE_FLAG_PROCESSING)) {
      relation_release(rel);
    }
  }

  return result;
}