Example #1
0
int main (int argc, char ** argv)
{
	//const clock_t begin_time = clock();
	
	if (argc<3) {
		print_usage ();
	}
	
	////////////////////////////////////////////////////////////
	// Init parameters
	//
	std::string input_file_name;
	std::string bv_file_name;
	std::string output_file_name;
	bool compress = false;
	
	////////////////////////////////////////////////////////////
	// Read command line arguments
	//
	int arg_pos = 1;
	while (arg_pos < argc){
		std::string flag = argv[arg_pos];
		if (flag[0] != '-') {
			if (input_file_name.empty()) {
				input_file_name = flag;
			} else if (bv_file_name.empty()){
				bv_file_name = flag;
			} else {
				std::cerr << "The mandatory files are already set, unknown file " << flag << " -> ignore\n";
			}
		} else if (flag.compare("-o") == 0) {
			arg_pos++;
			output_file_name = argv[arg_pos];
		} else if (flag.compare("-h") == 0) {
			print_usage ();
			return 0;
		} else if (flag.compare("-v") == 0) {
			std::cout << "\nextract_reads version " << version << "\n";
			return 0;
		} else {
			std::cerr << "Unknown option " << flag << "\n";
            print_usage ();
			return (0);
		}
		arg_pos++;
	}
	
	if (input_file_name.empty()) {
		std::cerr << "Error: An input file name is needed -> exit\n";
		print_usage ();
		return (0);
	} else if (bv_file_name.empty()) {
		std::cerr << "Error: A bv file name is needed -> exit\n";
		print_usage ();
		return (0);
	}
	
	////////////////////////////////////////////////////////////
	// Open the given file to check its type (fasta, fastq, gzip ?)
	//
	ReadFile * read_file = NULL;
	
	std::ifstream infile;
	infile.open(input_file_name.c_str());
	if (!infile.good()) {
		std::cerr << "Cannot open file file " << input_file_name << " -> ignore\n";
	}
	// Check the first char
	std::string basename = input_file_name.substr(input_file_name.rfind("/")+1);
	char c = infile.get();
	if (c == '>') {
		infile.close();
		read_file = new FastaFile(input_file_name, bv_file_name);
	} else if (c == '@') {
		infile.close();
		read_file = new FastqFile(input_file_name, bv_file_name);
	} else {
		infile.close();
		gzFile tmp_gz_file = (gzFile) gzopen(input_file_name.c_str(), "r");
		if (!tmp_gz_file) {
			std::cerr << "Cannot open file " << input_file_name << " -> ignore\n";
			exit(1);
		}
		c = gzgetc(tmp_gz_file);
		if (c == '>') {
			gzclose(tmp_gz_file);
			compress = true;
			read_file = new GzFastaFile(input_file_name, bv_file_name);
		} else if (c == '@') {
			gzclose(tmp_gz_file);
			compress = true;
			read_file = new GzFastqFile(input_file_name, bv_file_name);
		} else {
			std::cerr << "Unknown format: " << input_file_name << " -> ignore\n";
		}
	}
	
	////////////////////////////////////////////////////////////
	// Open the output file and write selected reads in it
	// Different behaviors if : - output file name is given
	//                          - compress is true or not
	//
	if (compress) {
		if (output_file_name.empty()) {
			std::cerr << "Error, try to compress results but no output file name is given\n";
			exit(1);
		}
		gzFile filetmp = (gzFile) gzopen(output_file_name.c_str(), "w6");
		if (filetmp == NULL) {
			std::cerr << "Error, cannot open file " << output_file_name << "\n";
		}
		std::string & current_read = read_file->get_next_read();
		while (!current_read.empty()) {
			gzprintf(filetmp, "%s", read_file->get_data().c_str());
			current_read = read_file->get_next_read();
		}
		gzclose(filetmp);
	} else {
		if (!output_file_name.empty()) {
			std::ofstream outfile;
			outfile.open (output_file_name.c_str());
			if (!outfile.good()) {
				std::cerr << "Cannot write on file " << output_file_name << "\n";
				return 1;
			}
			std::string & current_read = read_file->get_next_read();
			while (!current_read.empty()) {
				outfile << read_file->get_data();
				current_read = read_file->get_next_read();
			}
			outfile.close();
		} else {
			std::string & current_read = read_file->get_next_read();
			while (!current_read.empty()) {
				std::cout << read_file->get_data();
				current_read = read_file->get_next_read();
			}
		}
	}
	
	if (read_file != NULL) {
		delete read_file;
	}
	return 0;
}
Example #2
0
boolean TxCache::load(const char *path, const char *filename, int config)
{
    /* find it on disk */
    CPath cbuf(path, filename);

    gzFile gzfp = gzopen(cbuf, "rb");
    DBG_INFO(80, L"gzfp:%x file:%ls\n", gzfp, filename);
    if (gzfp)
    {
        /* yep, we have it. load it into memory cache. */
        int dataSize;
        uint64 checksum;
        GHQTexInfo tmpInfo;
        int tmpconfig;
        /* read header to determine config match */
        gzread(gzfp, &tmpconfig, 4);

        if (tmpconfig == config)
        {
            do
            {
                memset(&tmpInfo, 0, sizeof(GHQTexInfo));

                gzread(gzfp, &checksum, 8);

                gzread(gzfp, &tmpInfo.width, 4);
                gzread(gzfp, &tmpInfo.height, 4);
                gzread(gzfp, &tmpInfo.format, 2);

                gzread(gzfp, &tmpInfo.smallLodLog2, 4);
                gzread(gzfp, &tmpInfo.largeLodLog2, 4);
                gzread(gzfp, &tmpInfo.aspectRatioLog2, 4);

                gzread(gzfp, &tmpInfo.tiles, 4);
                gzread(gzfp, &tmpInfo.untiled_width, 4);
                gzread(gzfp, &tmpInfo.untiled_height, 4);

                gzread(gzfp, &tmpInfo.is_hires_tex, 1);

                gzread(gzfp, &dataSize, 4);

                tmpInfo.data = (uint8*)malloc(dataSize);
                if (tmpInfo.data)
                {
                    gzread(gzfp, tmpInfo.data, dataSize);

                    /* add to memory cache */
                    add(checksum, &tmpInfo, (tmpInfo.format & GR_TEXFMT_GZ) ? dataSize : 0);

                    free(tmpInfo.data);
                }
                else
                {
                    gzseek(gzfp, dataSize, SEEK_CUR);
                }

                /* skip in between to prevent the loop from being tied down to vsync */
                if (_callback && (!(_cache.size() % 100) || gzeof(gzfp)))
                    (*_callback)(L"[%d] total mem:%.02fmb - %ls\n", _cache.size(), (float)_totalSize / 1000000, filename);
            } while (!gzeof(gzfp));
            gzclose(gzfp);
        }
    }
    return !_cache.empty();
}
Example #3
0
/* If `name' has a gzip-like extension, try to uncompress it into a temporary
   file using gzip or zlib if available.  If this succeeds, return the name
   of the temporary file; return NULL otherwise.  */
static char *try_uncompress_with_gzip(const char *name)
{
#ifdef HAVE_ZLIB
    FILE *fddest;
    gzFile fdsrc;
    char *tmp_name = NULL;
    int len;

    if (!archdep_file_is_gzip(name)) {
        return NULL;
    }

    fddest = archdep_mkstemp_fd(&tmp_name, MODE_WRITE);

    if (fddest == NULL) {
        return NULL;
    }

    fdsrc = gzopen(name, MODE_READ);
    if (fdsrc == NULL) {
        fclose(fddest);
        ioutil_remove(tmp_name);
        lib_free(tmp_name);
        return NULL;
    }

    do {
        char buf[256];

        len = gzread(fdsrc, (void *)buf, 256);
        if (len > 0) {
            if (fwrite((void *)buf, 1, (size_t)len, fddest) < len) {
                gzclose(fdsrc);
                fclose(fddest);
                ioutil_remove(tmp_name);
                lib_free(tmp_name);
                return NULL;
            }
        }
    } while (len > 0);

    gzclose(fdsrc);
    fclose(fddest);

    return tmp_name;
#else
    char *tmp_name = NULL;
    int exit_status;
    char *argv[4];

    if (!archdep_file_is_gzip(name)) {
        return NULL;
    }

    /* `exec*()' does not want these to be constant...  */
    argv[0] = lib_stralloc("gzip");
    argv[1] = lib_stralloc("-cd");
    argv[2] = archdep_filename_parameter(name);
    argv[3] = NULL;

    ZDEBUG(("try_uncompress_with_gzip: spawning gzip -cd %s", name));
    exit_status = archdep_spawn("gzip", argv, &tmp_name, NULL);

    lib_free(argv[0]);
    lib_free(argv[1]);
    lib_free(argv[2]);

    if (exit_status == 0) {
        ZDEBUG(("try_uncompress_with_gzip: OK"));
        return tmp_name;
    } else {
        ZDEBUG(("try_uncompress_with_gzip: failed"));
        ioutil_remove(tmp_name);
        lib_free(tmp_name);
        return NULL;
    }
#endif
}
Example #4
0
/**
 * zif_file_decompress_zlib:
 **/
static gboolean
zif_file_decompress_zlib (const gchar *in, const gchar *out, ZifState *state, GError **error)
{
	gboolean ret = FALSE;
	gint size;
	gint written;
	gzFile f_in = NULL;
	FILE *f_out = NULL;
	guchar buf[ZIF_BUFFER_SIZE];
	GCancellable *cancellable;

	g_return_val_if_fail (in != NULL, FALSE);
	g_return_val_if_fail (out != NULL, FALSE);
	g_return_val_if_fail (zif_state_valid (state), FALSE);

	/* get cancellable */
	cancellable = zif_state_get_cancellable (state);

	/* open file for reading */
	f_in = gzopen (in, "rb");
	if (f_in == NULL) {
		g_set_error (error,
			     ZIF_UTILS_ERROR,
			     ZIF_UTILS_ERROR_FAILED_TO_READ,
			     "cannot open %s for reading", in);
		goto out;
	}

	/* open file for writing */
	f_out = fopen (out, "w");
	if (f_out == NULL) {
		g_set_error (error, ZIF_UTILS_ERROR, ZIF_UTILS_ERROR_FAILED_TO_WRITE,
			     "cannot open %s for writing", out);
		goto out;
	}

	/* read in all data in chunks */
	while (TRUE) {
		/* read data */
		size = gzread (f_in, buf, ZIF_BUFFER_SIZE);
		if (size == 0)
			break;

		/* error */
		if (size < 0) {
			g_set_error_literal (error, ZIF_UTILS_ERROR, ZIF_UTILS_ERROR_FAILED_TO_READ,
					     "failed read");
			goto out;
		}

		/* write data */
		written = fwrite (buf, 1, size, f_out);
		if (written != size) {
			g_set_error (error, ZIF_UTILS_ERROR, ZIF_UTILS_ERROR_FAILED_TO_WRITE,
				     "only wrote %i/%i bytes", written, size);
			goto out;
		}

		/* is cancelled */
		ret = !g_cancellable_is_cancelled (cancellable);
		if (!ret) {
			g_set_error_literal (error, ZIF_UTILS_ERROR, ZIF_UTILS_ERROR_CANCELLED, "cancelled");
			goto out;
		}
	}

	/* success */
	ret = TRUE;
out:
	if (f_in != NULL)
		gzclose (f_in);
	if (f_out != NULL)
		fclose (f_out);
	return ret;
}
Example #5
0
static int
gen_inp_init(void *conf, char *objname)
{
    gen_inp_conf_t *cf = (gen_inp_conf_t *)conf, cfd;
    char *s, value[128], truefn[256];
    char *sub_path = "tables";
    int ret;

    bzero(&cfd, sizeof(gen_inp_conf_t));
    cfd.mode = DEFAULT_INP_MODE;
    cf->modesc = 0;
    gen_inp_resource(&cfd, objname);
/*
 *  Resource setup.
 */
    if ((cfd.mode & INP_MODE_AUTOCOMPOSE))
	cf->mode |= INP_MODE_AUTOCOMPOSE;
    if ((cfd.mode & INP_MODE_AUTOUPCHAR)) {
	cf->mode |= INP_MODE_AUTOUPCHAR;
	if ((cfd.mode & INP_MODE_SPACEAUTOUP))
	    cf->mode |= INP_MODE_SPACEAUTOUP;
	if ((cfd.mode & INP_MODE_SELKEYSHIFT))
	    cf->mode |= INP_MODE_SELKEYSHIFT;
    }
    if ((cfd.mode & INP_MODE_AUTOFULLUP)) {
	cf->mode |= INP_MODE_AUTOFULLUP;
	if ((cfd.mode & INP_MODE_SPACEIGNOR))
	    cf->mode |= INP_MODE_SPACEIGNOR;
    }
    if ((cfd.mode & INP_MODE_AUTORESET))
	cf->mode |= INP_MODE_AUTORESET;
    else if ((cfd.mode & INP_MODE_SPACERESET))
	cf->mode |= INP_MODE_SPACERESET;
    if ((cfd.mode & INP_MODE_WILDON))
	cf->mode |= INP_MODE_WILDON;
    cf->modesc = cfd.modesc;
    cf->disable_sel_list = cfd.disable_sel_list;
/*
 *  Read the IM tab file.
 */
    sprintf(value, "%s.tab", objname);
    if (oxim_check_datafile(value, sub_path, truefn, 256) == True) {
	cf->tabfn = (char *)strdup(truefn);
	if ((cf->zfp = gzopen(truefn, "rb")) == NULL)
	    return False;
        ret = loadtab(cf);
	if (!ret)
	{
	    gzclose(cf->zfp);
	    cf->zfp = NULL;
	}
    }
    else
	return False;

    if (cf->header.n_endkey) {
	if ((cfd.mode & INP_MODE_ENDKEY))
	    cf->mode |= INP_MODE_ENDKEY;
    }

    return ret;
}
Example #6
0
/*
 * GZIP file saver
 */ 
static void *gzip_open(const char *filename)
{
	return gzopen(filename, "wb9");
}
Example #7
0
File: a.c Project: afeldman/waf
void a() {
	gzFile f = gzopen("test.gz", "wb9");
}
Example #8
0
int pairs_split(int argc, char *argv[]) {
  kseq_t **seq, *tmp;
  FILE *fpout[] = {NULL, NULL, NULL};
  gzFile fp;
  int c, l, i, strict=1, min_length=0;
  unsigned total[]={0, 0}, removed[]={0, 0}, both_removed = 0;
  unsigned is_empty[]={0, 0};
  char *p, *tags[] = {"/1", "/2"};
  while ((c = getopt(argc, argv, "1:2:u:n")) >= 0) {
    switch (c) {
    case '1': 
      fpout[0] = fopen(optarg, "w");
      break;
    case '2':
      fpout[1] = fopen(optarg, "w");
      break;
    case 'u':
      fpout[2] = fopen(optarg, "w");
      break;
    case 'm': 
      min_length = atoi(optarg); 
      if (min_length < 1) fprintf(stderr, "[%s] error: minimum length must be >= 1\n", __func__);
      return 1;
      break;
    case 'n': strict = 0; break;
    default: return 1;
    }
  }

  if (optind == argc) return split_usage();

  for (i = 0; i < 3; ++i) {
    if (!fpout[i]) {
      fprintf(stderr, "[%s] error: arguments -1, -2, and -u are required.", __func__);
      return 1;
    }
  }

  fp = (strcmp(argv[optind], "-") == 0) ? gzdopen(fileno(stdin), "r") : gzopen(argv[optind], "r");
  
  seq = calloc(2, sizeof(kseq_t*));
  for (i = 0; i < 2; ++i) seq[i] = malloc(sizeof(kseq_t));
  tmp = kseq_init(fp);
  while ((l=kseq_read(tmp)) >= 0) {
    /* always read in chunks of two FASTX entries */
    cpy_kseq(seq[0], tmp);
    l = kseq_read(tmp);
    if (l < 0) break;
    cpy_kseq(seq[1], tmp);

    for (i = 0; i < 2; ++i) {
      /* remove /1 and /2 tags */
      p = strstr(seq[i]->name.s, tags[i]);
      if (p) {
	strncpy(p, "\0", 1);
      }
    }

    if (strcmp(seq[0]->name.s, seq[1]->name.s) != 0) {
      fprintf(stderr, "[%s] warning: interleaved reads names differ '%s' != '%s'\n", __func__, seq[0]->name.s, seq[1]->name.s);
      if (strict) return 1;
    }

    /* deal with unpaired cases (either no seq or single 'N') */
    for (i = 0; i < 2; ++i) {
      is_empty[i] = seq[i]->seq.l <= min_length || strcmp(seq[i]->seq.s, "N") == 0;
      removed[i] += is_empty[i];
      total[i] += 1;
    }
    
    if (!is_empty[0] && !is_empty[1]) {
      for (i = 0; i < 2; i++)      
	printseq(fpout[i], seq[i], seq[i]->seq.l, 0);
    } else if (is_empty[0] && is_empty[1]) {
      both_removed += 1;
      continue;
    } else {
      i = is_empty[0] ? 1 : 0;
      printseq(fpout[2], seq[i], seq[i]->seq.l, 0);
    }
  }
  if (total[0] != total[1]) {
    fprintf(stderr, "[%s] error: mismatched totals of interleaved pairs! %u != %u\n", __func__, total[0], total[1]);
    return 1;
  }
  fprintf(stderr, "totals: %u %u\nremoved: %u %u\n", total[0], total[1], removed[0], removed[1]);
  return 0;
}
Example #9
0
int main_bedcov(int argc, char *argv[])
{
    gzFile fp;
    kstring_t str;
    kstream_t *ks;
    hts_idx_t **idx;
    aux_t **aux;
    int *n_plp, dret, i, n, c, min_mapQ = 0;
    int64_t *cnt;
    const bam_pileup1_t **plp;
    int usage = 0;

    sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
    static const struct option lopts[] = {
        SAM_OPT_GLOBAL_OPTIONS('-', 0, '-', '-', 0),
        { NULL, 0, NULL, 0 }
    };

    while ((c = getopt_long(argc, argv, "Q:", lopts, NULL)) >= 0) {
        switch (c) {
        case 'Q': min_mapQ = atoi(optarg); break;
        default:  if (parse_sam_global_opt(c, optarg, lopts, &ga) == 0) break;
                  /* else fall-through */
        case '?': usage = 1; break;
        }
        if (usage) break;
    }
    if (usage || optind + 2 > argc) {
        fprintf(stderr, "Usage: samtools bedcov [options] <in.bed> <in1.bam> [...]\n\n");
        fprintf(stderr, "Options:\n");
        fprintf(stderr, "   -Q <int>            mapping quality threshold [0]\n");
        sam_global_opt_help(stderr, "-.--.");
        return 1;
    }
    memset(&str, 0, sizeof(kstring_t));
    n = argc - optind - 1;
    aux = calloc(n, sizeof(aux_t*));
    idx = calloc(n, sizeof(hts_idx_t*));
    for (i = 0; i < n; ++i) {
        aux[i] = calloc(1, sizeof(aux_t));
        aux[i]->min_mapQ = min_mapQ;
        aux[i]->fp = sam_open_format(argv[i+optind+1], "r", &ga.in);
        if (aux[i]->fp)
            idx[i] = sam_index_load(aux[i]->fp, argv[i+optind+1]);
        if (aux[i]->fp == 0 || idx[i] == 0) {
            fprintf(stderr, "ERROR: fail to open index BAM file '%s'\n", argv[i+optind+1]);
            return 2;
        }
        // TODO bgzf_set_cache_size(aux[i]->fp, 20);
        aux[i]->header = sam_hdr_read(aux[i]->fp);
        if (aux[i]->header == NULL) {
            fprintf(stderr, "ERROR: failed to read header for '%s'\n",
                    argv[i+optind+1]);
            return 2;
        }
    }
    cnt = calloc(n, 8);

    fp = gzopen(argv[optind], "rb");
    ks = ks_init(fp);
    n_plp = calloc(n, sizeof(int));
    plp = calloc(n, sizeof(bam_pileup1_t*));
    while (ks_getuntil(ks, KS_SEP_LINE, &str, &dret) >= 0) {
        char *p, *q;
        int tid, beg, end, pos;
        bam_mplp_t mplp;

        for (p = q = str.s; *p && *p != '\t'; ++p);
        if (*p != '\t') goto bed_error;
        *p = 0; tid = bam_name2id(aux[0]->header, q); *p = '\t';
        if (tid < 0) goto bed_error;
        for (q = p = p + 1; isdigit(*p); ++p);
        if (*p != '\t') goto bed_error;
        *p = 0; beg = atoi(q); *p = '\t';
        for (q = p = p + 1; isdigit(*p); ++p);
        if (*p == '\t' || *p == 0) {
            int c = *p;
            *p = 0; end = atoi(q); *p = c;
        } else goto bed_error;

        for (i = 0; i < n; ++i) {
            if (aux[i]->iter) hts_itr_destroy(aux[i]->iter);
            aux[i]->iter = sam_itr_queryi(idx[i], tid, beg, end);
        }
        mplp = bam_mplp_init(n, read_bam, (void**)aux);
        bam_mplp_set_maxcnt(mplp, 64000);
        memset(cnt, 0, 8 * n);
        while (bam_mplp_auto(mplp, &tid, &pos, n_plp, plp) > 0)
            if (pos >= beg && pos < end)
                for (i = 0; i < n; ++i) cnt[i] += n_plp[i];
        for (i = 0; i < n; ++i) {
            kputc('\t', &str);
            kputl(cnt[i], &str);
        }
        puts(str.s);
        bam_mplp_destroy(mplp);
        continue;

bed_error:
        fprintf(stderr, "Errors in BED line '%s'\n", str.s);
    }
    free(n_plp); free(plp);
    ks_destroy(ks);
    gzclose(fp);

    free(cnt);
    for (i = 0; i < n; ++i) {
        if (aux[i]->iter) hts_itr_destroy(aux[i]->iter);
        hts_idx_destroy(idx[i]);
        bam_hdr_destroy(aux[i]->header);
        sam_close(aux[i]->fp);
        free(aux[i]);
    }
    free(aux); free(idx);
    free(str.s);
    sam_global_args_free(&ga);
    return 0;
}
Example #10
0
int main(int argc, char **argv)
{
  if(argc < 3) print_usage(usage, NULL);

  // Sample reads from ref
  char *refpath = NULL;
  // int optt = 0, tlen = 800; double tlen_stddev = 0.1;
  int insert = 250, rlen = 250, single_ended = 0;
  double depth = 1.0, insert_stddev_prop = 0.2; // stddev as proportion of insert
  int optr = 0, opti = 0, optv = 0, optl = 0, optd = 0; // keeps track of values
  uint64_t seed = generate_seed(); // default RNG seed

  char *in0path = NULL, *in1path = NULL;

  char *profile_paths[argc];
  size_t num_profile_paths = 0, i, total_seq = 0;
  float err_rate = -1;

  int c;
  while((c = getopt(argc, argv, "p:r:i:v:l:d:s1:2:e:g:")) >= 0) {
    switch (c) {
      case 'p': profile_paths[num_profile_paths++] = optarg; break;
      case 'r': refpath = optarg; optr++; break;
      // case 't': tlen = atoi(optarg); optt++; break;
      // case 'v': tlen_stddev = atof(optarg); optv++; break;
      case 'i': insert = atoi(optarg); opti++; break;
      case 'v': insert_stddev_prop = atof(optarg); optv++; break;
      case 'l': rlen = atoi(optarg); optl++; break;
      case 'd': depth = atof(optarg); optd++; break;
      case 's': single_ended = 1; break;
      case '1': in0path = optarg; break;
      case '2': in1path = optarg; break;
      case 'e': err_rate = atof(optarg); break;
      case 'g': seed = atoi(optarg); break;
      default: die("Unknown option: %c", c);
    }
  }

  // Set up
  seed_random(seed);
  init_qual_prob();

  char *outbase = NULL;

  if(optind == argc) {}//print_usage(usage, "Missing <out_base>");
  else if(optind + 1 == argc) outbase = argv[optind];
  else if(optind + 1 < argc) print_usage(usage, "Too many args after %s", outbase);

  if(depth <= 0) print_usage(usage, "Depth [-d] cannot be <= 0");

  if(insert_stddev_prop < 0)
    print_usage(usage, "Insert length standard deviation [-v] cannot be < 0");

  if((opti > 0 || optv > 0 || optl > 0 || optd > 0) && refpath == NULL)
    print_usage(usage, "Missing -r <in.fa>");

  if(optr > 1 || opti > 1 || optv > 1 || optl > 1 || optd > 1)
    print_usage(usage, "Duplicate args");

  if(in0path == NULL && in1path != NULL)
    print_usage(usage, "-2 <in> requires -1 <in>");

  if(in0path != NULL && in1path == NULL) {
    if(refpath == NULL) single_ended = 1;
    else if(!single_ended) print_usage(usage, "Missing -2 for paired-end output");
  }

  if(in0path != NULL && num_profile_paths == 0)
    print_usage(usage, "Need at least one -p <profile.fq.gz> to use -1 .. -2 ..");

  if(num_profile_paths == 0 && refpath == NULL)
    print_usage(usage, "Need one of -p or -r");

  if(num_profile_paths == 0 && outbase == NULL)
    print_usage(usage, "More options required");

  if(num_profile_paths > 0 && err_rate >= 0)
    print_usage(usage, "Cannot use both -p and -E");

  // Profile reads
  FileList fliststore, *flist = NULL;
  if(num_profile_paths > 0) {
    flist = &fliststore;
    filelist_alloc(flist, profile_paths, num_profile_paths);
  }

  if(outbase == NULL)
  {
    // Summarise error profile in input
    filelist_mean_err(flist);
  }
  else
  {
    size_t outlen = strlen(outbase), extlen = strlen(".1.fa.gz");
    char out0path[outlen+extlen+1], out1path[outlen+extlen+1];
    memcpy(out0path, outbase, outlen);
    memcpy(out1path, outbase, outlen);

    if(single_ended) strcpy(out0path+outlen, ".fa.gz");
    else {
      strcpy(out0path+outlen, ".1.fa.gz");
      strcpy(out1path+outlen, ".2.fa.gz");
    }

    gzFile gzout0 = NULL, gzout1 = NULL;
    seq_file_t *sf0 = NULL, *sf1 = NULL, *reffile = NULL;

    if(in0path != NULL && (sf0 = seq_open(in0path)) == NULL) die("Cannot read: %s", in0path);
    if(in1path != NULL && (sf1 = seq_open(in1path)) == NULL) die("Cannot read: %s", in1path);

    if(refpath != NULL)
    {
      if((reffile = seq_open(refpath)) == NULL) die("Cannot read: %s", refpath);
      if((gzout0 = gzopen(out0path, "w")) == NULL) die("Cannot open: %s", out0path);
      if(!single_ended && (gzout1 = gzopen(out1path, "w")) == NULL)
        die("Cannot open: %s", out1path);
    }

    if(sf0 != NULL) {
      printf("Adding error to input reads...\n");
      total_seq += mutate_reads(sf0, gzout0, flist, err_rate);
      seq_close(sf0);
    }
    if(sf1 != NULL) {
      total_seq += mutate_reads(sf1, single_ended ? gzout0 : gzout1, flist, err_rate);
      seq_close(sf1);
    }

    if(refpath != NULL)
    {
      printf("Sampling from %s\n", refpath);
      printf(" sequencing depth: %.2f\n", depth);
      printf(" read length: %i\n", rlen);
      printf(" read pairs: %s\n", single_ended ? "no" : "yes");
      if(!single_ended) {
        printf(" insert length: %i\n", insert);
        printf(" insert stddev: %.2f * insert = %.2f\n",
               insert_stddev_prop, insert_stddev_prop*insert);
      }
      if(num_profile_paths > 0) {
        printf(" seq error files: %s", flist->files[0]->path);
        for(i = 1; i < num_profile_paths; i++)
          printf(",%s", flist->files[i]->path);
        printf("\n");
      } else if(err_rate >= 0) {
        printf(" seq error rate: %.2f%%\n", err_rate * 100.0);
      } else {
        printf(" sequencing errors: no\n");
      }
      total_seq += sim_reads(reffile, gzout0, gzout1, flist, err_rate,
                             insert, insert_stddev_prop*insert, rlen, depth);
      seq_close(reffile);
    }

    if(gzout0 != NULL && gzout1 != NULL)
      printf("Wrote %zu bases to: %s and %s\n", total_seq, out0path, out1path);
    else if(gzout0 != NULL)
      printf("Wrote %zu bases to: %s\n", total_seq, out0path);

    if(gzout0 != NULL) gzclose(gzout0);
    if(gzout1 != NULL) gzclose(gzout1);
  }

  if(flist != NULL)
  {
    // Print error distribution
    size_t err_total = 0;
    for(i = 0; i < flist->errors_len; i++) err_total += flist->errors[i];
    printf("Errors: %zu / %zu (%.2f%%)\n", err_total, total_seq,
                                           (100.0*err_total) / total_seq);
    for(i = 0; i < flist->errors_len; i++) printf(" %zu", flist->errors[i]);
    printf("\n");

    filelist_dealloc(flist);
  }

  return EXIT_SUCCESS;
}
Example #11
0
int main (int argc, char * const argv[]) {
	clock_t start, end;
	float cpu_time;
	gzFile read_fp, ref_fp;
	kseq_t *read_seq, *ref_seq;
	int32_t l, m, k, match = 2, mismatch = 2, gap_open = 3, gap_extension = 1, path = 0, reverse = 0, n = 5, sam = 0, protein = 0, header = 0, s1 = 67108864, s2 = 128, filter = 0;
	int8_t* mata = (int8_t*)calloc(25, sizeof(int8_t));
	const int8_t* mat = mata;
	char mat_name[16];
	mat_name[0] = '\0';
	int8_t* ref_num = (int8_t*)malloc(s1);
	int8_t* num = (int8_t*)malloc(s2), *num_rc = 0;
	char* read_rc = 0;

	static const int8_t mat50[] = {
	//  A   R   N   D   C   Q   E   G   H   I   L   K   M   F   P   S   T   W   Y   V   B   Z   X   *
     	5, -2, -1, -2, -1, -1, -1,  0, -2, -1, -2, -1, -1, -3, -1,  1,  0, -3, -2,  0, -2, -1, -1, -5,	// A
       -2,  7, -1, -2, -4,  1,  0, -3,  0, -4, -3,  3, -2, -3, -3, -1, -1, -3, -1, -3, -1,  0, -1, -5,	// R
       -1, -1,  7,  2, -2,  0,  0,  0,  1, -3, -4,  0, -2, -4, -2,  1,  0, -4, -2, -3,  5,  0, -1, -5,	// N
       -2, -2,  2,  8, -4,  0,  2, -1, -1, -4, -4, -1, -4, -5, -1,  0, -1, -5, -3, -4,  6,  1, -1, -5,	// D
       -1, -4, -2, -4, 13, -3, -3, -3, -3, -2, -2, -3, -2, -2, -4, -1, -1, -5, -3, -1, -3, -3, -1, -5,	// C
       -1,  1,  0,  0, -3,  7,  2, -2,  1, -3, -2,  2,  0, -4, -1,  0, -1, -1, -1, -3,  0,  4, -1, -5,	// Q
       -1,  0,  0,  2, -3,  2,  6, -3,  0, -4, -3,  1, -2, -3, -1, -1, -1, -3, -2, -3,  1,  5, -1, -5,	// E
     	0, -3,  0, -1, -3, -2, -3,  8, -2, -4, -4, -2, -3, -4, -2,  0, -2, -3, -3, -4, -1, -2, -1, -5,	// G
       -2,  0,  1, -1, -3,  1,  0, -2, 10, -4, -3,  0, -1, -1, -2, -1, -2, -3,  2, -4,  0,  0, -1, -5,	// H
       -1, -4, -3, -4, -2, -3, -4, -4, -4,  5,  2, -3,  2,  0, -3, -3, -1, -3, -1,  4, -4, -3, -1, -5,	// I
       -2, -3, -4, -4, -2, -2, -3, -4, -3,  2,  5, -3,  3,  1, -4, -3, -1, -2, -1,  1, -4, -3, -1, -5,	// L
       -1,  3,  0, -1, -3,  2,  1, -2,  0, -3, -3,  6, -2, -4, -1,  0, -1, -3, -2, -3,  0,  1, -1, -5,	// K
       -1, -2, -2, -4, -2,  0, -2, -3, -1,  2,  3, -2,  7,  0, -3, -2, -1, -1,  0,  1, -3, -1, -1, -5,	// M
       -3, -3, -4, -5, -2, -4, -3, -4, -1,  0,  1, -4,  0,  8, -4, -3, -2,  1,  4, -1, -4, -4, -1, -5,	// F
       -1, -3, -2, -1, -4, -1, -1, -2, -2, -3, -4, -1, -3, -4, 10, -1, -1, -4, -3, -3, -2, -1, -1, -5,	// P
     	1, -1,  1,  0, -1,  0, -1,  0, -1, -3, -3,  0, -2, -3, -1,  5,  2, -4, -2, -2,  0,  0, -1, -5,	// S
    	0, -1,  0, -1, -1, -1, -1, -2, -2, -1, -1, -1, -1, -2, -1,  2,  5, -3, -2,  0,  0, -1, -1, -5, 	// T
       -3, -3, -4, -5, -5, -1, -3, -3, -3, -3, -2, -3, -1,  1, -4, -4, -3, 15,  2, -3, -5, -2, -1, -5, 	// W
       -2, -1, -2, -3, -3, -1, -2, -3,  2, -1, -1, -2,  0,  4, -3, -2, -2,  2,  8, -1, -3, -2, -1, -5, 	// Y
     	0, -3, -3, -4, -1, -3, -3, -4, -4,  4,  1, -3,  1, -1, -3, -2,  0, -3, -1,  5, -3, -3, -1, -5, 	// V
       -2, -1,  5,  6, -3,  0,  1, -1,  0, -4, -4,  0, -3, -4, -2,  0,  0, -5, -3, -3,  6,  1, -1, -5, 	// B
       -1,  0,  0,  1, -3,  4,  5, -2,  0, -3, -3,  1, -1, -4, -1,  0, -1, -2, -2, -3,  1,  5, -1, -5, 	// Z
       -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -5, 	// X
       -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5,  1 	// *
	};

	/* This table is used to transform amino acid letters into numbers. */
	int8_t aa_table[128] = {
		23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
		23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
		23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
		23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
		23, 0,  20, 4,  3,  6,  13, 7,  8,  9,  23, 11, 10, 12, 2,  23,
		14, 5,  1,  15, 16, 23, 19, 17, 22, 18, 21, 23, 23, 23, 23, 23,
		23, 0,  20, 4,  3,  6,  13, 7,  8,  9,  23, 11, 10, 12, 2,  23,
		14, 5,  1,  15, 16, 23, 19, 17, 22, 18, 21, 23, 23, 23, 23, 23
	};

	/* This table is used to transform nucleotide letters into numbers. */
	int8_t nt_table[128] = {
		4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
		4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
		4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
		4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
		4, 0, 4, 1,  4, 4, 4, 2,  4, 4, 4, 4,  4, 4, 4, 4,
		4, 4, 4, 4,  3, 3, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
		4, 0, 4, 1,  4, 4, 4, 2,  4, 4, 4, 4,  4, 4, 4, 4,
		4, 4, 4, 4,  3, 3, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4
	};

	int8_t* table = nt_table;

	// Parse command line.
	while ((l = getopt(argc, argv, "m:x:o:e:a:f:pcrsh")) >= 0) {
		switch (l) {
			case 'm': match = atoi(optarg); break;
			case 'x': mismatch = atoi(optarg); break;
			case 'o': gap_open = atoi(optarg); break;
			case 'e': gap_extension = atoi(optarg); break;
			case 'a': strcpy(mat_name, optarg); break;
			case 'f': filter = atoi(optarg); break;
			case 'p': protein = 1; break;
			case 'c': path = 1; break;
			case 'r': reverse = 1; break;
			case 's': sam = 1; break;
			case 'h': header = 1; break;
		}
	}
	if (optind + 2 > argc) {
		fprintf(stderr, "\n");
		fprintf(stderr, "Usage: ssw_test [options] ... <target.fasta> <query.fasta>(or <query.fastq>)\n");
		fprintf(stderr, "Options:\n");
		fprintf(stderr, "\t-m N\tN is a positive integer for weight match in genome sequence alignment. [default: 2]\n");
		fprintf(stderr, "\t-x N\tN is a positive integer. -N will be used as weight mismatch in genome sequence alignment. [default: 2]\n");
		fprintf(stderr, "\t-o N\tN is a positive integer. -N will be used as the weight for the gap opening. [default: 3]\n");
		fprintf(stderr, "\t-e N\tN is a positive integer. -N will be used as the weight for the gap extension. [default: 1]\n");
		fprintf(stderr, "\t-p\tDo protein sequence alignment. Without this option, the ssw_test will do genome sequence alignment.\n");
		fprintf(stderr, "\t-a FILE\tFILE is either the Blosum or Pam weight matrix. [default: Blosum50]\n");
		fprintf(stderr, "\t-c\tReturn the alignment path.\n");
		fprintf(stderr, "\t-f N\tN is a positive integer. Only output the alignments with the Smith-Waterman score >= N.\n");
		fprintf(stderr, "\t-r\tThe best alignment will be picked between the original read alignment and the reverse complement read alignment.\n");
		fprintf(stderr, "\t-s\tOutput in SAM format. [default: no header]\n");
		fprintf(stderr, "\t-h\tIf -s is used, include header in SAM output.\n\n");
		return 1;
	}

	// initialize scoring matrix for genome sequences
	for (l = k = 0; LIKELY(l < 4); ++l) {
		for (m = 0; LIKELY(m < 4); ++m) mata[k++] = l == m ? match : -mismatch;	/* weight_match : -weight_mismatch */
		mata[k++] = 0; // ambiguous base
	}
	for (m = 0; LIKELY(m < 5); ++m) mata[k++] = 0;

	if (protein == 1 && (! strcmp(mat_name, "\0"))) {
		n = 24;
		table = aa_table;
		mat = mat50;
	} else if (strcmp(mat_name, "\0")) {

	// Parse score matrix.
		FILE *f_mat = fopen(mat_name, "r");
		char line[128];
		mata = (int8_t*)realloc(mata, 1024 * sizeof(int8_t));
		k = 0;
		m = 0;
		while (fgets(line, 128, f_mat)) {
			if (line[0] == '*' || (line[0] >= 'A' && line[0] <= 'Z')) {
				if (line[0] >= 'A' && line[0] <= 'Z') aa_table[(int)line[0]] = aa_table[(int)line[0] + 32] = m;
				char str[4], *s = str;
				str[0] = '\0';
				l = 1;
				while (line[l]) {
					if ((line[l] >= '0' && line[l] <= '9') || line[l] == '-') *s++ = line[l];
					else if (str[0] != '\0') {
						*s = '\0';
						mata[k++] = (int8_t)atoi(str);
						s = str;
						str[0] = '\0';
					}
					++l;
				}
				if (str[0] != '\0') {
					*s = '\0';
					mata[k++] = (int8_t)atoi(str);
					s = str;
					str[0] = '\0';
				}
				++m;
			}
		}
		if (k == 0) {
			fprintf(stderr, "Problem of reading the weight matrix file.\n");
			return 1;
		}
		fclose(f_mat);
		n = m;
		table = aa_table;
		mat = mata;
	}

	read_fp = gzopen(argv[optind + 1], "r");

    if (! read_fp) {
        fprintf (stderr, "gzopen of '%s' failed.\n", argv[optind + 1]);
            exit (EXIT_FAILURE);
    }

	read_seq = kseq_init(read_fp);
	if (sam && header && path) {
		fprintf(stdout, "@HD\tVN:1.4\tSO:queryname\n");
		ref_fp = gzopen(argv[optind], "r");
		ref_seq = kseq_init(ref_fp);
		while ((l = kseq_read(ref_seq)) >= 0) fprintf(stdout, "@SQ\tSN:%s\tLN:%d\n", ref_seq->name.s, (int32_t)ref_seq->seq.l);
		kseq_destroy(ref_seq);
		gzclose(ref_fp);
	} else if (sam && !path) {
		fprintf(stderr, "SAM format output is only available together with option -c.\n");
		sam = 0;
	}

	// alignment
	if (reverse == 1 && n == 5) {
		read_rc = (char*)malloc(s2);
		num_rc = (int8_t*)malloc(s2);
	}
	start = clock();
	while (kseq_read(read_seq) >= 0) {
		s_profile* p, *p_rc = 0;
		int32_t readLen = read_seq->seq.l;
		int32_t maskLen = readLen / 2;

		while (readLen >= s2) {
			++s2;
			kroundup32(s2);
			num = (int8_t*)realloc(num, s2);
			if (reverse == 1 && n == 5) {
				read_rc = (char*)realloc(read_rc, s2);
				num_rc = (int8_t*)realloc(num_rc, s2);
			}
		}
		for (m = 0; m < readLen; ++m) num[m] = table[(int)read_seq->seq.s[m]];
		p = ssw_init(num, readLen, mat, n, 2);
		if (reverse == 1 && n == 5) {
			reverse_comple(read_seq->seq.s, read_rc);
			for (m = 0; m < readLen; ++m) num_rc[m] = table[(int)read_rc[m]];
			p_rc = ssw_init(num_rc, readLen, mat, n, 2);
		}else if (reverse == 1 && n == 24) {
			fprintf (stderr, "Reverse complement alignment is not available for protein sequences. \n");
			return 1;
		}

		ref_fp = gzopen(argv[optind], "r");
		ref_seq = kseq_init(ref_fp);
		while (kseq_read(ref_seq) >= 0) {
			s_align* result, *result_rc = 0;
			int32_t refLen = ref_seq->seq.l;
			int8_t flag = 0;
			while (refLen > s1) {
				++s1;
				kroundup32(s1);
				ref_num = (int8_t*)realloc(ref_num, s1);
			}
			for (m = 0; m < refLen; ++m) ref_num[m] = table[(int)ref_seq->seq.s[m]];
			if (path == 1) flag = 2;
			result = ssw_align (p, ref_num, refLen, gap_open, gap_extension, flag, filter, 0, maskLen);
			if (reverse == 1 && protein == 0)
				result_rc = ssw_align(p_rc, ref_num, refLen, gap_open, gap_extension, flag, filter, 0, maskLen);
			if (result_rc && result_rc->score1 > result->score1 && result_rc->score1 >= filter) {
				if (sam) ssw_write (result_rc, ref_seq, read_seq, read_rc, ref_num, num_rc, table, 1, 1);
				else ssw_write (result_rc, ref_seq, read_seq, read_rc, ref_num, num_rc, table, 1, 0);
			}else if (result && result->score1 >= filter){
				if (sam) ssw_write(result, ref_seq, read_seq, read_seq->seq.s, ref_num, num, table, 0, 1);
				else ssw_write(result, ref_seq, read_seq, read_seq->seq.s, ref_num, num, table, 0, 0);
			} else if (! result) return 1;
			if (result_rc) align_destroy(result_rc);
			align_destroy(result);
		}

		if(p_rc) init_destroy(p_rc);
		init_destroy(p);
		kseq_destroy(ref_seq);
		gzclose(ref_fp);
	}
	end = clock();
	cpu_time = ((float) (end - start)) / CLOCKS_PER_SEC;
	fprintf(stderr, "CPU time: %f seconds\n", cpu_time);

	if (num_rc) {
		free(num_rc);
		free(read_rc);
	}
	kseq_destroy(read_seq);
	gzclose(read_fp);
	free(num);
	free(ref_num);
 	free(mata);
	return 0;
}
Example #12
0
char *query (char *query, char *bloom_filter, double tole_rate, double sampling_rate, char *list, char *target_path, char *report_fmt, char mode)
{
  gzFile zip = NULL;
  char type = '@';
  int normal = 0;
  int threads = 0;
  BIGCAST offset = 0;
  char *position = NULL;
  static char timestamp[40] = {0};

  // Get current timestamp, for benchmarking purposes (begin of run timestamp)
  isodate(timestamp);
  bloom *bl_2 = NEW (bloom);
  F_set *File_head = make_list (bloom_filter, list);
  /*initialization for python interface*/
  File_head->hits = 0;
  File_head->all_k = 0;
  File_head->reads_num = 0;
  File_head->reads_contam = 0;
  File_head->filename = bloom_filter;           //extra initialization for python interface
  if (load_bloom (File_head->filename, bl_2)<=0)	//load a bloom filter
	exit(-1);
  
  if (tole_rate == 0)
  {
  	tole_rate = mco_suggestion (bl_2->k_mer); // suggest an optimal match cut-off
  }
  if (mode == 'r')
  {
 	init_string(ONEG); // initialize strings for containing reads
  }
/*
  if ((get_size (query) < 2 * ONEG) && !strstr (query, ".gz") && !strstr (query, ".tar"))
        normal = 0;
  else
  {
      if ((zip = gzopen (query, "rb")) < 0)
	{
          perror ("query open error...\n");
          exit (-1);
	}
      normal = 0;
  }
*/
  if ((zip = gzopen (query, "rb")) <= 0)
  {
  	fprintf(stderr, "%s\n", strerror(errno));
  	exit(EXIT_FAILURE);
  }
  
  if (strstr (query, ".fastq") != NULL || strstr (query, ".fq") != NULL)
  	type = '@';
  else
  	type = '>';
  if (normal == 0)
  	position = (char *) calloc (1,(ONEG+1)*sizeof (char));
  while (offset != -1)
  {
      if (normal == 1)
      {
	  position = mmaping (query);
	  offset = -1;
      }
      else
      {
	  offset = CHUNKer (zip, offset, ONEG, position, type);
      }
      Queue *head = NEW (Queue);
      head->location = NULL;
      Queue *tail = NEW (Queue);
      head->next = tail;
      Queue *head2 = head;
      get_parainfo (position, head, type);
#pragma omp parallel
      {
// XXX: Awesome will be the day when OpenMP is in OSX
#ifndef __APPLE__ 
          threads = omp_get_num_threads();
#endif
#pragma omp single nowait
	{
	  while (head != tail)
	    {
#pragma omp task firstprivate(head)
	      {
		if (head->location != NULL)
		{
			read_process (bl_2, head, tail, File_head, sampling_rate, tole_rate, mode, type);
		}
	      }
	      head = head->next;
	    }			// End of firstprivate
	}			// End of single - no implied barrier (nowait)
      }				// End of parallel region - implied barrier
  if (position != NULL && normal == 0)
  {
  	memset (position, 0, strlen (position));
  } 
  else if (normal == 1)
  {
	munmap (position, strlen (position));
  } 
  else
  {
 	perror ("Cannot memset, wrong position on fastq file\n");
	exit (-1);
  }
  clean_list (head2, tail);
  if (mode == 'r')
  {
	if (target_path!=NULL)
	{
      		save_result (query, File_head->filename, type, target_path, re_clean(), re_contam()); //save results into file if facs remove is called
  	}
	else
	{
		write_default(re_clean(), re_contam(), offset);
	}
	reset_string();
  }
  }				//end while
  if (normal == 0)
  {
 	bloom_destroy(bl_2);
  	gzclose(zip);
  	free (position);        //dont like file mapping, strings need to be freed in a normal way
  }

  /*
  mode c and r refer to contamination checking and removal function respectively. 
  The following 9 lines make sure that json/tsv output is printed after the checking 
  process, but willnot be written in stdout when running removal process.
  */
  if (target_path!=NULL || mode == 'c')
  {
  	return report(File_head, query, report_fmt, target_path, timestamp, prob_suggestion(bl_2->k_mer), threads);
  }
  else
  {
	char *s = "";
	return s;
  }
}
Example #13
0
int CFile::PImpl::open(const char *name, long openflags)
{
	char buf[512];
	const char *openstring;

	if ((openflags & CL_OPEN_READ) && (openflags & CL_OPEN_WRITE)) {
		openstring = "rwb";
	} else if (openflags & CL_OPEN_READ) {
		openstring = "rb";
	} else if (openflags & CL_OPEN_WRITE) {
		openstring = "wb";
	} else {
		DebugPrint("Bad CLopen flags");
		Assert(0);
		return -1;
	}

	cl_type = CLF_TYPE_INVALID;

	if (openflags & CL_OPEN_WRITE) {
#ifdef USE_BZ2LIB
		if ((openflags & CL_WRITE_BZ2)
			&& (cl_bz = BZ2_bzopen(strcat(strcpy(buf, name), ".bz2"), openstring))) {
			cl_type = CLF_TYPE_BZIP2;
		} else
#endif
#ifdef USE_ZLIB
			if ((openflags & CL_WRITE_GZ)
				&& (cl_gz = gzopen(strcat(strcpy(buf, name), ".gz"), openstring))) {
				cl_type = CLF_TYPE_GZIP;
			} else
#endif
				if ((cl_plain = fopen(name, openstring))) {
					cl_type = CLF_TYPE_PLAIN;
				}
	} else {
		if (!(cl_plain = fopen(name, openstring))) { // try plain first
#ifdef USE_ZLIB
			if ((cl_gz = gzopen(strcat(strcpy(buf, name), ".gz"), "rb"))) {
				cl_type = CLF_TYPE_GZIP;
			} else
#endif
#ifdef USE_BZ2LIB
				if ((cl_bz = BZ2_bzopen(strcat(strcpy(buf, name), ".bz2"), "rb"))) {
					cl_type = CLF_TYPE_BZIP2;
				} else
#endif
				{ }

		} else {
			cl_type = CLF_TYPE_PLAIN;
			// Hmm, plain worked, but nevertheless the file may be compressed!
			if (fread(buf, 2, 1, cl_plain) == 1) {
#ifdef USE_BZ2LIB
				if (buf[0] == 'B' && buf[1] == 'Z') {
					fclose(cl_plain);
					if ((cl_bz = BZ2_bzopen(name, "rb"))) {
						cl_type = CLF_TYPE_BZIP2;
					} else {
						if (!(cl_plain = fopen(name, "rb"))) {
							cl_type = CLF_TYPE_INVALID;
						}
					}
				}
#endif // USE_BZ2LIB
#ifdef USE_ZLIB
				if (buf[0] == 0x1f) { // don't check for buf[1] == 0x8b, so that old compress also works!
					fclose(cl_plain);
					if ((cl_gz = gzopen(name, "rb"))) {
						cl_type = CLF_TYPE_GZIP;
					} else {
						if (!(cl_plain = fopen(name, "rb"))) {
							cl_type = CLF_TYPE_INVALID;
						}
					}
				}
#endif // USE_ZLIB
			}
			if (cl_type == CLF_TYPE_PLAIN) { // ok, it is not compressed
				rewind(cl_plain);
			}
		}
	}

	if (cl_type == CLF_TYPE_INVALID) {
		//fprintf(stderr, "%s in ", buf);
		return -1;
	}
	return 0;
}
Example #14
0
static ClassRegister *newClassRegister(char *fname)
{
   ClassRegister *cr =
       (ClassRegister *) calloc(1,sizeof(ClassRegister) + sizeof(ClassBase));
   ClassBase *cb = (ClassBase *) (cr + 1);
   char fin[1024];
   long s, total=0;
   ClObjectHdr hdr;
   ClVersionRecord *vrp=(ClVersionRecord*)&hdr;
   int vRec=0,first=1;
   
   z_off_t pos=0;
   ClassRecord *crec;
   
   cr->hdl = cb;
   cr->ft = ClassRegisterFT;
   cr->vr = NULL;
   cr->assocs = cr->topAssocs = 0;
   
   strcpy(fin, fname);
   strcat(fin, "/classSchemas");
   cr->f = gzopen(fin, "r");
   if (cr->f==NULL) {
      strcat(fin, ".gz");
      cr->f = gzopen(fin, "r");
   }
   
   
   cb->ht = UtilFactory->newHashTable(61,
            UtilHashTable_charKey | UtilHashTable_ignoreKeyCase);
   cb->it = UtilFactory->newHashTable(61,
            UtilHashTable_charKey | UtilHashTable_ignoreKeyCase);
   MRWInit(&cb->mrwLock);
      
   if (cr->f == NULL)  return cr;

   cr->fn = strdup(fin);
   cr->vr=NULL;
   pos=gztell(cr->f);

   while ((s = gzread(cr->f, &hdr, sizeof(hdr))) == sizeof(hdr)) {
      CMPIConstClass *cc=NULL;
      char *buf=NULL;
      const char *cn,*pn;
      
      if (first) {
         if (vrp->size==sizeof(ClVersionRecord) && vrp->type==HDR_Version) vRec=1;
         else if (vrp->size==sizeof(ClVersionRecord)<<24 && vrp->type==HDR_Version) {
            mlogf(M_ERROR,M_SHOW,"--- %s is in wrong endian format - directory skipped\n",fin);
            return NULL;
         }
      }
      
      if (vRec==0 && hdr.type!=HDR_Class) {
         mlogf(M_ERROR,M_SHOW,"--- %s contains non-class record(s) - directory skipped\n",fin);
         return NULL;
     }
      
      buf = (char *) malloc(hdr.size);
      if (buf==NULL) {
         mlogf(M_ERROR,M_SHOW,"--- %s contains record(s) that are too long - directory skipped\n",fin);
         return NULL;
      }
      
      s=hdr.size;
      *((ClObjectHdr *) buf) = hdr;
      
      if (gzread(cr->f, buf + sizeof(hdr), hdr.size - sizeof(hdr)) == hdr.size - sizeof(hdr)) {
         if (vRec) {
            cr->vr=(ClVersionRecord*)buf;
            if (strcmp(cr->vr->id,"sfcb-rep")) {
               mlogf(M_ERROR,M_SHOW,"--- %s contains invalid version record - directory skipped\n",fin);
               return NULL;
            }   
            pos=gztell(cr->f);
            vRec=0;
         }
         
         if (first) {
            int v=-1;
            first=0;
            if (ClVerifyObjImplLevel(cr->vr)) continue;
            if (cr->vr) v=cr->vr->objImplLevel;
            mlogf(M_ERROR,M_SHOW,"--- %s contains unsupported object implementation format (%d) - directory skipped\n",
               fin,v);
            return NULL;
         }
         
         cc = NEW(CMPIConstClass);
         cc->hdl = buf;
         cc->ft = CMPIConstClassFT;
         cc->ft->relocate(cc);
         cn=(char*)cc->ft->getCharClassName(cc);
         
         if (strncmp(cn,"DMY_",4)!=0) {
            total+=sizeof(ClassRecord);   
            crec=(ClassRecord*)calloc(1,sizeof(ClassRecord));  
            
            if ((pn=cc->ft->getCharSuperClassName(cc))) {
               crec->parent=strdup(pn);
            }
            crec->position=pos;
            crec->length=s;
            cr->ft->putClass(cr, strdup(cn), crec);
            
            if (cc->ft->isAssociation(cc)) {
               crec->flags|=CREC_isAssociation;
               cr->assocs++;
               if (pn == NULL) cr->topAssocs++;
            }   
         } 
         first=0;
      }
      else {
         mlogf(M_ERROR,M_SHOW,"--- %s contains invalid record(s) - directory skipped\n",fin);
         return NULL;
      }
      pos=gztell(cr->f);
      free(buf);
      free(cc);
   }

   if (cr->vr) {
      mlogf(M_INFO,M_SHOW,"--- Caching ClassProvider for %s (%d.%d-%d) using %ld bytes\n", 
          fin, cr->vr->version, cr->vr->level, cr->vr->objImplLevel, total);
   }
   else mlogf(M_INFO,M_SHOW,"--- Caching ClassProvider for %s (no-version) using %ld bytes\n", fin, total);

   buildInheritanceTable(cr);
   
   return cr;
}
Example #15
0
/*
 * concatenate two (or more) files into one single file 
 */
void concatenate(int argc, char **argv)
{
   int zerr;
   gzFile fd;
   struct log_global_header hdr, tmp;

   memset(&hdr, 0, sizeof(struct log_global_header));

   /* open the output file for writing */
   fd = gzopen(GBL_LOGFILE, "wb");
   ON_ERROR(fd, NULL, "%s", gzerror(fd, &zerr));

   /* 
    * use GBL_LOG_FD here so the get_header function
    * will use this file 
    */
   GBL_LOG_FD = gzopen(argv[argc], "rb");
   ON_ERROR(GBL_LOG_FD, NULL, "%s", gzerror(GBL_LOG_FD, &zerr));
   
   /* get the file header */
   if (get_header(&hdr) != E_SUCCESS)
      FATAL_ERROR("Invalid log file (%s)", argv[argc]);

   /* write the header */
   put_header(fd, &hdr);
      
   printf("Concatenating file [%s]", argv[argc]);
   
   /* copy the first file into the output */
   dump_file(fd, &hdr);
     
   /* move the pointer to the next file */
   argc++;
   
   /* cicle thru the file list */
   while(argv[argc] != NULL) {
   
      GBL_LOG_FD = gzopen(argv[argc], "rb");
      ON_ERROR(GBL_LOG_FD, NULL, "%s", gzerror(GBL_LOG_FD, &zerr));
   
      /* get the file header */
      if (get_header(&tmp) != E_SUCCESS)
         FATAL_ERROR("Invalid log file (%s)", argv[argc]);
     
      /* check if the files are compatible */
      if (hdr.type != tmp.type)
         FATAL_ERROR("Cannot concatenate different type of file");

      printf("Concatenating file [%s]", argv[argc]);
      
      /* concatenate this file */
      dump_file(fd, &tmp);

      gzclose(GBL_LOG_FD);
      argc++;
   }

   gzclose(fd);

   printf("\nAll files concatenated into: %s\n\n", GBL_LOGFILE);

   exit(0);
}
Example #16
0
int
main(int argc, char **argv)
{
	gzFile	       *infp;
	char		buf[BUFSIZ], *bp, **aliases;
	struct hostent  *he;
	int		ch, errors = 0;
	int		backcount = 0;

	progname = argv[0];

	while ((ch = getopt(argc, argv, "a:")) != -1)
		switch(ch) {
		case 'a':
			backcount = atoi(optarg);
			break;
			
		case 'h':
		case '?':
		default:
			usage();
		}
	argc -= optind;
	argv += optind;

	if (argc)
		usage();

	openlog("genlastlog", LOG_PID, LOG_TESTBED);
	syslog(LOG_NOTICE, "genlastlog starting");

	if (!dbinit()) {
		syslog(LOG_ERR, "Could not connect to DB!");
		exit(1);
	}

	/*
	 * We need the canonical hostname for the usersnode so that we can
	 * put those logins in another table.
	 */
	if ((he = gethostbyname(USERNODE)) == NULL) {
		syslog(LOG_ERR, "gethostname %s: %s",
		       USERNODE, hstrerror(h_errno));
		exit(-1);
	}
	strncpy(opshostname, he->h_name, sizeof(opshostname));

	if (bp = strchr(opshostname, '.')) 
		*bp = 0;

	while (backcount) {
		sprintf(buf, "%s/%s.%d.gz", USERSVAR, LOGINS, backcount);

		/*
		 * Use setjmp and timer to prevent NFS lockup.
		 */
		if (setjmp(deadline) == 0) {
			alarm(15);

			if ((infp = gzopen(buf, "r")) == NULL) {
				syslog(LOG_ERR, "Opening %s: %m", buf);
				errors++;
			}
			else {
				doit(infp);
				gzclose(infp);
			}
		}
		backcount--;
		alarm(0);
	}
	
	sprintf(buf, "%s/%s", USERSVAR, LOGINS);

	if (setjmp(deadline) == 0) {
		alarm(15);

		if ((infp = gzopen(buf, "r")) == NULL) {
			syslog(LOG_ERR, "Opening %s: %m", buf);
			errors++;
		}
		else {
			doit(infp);
			gzclose(infp);
		}
	}
	alarm(0);

	syslog(LOG_NOTICE, "genlastlog ending");
	exit(errors);
	
}
Example #17
0
CFRFILE *cfr_open(const char *path) {
  /*******************************/
  // Analog to 'fopen'. Error in result has to be tested using
  // 'cfr_error' on the result!
  // Note: The user needs to free the reurn value!
  // Opens a possibly compressed file for reading.
  // File type is determined by file name ending

  int format, ext_len, name_len;
  CFRFILE * retval = NULL;

  // determine file format
  name_len = strlen(path);
  format = 2;  // skip specials 0, 1 

  // Do action dependent on file format
  retval = (CFRFILE *) calloc(1,sizeof(CFRFILE));
  retval->eof = 0;
  retval->error1 = 0;
  retval->error2 = 0;


#ifndef DONT_HAVE_GZ
  if((path == NULL) || (strcmp(path, "-") == 0)) {
	/* dump from stdin */
	gzFile *f;
	while (format < CFR_NUM_FORMATS) {
		if (strcmp(cfr_extensions[format], ".gz") == 0)
        		break;
    		format ++;
  	}
	f = gzdopen(0, "r");
	if(f == NULL) {
		free(retval);
		return (NULL);
    	}
        retval->data2 = f;
	retval->format = format;
	return (retval);
  }
#endif
  while (format < CFR_NUM_FORMATS) {
    ext_len = strlen(cfr_extensions[format]);
    if (strncmp(cfr_extensions[format],
                path+(name_len-ext_len),
                ext_len) == 0
        ) break;
    format ++;
  }
  if (format >= CFR_NUM_FORMATS) 
	format = 1;  // uncompressed 
  retval->format = format;

  switch (format) {
  case 1:  // uncompressed
    { 
      FILE * in;
      in = fopen(path,"r");
      if (in == NULL) { 
	free(retval);
        return(NULL);
      }
      retval->data1 = in;
      return(retval);
    }
    break;
#ifndef DONT_HAVE_BZ2
  case 2:  // bzip2
    { 
      int bzerror;
      BZFILE * bzin;
      FILE * in;
      
      retval->bz2_stream_end = 0;
      
      // get file
      in = fopen(path,"r");
      if (in == NULL) { 
        free(retval);
        return(NULL);
      }
      retval->data1 = in;
      
      // bzip2ify file
      bzin = BZ2_bzReadOpen( &bzerror, in, 0, 0, NULL, 0); 
      if (bzerror != BZ_OK) {
        errno = bzerror;
        BZ2_bzReadClose( &bzerror, bzin);
        fclose(in);
	free(retval);
        return(NULL);
      }
      retval->data2 = bzin;
      return(retval);
    }
    break;
#endif
#ifndef DONT_HAVE_GZ
  case 3:  // gzip
    { 
	gzFile *f;
      	// get file 
	f = gzopen(path, "r");
	if(f == NULL) {
		free(retval);
		return (NULL);
    	}
        retval->data2 = f;
	return (retval);
   }
   break;
#endif
  default:  // this is an internal error, no diag yet.
    fprintf(stderr,"illegal format '%d' in cfr_open!\n", format);
    exit(1);
  }
  return NULL;
} 
Example #18
0
void
LoadTerrain(int lat,              /* I - Latitude in degrees */
            int lon)              /* I - Longitude in degrees */
    {
    int           i, j;           /* Looping vars */
    TP            *tp;            /* Terrain posting */
    gzFile        fp;             /* DTED file */
    char          filename[255];  /* Name of DTED file */
    unsigned char buf[254],       /* Record buffer */
                  *bufptr;        /* Pointer to current elevation */
    GLfloat       escale,         /* East-west scaling */
                  nscale;         /* North-south scaling */
    GLfloat       nx, ny, nz, nw; /* Normal components */
    int           cell;           /* Current cell */
    int           strip;          /* 0 = no strip, 1 or -1 = strip */
    int           num_triangles;  /* Number of triangles all together */

    /* Set the size of the terrain... */
    TerrainCount = (abs(lat) >= 80) ? 21 : (abs(lat) >= 75) ? 31 :
                   (abs(lat) >= 70) ? 41 : (abs(lat) >= 50) ? 61 : 121;
    nscale       = TERRAIN_SIZE / (TERRAIN_COUNT - 1);
    escale       = TERRAIN_SIZE / (TerrainCount - 1);

    /* Try opening the DTED file... */
    sprintf(filename, "dted/%c%03d/%c%02d.dt0",
            lon < 0 ? 'w' : 'e', abs(lon), lat < 0 ? 's' : 'n', abs(lat));
    if ((fp = gzopen(filename, "rb")) != NULL)
        {
        printf("Reading %s...\n", filename);

        /*
         * Seek to the first data record and read all the columns.  This
	 * offset is valid for all Level 0 and most Level 1 and 2 DTED
	 * files.
         */

        gzseek(fp, 3436, SEEK_SET);

        WaterLevel = 100000.0;

        for (i = 0, tp = Terrain[0]; i < TERRAIN_COUNT; i ++)
            {
	    printf("\rColumn %d", i);
	    fflush(stdout);

            gzread(fp, buf, sizeof(buf)); /* Read record */

            for (j = TERRAIN_COUNT - 1, bufptr = buf;
	         j >= 0;
		 j --, bufptr += 2, tp ++)
                {
		/* Convert signed-magnitude */
	        if (*bufptr & 0x80)
                    tp->v[1] = -0.5 * (((bufptr[0] & 0x7f) << 8) | bufptr[1]);
	        else
	            tp->v[1] = 0.5 * ((bufptr[0] << 8) | bufptr[1]);

                if (tp->v[1] < WaterLevel)
		    WaterLevel = tp->v[1];
		}
            }

        gzclose(fp);

        WaterLevel += 200.0; /* Purely arbitrary... */

        puts("\rDone reading terrain.");
        }
    else
        memset(Terrain, 0, sizeof(Terrain));

    /* Set the coordinates of each posting */
    for (i = 0, tp = Terrain[0]; i < TerrainCount; i ++)
        for (j = 0; j < TERRAIN_COUNT; j ++, tp ++)
            {
	    tp->v[0] = (i - TerrainCount / 2) * escale;
	    tp->v[2] = (TERRAIN_COUNT / 2 - j) * nscale;
	    tp->used = 0;
            }

    /*
     * Loop through the terrain arrays and regenerate the
     * lighting normals based on the terrain height.
     */
    for (i = 0, tp = Terrain[0]; i < (TerrainCount - 1); i ++, tp ++)
        for (j = 0; j < (TERRAIN_COUNT - 1); j ++, tp ++)
            {
            /*
             * Compute the cross product of the vectors above and to
             * the right (simplified for this special case).
             */

            nx = tp[0].v[1] - tp[1].v[1];
            ny = -1.0;
            nz = tp[0].v[1] - tp[TERRAIN_COUNT].v[1];
            nw = -1.0 / sqrt(nx * nx + ny * ny + nz * nz);

            /* Normalize the normal vector and store it... */
            tp->n[0] = nx * nw;
            tp->n[1] = ny * nw;
            tp->n[2] = nz * nw;
            }

    /*
     * Set the top and right normals to be the same as the
     * second-to-last normals.
     */

    for (i = 0, tp = Terrain[TerrainCount - 2];
         i < TERRAIN_COUNT;
	 i ++, tp ++)
        {
	  tp[TERRAIN_COUNT].n[0] = tp[0].n[0];
	  tp[TERRAIN_COUNT].n[1] = tp[0].n[1];
	  tp[TERRAIN_COUNT].n[2] = tp[0].n[2];
	}

    for (i = 0, tp = Terrain[0] + TERRAIN_COUNT - 2;
         i < (TerrainCount - 1);
	 i ++, tp += TERRAIN_COUNT)
        {
	  tp[1].n[0] = tp[0].n[0];
	  tp[1].n[1] = tp[0].n[1];
	  tp[1].n[2] = tp[0].n[2];
	}

    /* Position the viewer just above the terrain */
    Position[1] = Terrain[TerrainCount / 2][TERRAIN_COUNT / 2].v[1] + 10.0;

    /* Now scan the terrain for "interesting" postings */
    for (i = TerrainCount, tp = Terrain[0] + 1; i > 0; i --, tp += 2)
        for (j = TERRAIN_COUNT - 2; j > 0; j --, tp ++)
	    if (fabs(2 * tp[0].v[1] - tp[1].v[1] - tp[-1].v[1]) > 100.0)
	        tp->used = 1;

    for (j = 0; j < TERRAIN_COUNT; j ++)
        for (i = TerrainCount - 2, tp = Terrain[1] + j;
	     i > 0;
	     i --, tp += TERRAIN_COUNT)
	    if (fabs(2 * tp[0].v[1] - tp[TERRAIN_COUNT].v[1] -
	             tp[-TERRAIN_COUNT].v[1]) > 100.0)
	        tp->used = 1;

    /* Now build a display list with the tessellation... */
    TerrainList = glGenLists(1);
    glNewList(TerrainList, GL_COMPILE);

    num_triangles = 0;

    for (i = TerrainCount / 2, tp = Terrain[0];
         i > 0;
         i --, tp += TERRAIN_COUNT + 1)
        {
        /* Start this pass with no strips going... */
	strip = 0;

	for (j = TERRAIN_COUNT / 2; j > 0; j --, tp += 2)
	    {
	    /* Figure out which points in the 3x3 cell are set... */
	    cell = 0;
	    
	    if (tp[1].used)
	        cell |= TC_LEFT;
	    if (tp[TERRAIN_COUNT].used)
	        cell |= TC_BOTTOM;
	    if (tp[TERRAIN_COUNT + 1].used)
	        cell |= TC_CENTER;
	    if (tp[TERRAIN_COUNT + 2].used)
	        cell |= TC_TOP;
	    if (tp[2 * TERRAIN_COUNT + 1].used)
	        cell |= TC_RIGHT;

            if ((cell & TC_VERT) == 0)
	        {
		/* OK, this cell can be represented using strips */
		if (strip == 0)
		    {
                    glBegin(GL_TRIANGLE_STRIP);
		    if (cell == TC_RIGHT)
		        {
		        glNormal3fv(tp[0].n);
                        glVertex3fv(tp[0].v);
		        glNormal3fv(tp[2 * TERRAIN_COUNT].n);
                        glVertex3fv(tp[2 * TERRAIN_COUNT].v);
		        glNormal3fv(tp[2 * TERRAIN_COUNT + 1].n);
                        glVertex3fv(tp[2 * TERRAIN_COUNT + 1].v);
			glEnd();

			glBegin(GL_TRIANGLE_STRIP);
		        glNormal3fv(tp[0].n);
                        glVertex3fv(tp[0].v);
		        glNormal3fv(tp[2 * TERRAIN_COUNT + 1].n);
                        glVertex3fv(tp[2 * TERRAIN_COUNT + 1].v);
			}
		    else
		        {
		        glNormal3fv(tp[0].n);
                        glVertex3fv(tp[0].v);
		        glNormal3fv(tp[2 * TERRAIN_COUNT].n);
                        glVertex3fv(tp[2 * TERRAIN_COUNT].v);

			}

		    strip = 1;
		    }

                if (cell == (TC_LEFT | TC_RIGHT))
		    {
		    if (strip > 0)
		        {
		        glNormal3fv(tp[1].n);
                        glVertex3fv(tp[1].v);
		        glNormal3fv(tp[2 * TERRAIN_COUNT + 1].n);
                        glVertex3fv(tp[2 * TERRAIN_COUNT + 1].v);
			}
		    else
		        {
		        glNormal3fv(tp[2 * TERRAIN_COUNT + 1].n);
                        glVertex3fv(tp[2 * TERRAIN_COUNT + 1].v);
		        glNormal3fv(tp[1].n);
                        glVertex3fv(tp[1].v);
			}

		    num_triangles += 2;
		    }
		else if (cell == TC_LEFT)
		    {
		    glNormal3fv(tp[1].n);
                    glVertex3fv(tp[1].v);

		    if (strip < 0)
		        {
			glEnd();

			strip = 1;

			glBegin(GL_TRIANGLE_STRIP);
		        glNormal3fv(tp[1].n);
                        glVertex3fv(tp[1].v);
		        glNormal3fv(tp[2 * TERRAIN_COUNT].n);
                        glVertex3fv(tp[2 * TERRAIN_COUNT].v);
			}
		    else
		        strip = -strip;

		    num_triangles ++;
		    }
		else if (cell == TC_RIGHT)
		    {
		    glNormal3fv(tp[2 * TERRAIN_COUNT + 1].n);
                    glVertex3fv(tp[2 * TERRAIN_COUNT + 1].v);

		    if (strip > 0)
		        {
			glEnd();

			glBegin(GL_TRIANGLE_STRIP);
		        glNormal3fv(tp[0].n);
                        glVertex3fv(tp[0].v);
		        glNormal3fv(tp[2 * TERRAIN_COUNT + 1].n);
                        glVertex3fv(tp[2 * TERRAIN_COUNT + 1].v);
			}
		    else
		        strip = -strip;

		    num_triangles ++;
		    }

                if (strip > 0)
		    {
		    glNormal3fv(tp[2].n);
                    glVertex3fv(tp[2].v);
		    glNormal3fv(tp[2 * TERRAIN_COUNT + 2].n);
                    glVertex3fv(tp[2 * TERRAIN_COUNT + 2].v);
		    }
		else
		    {
		    glNormal3fv(tp[2 * TERRAIN_COUNT + 2].n);
                    glVertex3fv(tp[2 * TERRAIN_COUNT + 2].v);
		    glNormal3fv(tp[2].n);
                    glVertex3fv(tp[2].v);
		    }

		num_triangles += 2;
		}
	    else
	        {
	        if (strip)
		    {
		    glEnd();
		    strip = 0;
		    }

                glBegin(GL_TRIANGLE_FAN);
		glNormal3fv(tp[TERRAIN_COUNT + 1].n);
                glVertex3fv(tp[TERRAIN_COUNT + 1].v);

		glNormal3fv(tp[0].n);
                glVertex3fv(tp[0].v);

                if (cell & TC_BOTTOM)
		    {
		    num_triangles ++;
		    glNormal3fv(tp[TERRAIN_COUNT].n);
                    glVertex3fv(tp[TERRAIN_COUNT].v);
		    }

		num_triangles ++;
		glNormal3fv(tp[2 * TERRAIN_COUNT].n);
                glVertex3fv(tp[2 * TERRAIN_COUNT].v);
                
		if (cell & TC_RIGHT)
		    {
		    num_triangles ++;
		    glNormal3fv(tp[2 * TERRAIN_COUNT + 1].n);
                    glVertex3fv(tp[2 * TERRAIN_COUNT + 1].v);
		    }

		num_triangles ++;
		glNormal3fv(tp[2 * TERRAIN_COUNT + 2].n);
                glVertex3fv(tp[2 * TERRAIN_COUNT + 2].v);

                if (cell & TC_TOP)
		    {
		    num_triangles ++;
		    glNormal3fv(tp[TERRAIN_COUNT + 2].n);
                    glVertex3fv(tp[TERRAIN_COUNT + 2].v);
		    }

		num_triangles ++;
		glNormal3fv(tp[2].n);
                glVertex3fv(tp[2].v);

		if (cell & TC_LEFT)
		    {
		    num_triangles ++;
		    glNormal3fv(tp[1].n);
                    glVertex3fv(tp[1].v);
		    }

		num_triangles ++;
		glNormal3fv(tp[0].n);
                glVertex3fv(tp[0].v);
		glEnd();
		}
	    }

        if (strip)
	    glEnd();
	}

    glEndList();

    printf("Number triangles = %d\n", num_triangles);
    }
Example #19
0
bool SymbolMap::LoadSymbolMap(const char *filename) {
	Clear();  // let's not recurse the lock

	std::lock_guard<std::recursive_mutex> guard(lock_);

#if defined(_WIN32) && defined(UNICODE)
	gzFile f = gzopen_w(ConvertUTF8ToWString(filename).c_str(), "r");
#else
	gzFile f = gzopen(filename, "r");
#endif

	if (f == Z_NULL)
		return false;

	//char temp[256];
	//fgets(temp,255,f); //.text section layout
	//fgets(temp,255,f); //  Starting        Virtual
	//fgets(temp,255,f); //  address  Size   address
	//fgets(temp,255,f); //  -----------------------

	bool started = false;
	bool hasModules = false;

	while (!gzeof(f)) {
		char line[512], temp[256] = {0};
		char *p = gzgets(f, line, 512);
		if (p == NULL)
			break;

		// Chop any newlines off.
		for (size_t i = strlen(line) - 1; i > 0; i--) {
			if (line[i] == '\r' || line[i] == '\n') {
				line[i] = '\0';
			}
		}

		if (strlen(line) < 4 || sscanf(line, "%255s", temp) != 1)
			continue;

		if (strcmp(temp,"UNUSED")==0) continue;
		if (strcmp(temp,".text")==0)  {started=true;continue;};
		if (strcmp(temp,".init")==0)  {started=true;continue;};
		if (strcmp(temp,"Starting")==0) continue;
		if (strcmp(temp,"extab")==0) continue;
		if (strcmp(temp,".ctors")==0) break;
		if (strcmp(temp,".dtors")==0) break;
		if (strcmp(temp,".rodata")==0) continue;
		if (strcmp(temp,".data")==0) continue;
		if (strcmp(temp,".sbss")==0) continue;
		if (strcmp(temp,".sdata")==0) continue;
		if (strcmp(temp,".sdata2")==0) continue;
		if (strcmp(temp,"address")==0)  continue;
		if (strcmp(temp,"-----------------------")==0)  continue;
		if (strcmp(temp,".sbss2")==0) break;
		if (temp[1]==']') continue;

		if (!started) continue;

		u32 address = -1, size, vaddress = -1;
		int moduleIndex = 0;
		int typeInt;
		SymbolType type;
		char name[128] = {0};

		if (sscanf(line, ".module %x %08x %08x %127c", (unsigned int *)&moduleIndex, &address, &size, name) >= 3) {
			// Found a module definition.
			ModuleEntry mod;
			mod.index = moduleIndex;
			strcpy(mod.name, name);
			mod.start = address;
			mod.size = size;
			modules.push_back(mod);
			hasModules = true;
			continue;
		}

		sscanf(line, "%08x %08x %x %i %127c", &address, &size, &vaddress, &typeInt, name);
		type = (SymbolType) typeInt;
		if (!hasModules) {
			if (!Memory::IsValidAddress(vaddress)) {
				ERROR_LOG(LOADER, "Invalid address in symbol file: %08x (%s)", vaddress, name);
				continue;
			}
		} else {
			// The 3rd field is now used for the module index.
			moduleIndex = vaddress;
			vaddress = GetModuleAbsoluteAddr(address, moduleIndex);
			if (!Memory::IsValidAddress(vaddress)) {
				ERROR_LOG(LOADER, "Invalid address in symbol file: %08x (%s)", vaddress, name);
				continue;
			}
		}

		if (type == ST_DATA && size == 0)
			size = 4;

		if (!strcmp(name, ".text") || !strcmp(name, ".init") || strlen(name) <= 1) {

		} else {
			switch (type)
			{
			case ST_FUNCTION:
				AddFunction(name, vaddress, size, moduleIndex);
				break;
			case ST_DATA:
				AddData(vaddress,size,DATATYPE_BYTE, moduleIndex);
				if (name[0] != 0)
					AddLabel(name, vaddress, moduleIndex);
				break;
			case ST_NONE:
			case ST_ALL:
				// Shouldn't be possible.
				break;
			}
		}
	}
	gzclose(f);
	SortSymbols();
	return started;
}
Example #20
0
boolean
TxCache::save(const wchar_t *path, const wchar_t *filename, int config)
{
  if (!_cache.empty()) {
    /* dump cache to disk */
    char cbuf[MAX_PATH];

    boost::filesystem::wpath cachepath(path);
    boost::filesystem::create_directory(cachepath);

    /* Ugly hack to enable fopen/gzopen in Win9x */
#ifdef BOOST_WINDOWS_API
    wchar_t curpath[MAX_PATH];
    GETCWD(MAX_PATH, curpath);
    CHDIR(cachepath.wstring().c_str());
#else
    char curpath[MAX_PATH];
    wcstombs(cbuf, cachepath.wstring().c_str(), MAX_PATH);
    if (GETCWD(MAX_PATH, curpath) == NULL)
        ERRLOG("Error while retrieving working directory!");
    if (CHDIR(cbuf) != 0)
        ERRLOG("Error while changing current directory to '%s'!", cbuf);
#endif

    wcstombs(cbuf, filename, MAX_PATH);

    gzFile gzfp = gzopen(cbuf, "wb1");
    DBG_INFO(80, L"gzfp:%x file:%ls\n", gzfp, filename);
    if (gzfp) {
      /* write header to determine config match */
      gzwrite(gzfp, &config, 4);

      std::map<uint64, TXCACHE*>::iterator itMap = _cache.begin();
      while (itMap != _cache.end()) {
        uint8 *dest    = (*itMap).second->info.data;
        uint32 destLen = (*itMap).second->size;
        uint16 format  = (*itMap).second->info.format;

        /* to keep things simple, we save the texture data in a zlib uncompressed state. */
        /* sigh... for those who cannot wait the extra few seconds. changed to keep
         * texture data in a zlib compressed state. if the GZ_TEXCACHE or GZ_HIRESTEXCACHE
         * option is toggled, the cache will need to be rebuilt.
         */
        /*if (format & GR_TEXFMT_GZ) {
          dest = _gzdest0;
          destLen = _gzdestLen;
          if (dest && destLen) {
            if (uncompress(dest, &destLen, (*itMap).second->info.data, (*itMap).second->size) != Z_OK) {
              dest = NULL;
              destLen = 0;
            }
            format &= ~GR_TEXFMT_GZ;
          }
        }*/

        if (dest && destLen) {
          /* texture checksum */
          gzwrite(gzfp, &((*itMap).first), 8);

          /* other texture info */
          gzwrite(gzfp, &((*itMap).second->info.width), 4);
          gzwrite(gzfp, &((*itMap).second->info.height), 4);
          gzwrite(gzfp, &format, 2);

          gzwrite(gzfp, &((*itMap).second->info.smallLodLog2), 4);
          gzwrite(gzfp, &((*itMap).second->info.largeLodLog2), 4);
          gzwrite(gzfp, &((*itMap).second->info.aspectRatioLog2), 4);

          gzwrite(gzfp, &((*itMap).second->info.tiles), 4);
          gzwrite(gzfp, &((*itMap).second->info.untiled_width), 4);
          gzwrite(gzfp, &((*itMap).second->info.untiled_height), 4);

          gzwrite(gzfp, &((*itMap).second->info.is_hires_tex), 1);

          gzwrite(gzfp, &destLen, 4);
          gzwrite(gzfp, dest, destLen);
        }

        itMap++;

        /* not ready yet */
        /*if (_callback)
          (*_callback)(L"Total textures saved to HDD: %d\n", std::distance(itMap, _cache.begin()));*/
      }
      gzclose(gzfp);
    }

    if (CHDIR(curpath) != 0)
        ERRLOG("Error while changing current directory back to original path of '%s'!", curpath);
  }

  return _cache.empty();
}
Example #21
0
int AFILE_DetectFileType(const char *filename)
{
	UBYTE header[4];
	int file_length;
	FILE *fp = fopen(filename, "rb");
	if (fp == NULL)
		return AFILE_ERROR;
	if (fread(header, 1, 4, fp) != 4) {
		fclose(fp);
		return AFILE_ERROR;
	}
	switch (header[0]) {
	case 0:
		if (header[1] == 0 && (header[2] != 0 || header[3] != 0) /* && file_length < 37 * 1024 */) {
			fclose(fp);
			return AFILE_BAS;
		}
		break;
	case 0x1f:
		if (header[1] == 0x8b) {
#ifndef HAVE_LIBZ
			fclose(fp);
			Log_print("\"%s\" is a compressed file.", filename);
			Log_print("This executable does not support compressed files. You can uncompress this file");
			Log_print("with an external program that supports gzip (*.gz) files (e.g. gunzip)");
			Log_print("and then load into this emulator.");
			return AFILE_ERROR;
#else /* HAVE_LIBZ */
			gzFile gzf;
			fclose(fp);
			gzf = gzopen(filename, "rb");
			if (gzf == NULL)
				return AFILE_ERROR;
			if (gzread(gzf, header, 4) != 4) {
				gzclose(gzf);
				return AFILE_ERROR;
			}
			gzclose(gzf);
			if (header[0] == 0x96 && header[1] == 0x02)
				return AFILE_ATR_GZ;
			if (header[0] == 'A' && header[1] == 'T' && header[2] == 'A' && header[3] == 'R')
				return AFILE_STATE_GZ;
			return AFILE_XFD_GZ;
#endif /* HAVE_LIBZ */
		}
		break;
	case '0':
	case '1':
	case '2':
	case '3':
	case '4':
	case '5':
	case '6':
	case '7':
	case '8':
	case '9':
		if ((header[1] >= '0' && header[1] <= '9') || header[1] == ' ') {
			fclose(fp);
			return AFILE_LST;
		}
		break;
	case 'A':
		if (header[1] == 'T' && header[2] == 'A' && header[3] == 'R') {
			fclose(fp);
			return AFILE_STATE;
		}
		if (header[1] == 'T' && header[2] == '8' && header[3] == 'X') {
			fclose(fp);
			return AFILE_ATX;
		}
		break;
	case 'C':
		if (header[1] == 'A' && header[2] == 'R' && header[3] == 'T') {
			fclose(fp);
			return AFILE_CART;
		}
		break;
	case 0x96:
		if (header[1] == 0x02) {
			fclose(fp);
			return AFILE_ATR;
		}
		break;
	case 0xf9:
	case 0xfa:
		fclose(fp);
		return AFILE_DCM;
	case 0xff:
		if (header[1] == 0xff && (header[2] != 0xff || header[3] != 0xff)) {
			fclose(fp);
			return AFILE_XEX;
		}
		break;
	default:
		break;
	}
	file_length = Util_flen(fp);
	fclose(fp);
	/* Detect .pro images */
	/* # of sectors is in header */
	if ((file_length-16)%(128+12) == 0 &&
			header[0]*256 + header[1] == (file_length-16)/(128+12) &&
			header[2] == 'P') {
#ifdef DEBUG_PRO
		Log_print(".pro file detected");
#endif
		return AFILE_PRO;
	}
	/* 40K or a-power-of-two between 4K and CARTRIDGE_MAX_SIZE */
	if (file_length >= 4 * 1024 && file_length <= CARTRIDGE_MAX_SIZE
	 && ((file_length & (file_length - 1)) == 0 || file_length == 40 * 1024))
		return AFILE_ROM;
	/* BOOT_TAPE is a raw file containing a program booted from a tape */
	if ((header[1] << 7) == file_length)
		return AFILE_BOOT_TAPE;
	if ((file_length & 0x7f) == 0)
		return AFILE_XFD;
	if (IMG_TAPE_FileSupported(header))
		return AFILE_CAS;
	return AFILE_ERROR;
}
Example #22
0
boolean
TxCache::load(const wchar_t *path, const wchar_t *filename, int config)
{
  /* find it on disk */
  char cbuf[MAX_PATH];

  boost::filesystem::wpath cachepath(path);

#ifdef BOOST_WINDOWS_API
  wchar_t curpath[MAX_PATH];
  GETCWD(MAX_PATH, curpath);
  CHDIR(cachepath.wstring().c_str());
#else
  char curpath[MAX_PATH];
  wcstombs(cbuf, cachepath.wstring().c_str(), MAX_PATH);
  if (GETCWD(MAX_PATH, curpath) == NULL)
      ERRLOG("Error while retrieving working directory!");
  if (CHDIR(cbuf) != 0)
      ERRLOG("Error while changing current directory to '%s'!", cbuf);
#endif

  wcstombs(cbuf, filename, MAX_PATH);

  gzFile gzfp = gzopen(cbuf, "rb");
  DBG_INFO(80, L"gzfp:%x file:%ls\n", gzfp, filename);
  if (gzfp) {
    /* yep, we have it. load it into memory cache. */
    int dataSize;
    uint64 checksum;
    GHQTexInfo tmpInfo;
    int tmpconfig;
    /* read header to determine config match */
    gzread(gzfp, &tmpconfig, 4);

    if (tmpconfig == config) {
      do {
        memset(&tmpInfo, 0, sizeof(GHQTexInfo));

        gzread(gzfp, &checksum, 8);

        gzread(gzfp, &tmpInfo.width, 4);
        gzread(gzfp, &tmpInfo.height, 4);
        gzread(gzfp, &tmpInfo.format, 2);

        gzread(gzfp, &tmpInfo.smallLodLog2, 4);
        gzread(gzfp, &tmpInfo.largeLodLog2, 4);
        gzread(gzfp, &tmpInfo.aspectRatioLog2, 4);

        gzread(gzfp, &tmpInfo.tiles, 4);
        gzread(gzfp, &tmpInfo.untiled_width, 4);
        gzread(gzfp, &tmpInfo.untiled_height, 4);

        gzread(gzfp, &tmpInfo.is_hires_tex, 1);

        gzread(gzfp, &dataSize, 4);

        tmpInfo.data = (uint8*)malloc(dataSize);
        if (tmpInfo.data) {
          gzread(gzfp, tmpInfo.data, dataSize);

          /* add to memory cache */
          add(checksum, &tmpInfo, (tmpInfo.format & GR_TEXFMT_GZ) ? dataSize : 0);

          free(tmpInfo.data);
        } else {
          gzseek(gzfp, dataSize, SEEK_CUR);
        }

        /* skip in between to prevent the loop from being tied down to vsync */
        if (_callback && (!(_cache.size() % 100) || gzeof(gzfp)))
          (*_callback)(L"[%d] total mem:%.02fmb - %ls\n", _cache.size(), (float)_totalSize/1000000, filename);

      } while (!gzeof(gzfp));
      gzclose(gzfp);
    } else {
      if ((tmpconfig & HIRESTEXTURES_MASK) != (config & HIRESTEXTURES_MASK)) {
        const char *conf_str;
        if ((tmpconfig & HIRESTEXTURES_MASK) == NO_HIRESTEXTURES)
          conf_str = "0";
        else if ((tmpconfig & HIRESTEXTURES_MASK) == RICE_HIRESTEXTURES)
          conf_str = "1";
        else
          conf_str = "set to an unsupported format";

        WriteLog(M64MSG_WARNING, "Ignored texture cache due to incompatible setting: ghq_hirs must be %s", conf_str);
      }
      if ((tmpconfig & COMPRESS_HIRESTEX) != (config & COMPRESS_HIRESTEX))
        WriteLog(M64MSG_WARNING, "Ignored texture cache due to incompatible setting: ghq_hirs_cmpr must be %s", (tmpconfig & COMPRESS_HIRESTEX) ? "True" : "False");
      if ((tmpconfig & COMPRESSION_MASK) != (config & COMPRESSION_MASK) && (tmpconfig & COMPRESS_HIRESTEX)) {
        const char *conf_str;
        if ((tmpconfig & COMPRESSION_MASK) == FXT1_COMPRESSION)
          conf_str = "1";
        else if ((tmpconfig & COMPRESSION_MASK) == S3TC_COMPRESSION)
          conf_str = "0";
        else
          conf_str = "set to an unsupported format";

        WriteLog(M64MSG_WARNING, "Ignored texture cache due to incompatible setting: ghq_cmpr must be %s", conf_str);
      }
      if ((tmpconfig & TILE_HIRESTEX) != (config & TILE_HIRESTEX))
        WriteLog(M64MSG_WARNING, "Ignored texture cache due to incompatible setting: ghq_hirs_tile must be %s", (tmpconfig & TILE_HIRESTEX) ? "True" : "False");
      if ((tmpconfig & FORCE16BPP_HIRESTEX) != (config & FORCE16BPP_HIRESTEX))
        WriteLog(M64MSG_WARNING, "Ignored texture cache due to incompatible setting: ghq_hirs_f16bpp must be %s", (tmpconfig & FORCE16BPP_HIRESTEX) ? "True" : "False");
      if ((tmpconfig & GZ_HIRESTEXCACHE) != (config & GZ_HIRESTEXCACHE))
        WriteLog(M64MSG_WARNING, "ghq_hirs_gz must be %s", (tmpconfig & GZ_HIRESTEXCACHE) ? "True" : "False");
      if ((tmpconfig & LET_TEXARTISTS_FLY) != (config & LET_TEXARTISTS_FLY))
        WriteLog(M64MSG_WARNING, "Ignored texture cache due to incompatible setting: ghq_hirs_let_texartists_fly must be %s", (tmpconfig & LET_TEXARTISTS_FLY) ? "True" : "False");

      if ((tmpconfig & FILTER_MASK) != (config & FILTER_MASK)) {
        const char *conf_str;
        if ((tmpconfig & FILTER_MASK) == NO_FILTER)
          conf_str = "0";
        else if ((tmpconfig & FILTER_MASK) == SMOOTH_FILTER_1)
          conf_str = "1";
        else if ((tmpconfig & FILTER_MASK) == SMOOTH_FILTER_2)
          conf_str = "2";
        else if ((tmpconfig & FILTER_MASK) == SMOOTH_FILTER_3)
          conf_str = "3";
        else if ((tmpconfig & FILTER_MASK) == SMOOTH_FILTER_4)
          conf_str = "4";
        else if ((tmpconfig & FILTER_MASK) == SHARP_FILTER_1)
          conf_str = "5";
        else if ((tmpconfig & FILTER_MASK) == SHARP_FILTER_2)
          conf_str = "6";
        else
          conf_str = "set to an unsupported format";
        WriteLog(M64MSG_WARNING, "Ignored texture cache due to incompatible setting: ghq_fltr must be %s", conf_str);
      }

      if ((tmpconfig & ENHANCEMENT_MASK) != (config & ENHANCEMENT_MASK)) {
        const char *conf_str;
        if ((tmpconfig & ENHANCEMENT_MASK) == NO_ENHANCEMENT)
          conf_str = "0";
        else if ((tmpconfig & ENHANCEMENT_MASK) == X2_ENHANCEMENT)
          conf_str = "2";
        else if ((tmpconfig & ENHANCEMENT_MASK) == X2SAI_ENHANCEMENT)
          conf_str = "3";
        else if ((tmpconfig & ENHANCEMENT_MASK) == HQ2X_ENHANCEMENT)
          conf_str = "4";
        else if ((tmpconfig & ENHANCEMENT_MASK) == HQ2XS_ENHANCEMENT)
          conf_str = "5";
        else if ((tmpconfig & ENHANCEMENT_MASK) == LQ2X_ENHANCEMENT)
          conf_str = "6";
        else if ((tmpconfig & ENHANCEMENT_MASK) == LQ2XS_ENHANCEMENT)
          conf_str = "7";
        else if ((tmpconfig & ENHANCEMENT_MASK) == HQ4X_ENHANCEMENT)
          conf_str = "8";
        else
          conf_str = "set to an unsupported format";
        WriteLog(M64MSG_WARNING, "Ignored texture cache due to incompatible setting: ghq_enht must be %s", conf_str);
      }

      if ((tmpconfig & COMPRESS_TEX) != (config & COMPRESS_TEX))
        WriteLog(M64MSG_WARNING, "Ignored texture cache due to incompatible setting: ghq_enht_cmpr must be %s", (tmpconfig & COMPRESS_TEX) ? "True" : "False");
      if ((tmpconfig & FORCE16BPP_TEX) != (config & FORCE16BPP_TEX))
        WriteLog(M64MSG_WARNING, "Ignored texture cache due to incompatible setting: ghq_enht_f16bpp must be %s", (tmpconfig & FORCE16BPP_TEX) ? "True" : "False");
      if ((tmpconfig & GZ_TEXCACHE) != (config & GZ_TEXCACHE))
        WriteLog(M64MSG_WARNING, "Ignored texture cache due to incompatible setting: ghq_enht_gz must be %s", (tmpconfig & GZ_TEXCACHE) ? "True" : "False");
    }
  }

  if (CHDIR(curpath) != 0)
      ERRLOG("Error while changing current directory back to original path of '%s'!", curpath);

  return !_cache.empty();
}
Example #23
0
int main() 
{
  u32 	nbufsread;     	// the number of buffers read in
  char		*savebuf;       // the buffer to save 
  SysInfo		*systmp;
  ChannelInfo		*chtmp;
  EventBuffer		*event;
  DIOBuffer		*dio;
  PosBuffer		posbuf[1];
  char 		tmpstring[TMPSTRINGLEN+1];	// a temporary buffer 
  char 		filename[TMPSTRINGLEN+1];	// a temporary buffer 
  int			tmplen; 	// the length of the data in tmbbuf
  int 		tmp1, tmp2, diskfree;
  short		datatype;
  char 		*outfilename;	// the name of the file to save data to
  gzFile		outfile;	// the pointer for the output file
  int                 message;	// a temporary message variable
  int			messagedata[MAX_BUFFER_SIZE]; // message data can contain a sysinfo or channelinfo structure
  int			messagedatalen; // the length of the data in the message
  int			nspikes;
  int			savebufsize;     // the size of the input data buffer in bytes
  u32			tmpbufsize;			
  int 		error;
  int 		i, id;
  int			lastmessage;	// the last socket we got a message on
  int			tmpsize;
  int			closefile;	// 1 if the file is to be closed
  int			newdata;	// 1 if new data were received 
  FILE		*pipe;
  OpenFileMessage 	*ofm;

  sysinfo.acq = 0;
  sysinfo.diskon = 0;
  sysinfo.fileopen = 0;
  closefile = 0;
  outfile = NULL;

  /* set the type of program we are in for messaging */
  sysinfo.program_type = SPIKE_SAVE_DATA;

  sysinfo.statusfile == NULL;

  if (STATUSFILE == NULL) {
    /* open up the status file if it is not stderr*/
    gethostname(tmpstring,80);
    sprintf(filename, "spike_save_data_status_%s", tmpstring);
    if ((STATUSFILE = fopen(filename, "w")) == NULL) {
      fprintf(stderr, "spike_main: error opening status file\n");
      exit(-1);
    }
  }

  fprintf(STATUSFILE, "spike_save_data: starting messaging\n");
  if (StartNetworkMessaging(server_message, client_message, server_data, 
        client_data) < 0) {
    fprintf(STATUSFILE, "spike_save: Error establishing messaging\n");
    saveexit(1);
  }

  // we need to allocate enough space for a buffer of any type, so
  // we allocate a large buffer
  //savebuf = (char *) malloc(SAVE_BUF_SIZE);
  savebuf = (char *) malloc(1000000 * sizeof(char));

  while (1) { 
    newdata = 0;
    /* set up the initial list of file descriptors to watch */
    SetupFDList(&readfds, &maxfds, server_message, server_data);
    select(maxfds, &readfds, NULL, NULL, NULL);
    id = 0;
    /* check for incoming data */
    while ((i = netinfo.datainfd[id]) != -1) {
      if (FD_ISSET(server_data[i].fd, &readfds)) {
        /* check to see if the disk is on, and if not, get the message
         * and discard it. Note that this works for everyone except
         * spike_process_posdata, as it only sends data out when the
         * disk is on.  */
        if ((!sysinfo.diskon) && (i != SPIKE_PROCESS_POSDATA)) {
          message = GetMessage(server_data[i].fd, savebuf, &savebufsize, 1);
        }
        else {
          newdata = 1;
          tmpsize = 0;
          switch(i) {
            case SPIKE_DAQ: 
              /* get the data */
              message = GetMessage(server_data[SPIKE_DAQ].fd, savebuf, &savebufsize, 1);
              /* check to see if this is an event */
              if (message == EVENT) {
                event = (EventBuffer *) messagedata;
                datatype = EVENT_DATA_TYPE;
                if ((tmpsize = gzwrite(outfile, &datatype, 
                        sizeof(short))) != sizeof(short)) {
                  error = 1;
                  fprintf(STATUSFILE, "spike_save_data: writing error: %d written, %d tried\n", tmpsize, sizeof(short));

                }
                if ((tmpsize = gzwrite(outfile, event, sizeof(EventBuffer))) != sizeof(EventBuffer)) {
                  error = 1;
                  fprintf(STATUSFILE, "spike_save_data: writing error: %d written, %d tried\n", tmpsize, savebufsize);
                }
              }
              else if (message == DIGITALIO_EVENT) {
                /* write the behav, the port, and the status */
                datatype = DIGITALIO_DATA_TYPE;
                if ((tmpsize = gzwrite(outfile, &datatype, 
                        sizeof(short))) != sizeof(short)) {
                  error = 1;
                  fprintf(STATUSFILE, "spike_save_data: writing error: %d written, %d tried\n", tmpsize, sizeof(short));

                }
                /* write out the data */
                if ((tmpsize = gzwrite(outfile, savebuf, savebufsize))
                    != savebufsize) {
                  fprintf(STATUSFILE, "spike_save_data: writing error: %d written, %d tried\n", tmpsize, savebufsize);
                  error = 1;
                }
              } 
              else {
                if (message == SPIKE_DATA) {
                  datatype = SPIKE_DATA_TYPE;
                }
                else if (message == CONTINUOUS_DATA) {
                  datatype = CONTINUOUS_DATA_TYPE;
                }
                /* write out the data type */
                if ((tmpsize = gzwrite(outfile, &datatype, 
                        sizeof(short))) != sizeof(short)) {
                  error = 1;
                  fprintf(STATUSFILE, "spike_save_data: writing error: %d written, %d tried\n", tmpsize, sizeof(short));

                }

                /* write out the data size */
                if ((tmpsize = gzwrite(outfile, &savebufsize, sizeof(int))) != sizeof(int)) {
                  error = 1;
                  fprintf(STATUSFILE, "spike_save_data: writing error: %d written, %d tried\n", tmpsize, savebufsize);

                }

                /* write out the data */
                if ((tmpsize = gzwrite(outfile, savebuf, savebufsize)) != savebufsize) {
                  error = 1;
                  fprintf(STATUSFILE, "spike_save_data: writing error: %d written, %d tried\n", tmpsize, savebufsize);

                }
              }
              break;
            case SPIKE_PROCESS_POSDATA:
              /* get the position buffer */
              GetMessage(server_data[SPIKE_PROCESS_POSDATA].fd, 
                  savebuf, &savebufsize, 1);
              /* write out the data type */
              datatype = POSITION_DATA_TYPE;
              if ((tmpsize = gzwrite(outfile, &datatype, 
                      sizeof(short))) != sizeof(short)) {
                error = 1;
                fprintf(STATUSFILE, "spike_save_data: writing error: %d written, %d tried\n", tmpsize, sizeof(short));

              }
              /* write out the data. Note the that first element of
               * the data is the size of the buffer, so we don't need
               * to*/
              //fprintf(stderr, "writing out %d position bytes \n", savebufsize);
              if ((tmpsize = gzwrite(outfile, savebuf, savebufsize))
                  != savebufsize) {
                fprintf(STATUSFILE, "spike_save_data: writing error: %d written, %d tried\n", tmpsize, savebufsize);
                error = 1;
              }
              break;
            case SPIKE_MAIN: 
              /* get the event or digital IO event*/
              message = GetMessage(server_data[SPIKE_MAIN].fd, 
                  savebuf, &savebufsize, 0);
              if (message == EVENT) {
                event = (EventBuffer *) savebuf;
                datatype = EVENT_DATA_TYPE;
                if ((tmpsize = gzwrite(outfile, &datatype, 
                        sizeof(short))) != sizeof(short)) {
                  error = 1;
                  fprintf(STATUSFILE, "spike_save_data: writing error: %d written, %d tried\n", tmpsize, sizeof(short));

                }
                if ((tmpsize = gzwrite(outfile, event, sizeof(EventBuffer))) != sizeof(EventBuffer)) {
                  error = 1;
                  fprintf(STATUSFILE, "spike_save_data: writing error: %d written, %d tried\n", tmpsize, savebufsize);
                }
              }
              else if (message == DIGITALIO_EVENT) {
                /* write the behav, the port, and the status */
                datatype = DIGITALIO_DATA_TYPE;
                if ((tmpsize = gzwrite(outfile, &datatype, 
                        sizeof(short))) != sizeof(short)) {
                  error = 1;
                  fprintf(STATUSFILE, "spike_save_data: writing error: %d written, %d tried\n", tmpsize, sizeof(short));

                }
                /* write out the data */
                if ((tmpsize = gzwrite(outfile, savebuf, savebufsize))
                    != savebufsize) {
                  fprintf(STATUSFILE, "spike_save_data: writing error: %d written, %d tried\n", tmpsize, savebufsize);
                  error = 1;
                }
              } 
              else if (message == TIME_CHECK) {
                /* write the time data */
                datatype = TIME_CHECK_DATA_TYPE;
                if ((tmpsize = gzwrite(outfile, &datatype, 
                        sizeof(short))) != sizeof(short)) {
                  error = 1;
                  fprintf(STATUSFILE, "spike_save_data: writing error: %d written, %d tried\n", tmpsize, sizeof(short));

                }
                /* write out the data */
                if ((tmpsize = gzwrite(outfile, savebuf, savebufsize))
                    != savebufsize) {
                  fprintf(STATUSFILE, "spike_save_data: writing error: %d written, %d tried\n", tmpsize, savebufsize);
                  error = 1;
                }
              }
              break;
          }
          if (error) {
            /* stop saving and send a SAVE_ERROR message */
            fprintf(STATUSFILE,"spike_save_data: Error in saving data\n");
            sysinfo.diskon = 0;
            SendMessage(client_message[SPIKE_MAIN].fd, SAVE_ERROR, NULL, 0);
          }
          lastmessage = i;
          break;
        }
      }
      id++;
    }
    id = 0;
    /* check for messages */
    while ((i = netinfo.messageinfd[id]) != -1) {
      if (FD_ISSET(server_message[i].fd, &readfds)) {
        message = GetMessage(server_message[i].fd,
            (char *) messagedata, &messagedatalen, 0);
        error = 0;
        switch(message) {
          case STOP_ACQUISITION:
            sysinfo.acq = 0;
            SendMessage(client_message[SPIKE_MAIN].fd, ACQUISITION_STOPPED, 
                NULL, 0);
            break;
          case START_ACQUISITION:
            sysinfo.acq = 1;
            SendMessage(client_message[SPIKE_MAIN].fd, ACQUISITION_STARTED, 
                NULL, 0);
            break;
          case START_SAVE:
            sysinfo.diskon = 1;
            SendMessage(client_message[SPIKE_MAIN].fd, 
                SAVE_STARTED, NULL, 0);
            break;
          case STOP_SAVE:
            sysinfo.diskon = 0;
            SendMessage(client_message[SPIKE_MAIN].fd, 
                SAVE_STOPPED, NULL, 0);
            break;
          case OPEN_FILE:
            /* we can get this message in the middle of saving 
             * data if the main program has determined that the 
             * current file size is too  large, so we need to 
             * close the current file if it is open */
            if (outfile != NULL)  gzclose(outfile);
            ofm = (OpenFileMessage *) messagedata;

            /* we will have already checked to make sure that the 
             * file can be opened */
            /* make sure the file name is correctly terminated */
            strncpy(sysinfo.datafilename, ofm->filename, 
                sizeof(sysinfo.datafilename));
            sysinfo.datafilename[sizeof(sysinfo.datafilename)-1] = '\0';
            if (ofm->type != OpenFileMessage::GZip) {
              sprintf(tmpstring, "file type is not GZip, but rather was: %d; defaulting to GZip!\n", (int)ofm->type);
              StatusMessage(tmpstring, client_message);
            }
            // build string to pass to gzopen specifying open more, 
            // compression level, etc
            snprintf(tmpstring, sizeof(tmpstring), "ab%u", 
                ofm->compression_level);
            sysinfo.use_compression = ofm->compression_level ? 
              true : false;
            sysinfo.compression_level = ofm->compression_level;
            outfile = gzopen(sysinfo.datafilename, tmpstring);
            if( Z_NULL == outfile) {
              fprintf(STATUSFILE, "gzopen() failed to open file: %s with options %s\n", sysinfo.datafilename, tmpstring);
            }
            else{
              fprintf(STATUSFILE, "gzopen() opened file: %s with options %s\n", sysinfo.datafilename, tmpstring);
              sysinfo.fileopen = 1;
            }     
            snprintf(tmpstring, sizeof(tmpstring), "df %s\n", 
                sysinfo.datafilename);
            diskfree = 0;
            if ((pipe = popen(tmpstring, "r")) == NULL) {
              sprintf(tmpstring, "Error opening pipe to check disk usage");
              StatusMessage(tmpstring, client_message);
            }
            else {
              fgets(tmpstring, TMPSTRINGLEN, pipe);
              fscanf(pipe, "%s%d%d%d", tmpstring, &tmp1, &tmp2, &diskfree);
            }
            /* convert KB to MB for diskfree */
            diskfree /= 1024;
            pclose(pipe);
            SendMessage(client_message[SPIKE_MAIN].fd, FILE_OPENED,
                (char *) &diskfree, sizeof(int));
            break;
          case CLOSE_FILE:
            closefile = 1;
            break;
          case SYSTEM_INFO:
            /* we don't need any systeminfo information in this
             * module */
            break;
          case EXIT:

            saveexit(0);		        
            break;
          default:
            break;
        }
      }
      id++;
    }
    /* close the file only if no new data came in on the last iteration of the for loop */
    if (closefile && !newdata) {
      gzclose(outfile);
      outfile = NULL;
      sysinfo.fileopen = 0;
      SendMessage(client_message[SPIKE_MAIN].fd, FILE_CLOSED, NULL, 0);
      closefile = 0;
    }
  }
  return 0;
}
static int check_path(const char *hostname, int tm_start, int tm_end, char *buffer, size_t bufferlen) /* {{{ */
{
        /* Path syntax where timestamp = AABBCCDDDD :
         * ${toppsdatadir}/${hostname}/AA/AABB/AABBCC0000-X.gz
         * Checking path means testing that the ${toppsdatadir}/${hostname}/AA/AABB directory exists.
         * If not, check with tm_margin.
         *
         * Start at tm_start. If tm_end < tm_start, search backward.
         */
        gzFile *gzfh=NULL;
        int offset = 0;
        int status;
        short file_found;
        time_t tm;
        int n=0;
        int best_distance;
        int max_distance;
        int best_n;
        int best_tm;
        int last_seen_n_low,last_seen_n_high;
        int last_seen_tm_low,last_seen_tm_high;
        int last_before_flush_n = -1;
        int last_before_flush_tm = -1;
        int flush_needed = 0;
        int flush_already_done = 0;
        short watchdog;
        int search_direction;

        if (toppsdatadir != NULL)
        {
                status = ssnprintf (buffer, bufferlen, "%s/", toppsdatadir);
                if ((status < 1) || (status >= bufferlen )) {
                        ERROR(OUTPUT_PREFIX_JSONRPC_CB_TOPPS "Filename buffer too small (%s:%d)", __FILE__, __LINE__);
                        return (JSONRPC_ERROR_CODE_32603_INTERNAL_ERROR);
                }
                offset += status;
        }
        DEBUG(OUTPUT_PREFIX_JSONRPC_CB_TOPPS "DEBUG toppsdatadir='%s' (%s:%d)", toppsdatadir, __FILE__, __LINE__);
        DEBUG(OUTPUT_PREFIX_JSONRPC_CB_TOPPS "DEBUG offset = %d (%s:%d)", offset, __FILE__, __LINE__);

        status = ssnprintf (buffer + offset, bufferlen - offset,
                        "%s/", hostname);
        if ((status < 1) || (status >= bufferlen - offset)) {
                ERROR(OUTPUT_PREFIX_JSONRPC_CB_TOPPS "Filename buffer too small (%s:%d)", __FILE__, __LINE__);
                return (JSONRPC_ERROR_CODE_32603_INTERNAL_ERROR);
        }
        offset += status;
        DEBUG(OUTPUT_PREFIX_JSONRPC_CB_TOPPS "DEBUG offset = %d (%s:%d)", offset, __FILE__, __LINE__);

        /* Start search */
        max_distance = abs(tm_start - tm_end); /* distance should be < max_distance */
        file_found = 0;
        best_distance = max_distance + 1;
        best_n = 0;
        best_tm = 0;
        last_seen_tm_high = 0;
        last_seen_tm_low = 0;
        last_seen_n_high = 0;
        last_seen_n_low = 0;
        search_direction = 1; /* positive value to go forward at first time */

        DEBUG(OUTPUT_PREFIX_JSONRPC_CB_TOPPS "DEBUG WE ARE SEARCHING FOR tm_start = '%d' max_distance = %d (%s:%d)", tm_start, max_distance, __FILE__, __LINE__);

        n = 0;
        tm = 10000 * (int)(tm_start / 10000);
        if(tm_start <= tm_end) tm -= 10000; /* if searching forward, search starts before tm_start. */
        //        else                   tm += 10000; /* if searching backward, search starts after tm_start */

#define WATCHDOGMAX 100 /* max number of cycles in this loop. Prevent from infinite loop if something is missing in this complex algo */
        for(watchdog = 0; watchdog < WATCHDOGMAX; watchdog++) { /* There are many cases to get out of this loop. See the many 'break' instructions */
                int local_err;

                if((0 == flush_already_done) && (2 == flush_needed)) {
                        /* Back to flush position */
                        assert(last_before_flush_tm != -1); /* ensure that it was initialized */
                        assert(last_before_flush_n != -1);  /* ensure that it was initialized */
                        tm = last_before_flush_tm;
                        n = last_before_flush_n;
                }

                if(mkpath_by_tm_and_num(buffer + offset, bufferlen - offset,tm, n)) {
                        return (JSONRPC_ERROR_CODE_32603_INTERNAL_ERROR);
                }

                /* Try to open the file.
                 * Flush if necessary
                 */
                DEBUG(OUTPUT_PREFIX_JSONRPC_CB_TOPPS "DEBUG tm = %ld filename = '%s' (%s:%d)", tm, buffer, __FILE__, __LINE__);
                if(NULL == (gzfh = gzopen(buffer, "r"))) {
                        DEBUG(OUTPUT_PREFIX_JSONRPC_CB_TOPPS "DEBUG COULD NOT OPEN = '%s' (%s:%d)", buffer, __FILE__, __LINE__);
                        if((0 == flush_already_done) && (2 == flush_needed)) {
                                /* Open failed. Maybe we should flush ? */
                                int status;
                                time_t flush_tm;
                                DEBUG(OUTPUT_PREFIX_JSONRPC_CB_TOPPS "DEBUG Calling plugin_flush('write_top',10,%s) (%s:%d)", hostname, __FILE__, __LINE__);
                                status = plugin_flush ("write_top", 0, hostname);
                                if (status == 0) {
                                        /* Flush done. Try again with older values */
                                }
                                flush_already_done = 1;

                                flush_tm = time(NULL);
                                while((time(NULL) - flush_tm) < 10) { /* wait no more than 10 seconds for flush */
                                        sleep(1);
                                        DEBUG(OUTPUT_PREFIX_JSONRPC_CB_TOPPS "DEBUG Trying to open '%s' again... (%s:%d)", buffer, __FILE__, __LINE__);
                                        if(NULL != (gzfh = gzopen(buffer, "r")))  break;
                                }
                        }
                }

                /* File is supposed to be opened, with or without a flush.
                 * Check that the file was really opened
                 */
                if(NULL == gzfh) { /* File could NOT be opened */
                        if((0 == flush_already_done) && (search_direction > 0)) {
                                if(0 == flush_needed) {
                                        last_before_flush_tm = tm; /* save this position */
                                        last_before_flush_n = n;
                                }
                                flush_needed++;
                        }
                } else { /* File could be opened */
                        int distance;

                        flush_needed = 0;
                        distance = check_if_file_contains_tm(gzfh, buffer, tm_start,&local_err);
                        gzclose(gzfh);
                        DEBUG(OUTPUT_PREFIX_JSONRPC_CB_TOPPS "DEBUG distance = '%d' (%s:%d)", distance, __FILE__, __LINE__);
                        DEBUG(OUTPUT_PREFIX_JSONRPC_CB_TOPPS "DEBUG best_distance was = '%d' (%s:%d)", best_distance, __FILE__, __LINE__);
                        if(0 == local_err) { /* ignore this file if something wrong happened */
                                int adistance = abs(distance);
                                /* Check if file found */
                                if(0 == distance) {
                                        best_distance = distance;
                                        best_n = n;
                                        best_tm = tm;
                                        file_found = 1;
                                        break;
                                }
                                /* Check if we found a better file */
                                if(adistance <= best_distance) {
                                        if(
                                                        ((distance < 0) && (tm_start <= tm_end))
                                                        ||
                                                        ((distance > 0) && (tm_start >= tm_end))
                                          ) {
                                                best_distance = adistance;
                                                best_n = n;
                                                best_tm = tm;
                                                if(adistance < max_distance) file_found = 1;

                                        }
                                }
                                search_direction = distance;
                        } /* 0 == local_err */
                        DEBUG(OUTPUT_PREFIX_JSONRPC_CB_TOPPS "DEBUG best_distance is now = '%d' (file found : %d)(%s:%d)", best_distance, file_found, __FILE__, __LINE__);
                        DEBUG(OUTPUT_PREFIX_JSONRPC_CB_TOPPS "DEBUG best_tm/n = '%d/%d' (%s:%d)", best_tm, best_n, __FILE__, __LINE__);
                } /* NULL != gzfh */

                /* Move to next file and check if we should
                 * leave.
                 */
                DEBUG(OUTPUT_PREFIX_JSONRPC_CB_TOPPS "DEBUG search_direction = '%d' (%s:%d)", search_direction, __FILE__, __LINE__);
                if(search_direction > 0) {
                        if(NULL == gzfh) {
                                n = 0;
                                tm += 10000;
                        } else {
                                last_seen_tm_low = tm;
                                last_seen_n_low = n;
                                n += 1;
                        }
                        if(last_seen_tm_high) {
                                if((tm >= last_seen_tm_high) && (n >= last_seen_n_high)) {
                                        break; /* already been there or after */
                                }
                        }
                } else { /* search_direction < 0 */
                        if(NULL != gzfh) {
                                last_seen_tm_high = tm;
                                last_seen_n_high = n;
                        }
                        n = 0;
                        tm -= 10000;
                        if(last_seen_tm_low) {
                                if((tm <= last_seen_tm_low) && (n <= last_seen_n_low)) {
                                        break; /* already been there or before */
                                }
                        }
                }
                DEBUG(OUTPUT_PREFIX_JSONRPC_CB_TOPPS "DEBUG fenetre tm/n = '%d/%d','%d/%d' (%s:%d)", last_seen_tm_low,last_seen_n_low,last_seen_tm_high,last_seen_n_high, __FILE__, __LINE__);
                if(tm_start <= tm_end) { /* When searching forward */
                        if((tm > tm_end) && (
                                                !((0 == flush_already_done) && (2 == flush_needed)) /* Do not break if flush needed */
                                            )) break;
                        /* There should be no reason to search and limit in the past */
                } else { /* When searching backward */
                        if((tm > (tm_start + 10000)) && (
                                                !((0 == flush_already_done) && (2 == flush_needed)) /* Do not break if flush needed */

                                                )) break; /* Going too far in the future (or recent past) */
                        if(tm < (tm_end - 86400)) break; /* Going too far in the past */
                        /* Note : a big old file could contain the data we are
                         * looking for. However, the user should not keep more
                         * than 1 day of data in memory for each hosts. This
                         * is not optimal and is dangerous for the data.
                         */
                }
        }
        if(watchdog >= WATCHDOGMAX) {
                ERROR (OUTPUT_PREFIX_JSONRPC_CB_TOPPS "Infinite loop in %s:%d. hostname='%s', tm=%d, tm_end=%d", __FILE__, __LINE__, hostname, tm_start, tm_end);
                return(JSONRPC_ERROR_CODE_32603_INTERNAL_ERROR);
        }
        DEBUG(OUTPUT_PREFIX_JSONRPC_CB_TOPPS "DEBUG file_found = '%d' (%s:%d)", file_found, __FILE__, __LINE__);
        if(file_found) {
                DEBUG(OUTPUT_PREFIX_JSONRPC_CB_TOPPS "DEBUG filename = '%s' (%s:%d)", buffer, __FILE__, __LINE__);
                if(mkpath_by_tm_and_num(buffer + offset, bufferlen - offset,best_tm, best_n)) {
                        return(JSONRPC_ERROR_CODE_32603_INTERNAL_ERROR);
                }
                DEBUG(OUTPUT_PREFIX_JSONRPC_CB_TOPPS "DEBUG filename = '%s' (%s:%d)", buffer, __FILE__, __LINE__);
        } else {
                buffer[0] = '\0';
        }

        return(0);
} /* }}} check_path */
Example #25
0
boolean TxCache::save(const char *path, const char *filename, int config)
{
    if (!_cache.empty())
    {
        CPath(path, "").DirectoryCreate();

        gzFile gzfp = gzopen(CPath(path, filename), "wb1");
        DBG_INFO(80, L"gzfp:%x file:%ls\n", gzfp, filename);
        if (gzfp)
        {
            /* write header to determine config match */
            gzwrite(gzfp, &config, 4);

            std::map<uint64, TXCACHE*>::iterator itMap = _cache.begin();
            while (itMap != _cache.end())
            {
                uint8 *dest = (*itMap).second->info.data;
                uint32 destLen = (*itMap).second->size;
                uint16 format = (*itMap).second->info.format;

                /* to keep things simple, we save the texture data in a zlib uncompressed state. */
                /* sigh... for those who cannot wait the extra few seconds. changed to keep
                 * texture data in a zlib compressed state. if the GZ_TEXCACHE or GZ_HIRESTEXCACHE
                 * option is toggled, the cache will need to be rebuilt.
                 */
                /*if (format & GR_TEXFMT_GZ) {
                  dest = _gzdest0;
                  destLen = _gzdestLen;
                  if (dest && destLen) {
                  if (uncompress(dest, &destLen, (*itMap).second->info.data, (*itMap).second->size) != Z_OK) {
                  dest = NULL;
                  destLen = 0;
                  }
                  format &= ~GR_TEXFMT_GZ;
                  }
                  }*/

                if (dest && destLen)
                {
                    /* texture checksum */
                    gzwrite(gzfp, &((*itMap).first), 8);

                    /* other texture info */
                    gzwrite(gzfp, &((*itMap).second->info.width), 4);
                    gzwrite(gzfp, &((*itMap).second->info.height), 4);
                    gzwrite(gzfp, &format, 2);

                    gzwrite(gzfp, &((*itMap).second->info.smallLodLog2), 4);
                    gzwrite(gzfp, &((*itMap).second->info.largeLodLog2), 4);
                    gzwrite(gzfp, &((*itMap).second->info.aspectRatioLog2), 4);

                    gzwrite(gzfp, &((*itMap).second->info.tiles), 4);
                    gzwrite(gzfp, &((*itMap).second->info.untiled_width), 4);
                    gzwrite(gzfp, &((*itMap).second->info.untiled_height), 4);

                    gzwrite(gzfp, &((*itMap).second->info.is_hires_tex), 1);

                    gzwrite(gzfp, &destLen, 4);
                    gzwrite(gzfp, dest, destLen);
                }

                itMap++;

                /* not ready yet */
                /*if (_callback)
                  (*_callback)(L"Total textures saved to HDD: %d\n", std::distance(itMap, _cache.begin()));*/
            }
            gzclose(gzfp);
        }
    }
    return _cache.empty();
}
static struct json_object *read_top_ps_file(const char *filename, int tm, short take_next, time_t *data_tm, int *err) /* {{{ */
{
        /* 
         * Return values :
         *   returned value : json array with the result if success. NULL otherwise.
         *   data_tm        : exact tm to search
         *   err            : not nul if an error occured.
         *
         * If returned value is not nul, it is the json array with the result. data_tm
         * contains the tm of the data found.
         * If the returned value is nul, check if err is nul or not.
         *   If err is nul, data_tm is set to the tm to search. Call again with this
         *   value.
         *   If err is not nul, an error occured.
         */
        gzFile *gzfh=NULL;
        int errnum;
        char line[4096];
        size_t l;
        struct json_object *top_ps_array = NULL;

        DEBUG(OUTPUT_PREFIX_JSONRPC_CB_TOPPS "DEBUG Trying to open '%s' (%s:%d)", filename, __FILE__, __LINE__);
        *data_tm = 0;
        *err = 0;
        if(NULL == (gzfh = gzopen(filename, "r"))) {
                *err = 1;
                ERROR (OUTPUT_PREFIX_JSONRPC_CB_TOPPS "'%s' : Could not gzopen for reading (%s:%d)", filename, __FILE__, __LINE__);
                return(NULL);
        }
        /* Read version */
        if(NULL == gzgets(gzfh, line, sizeof(line))) {
                gzclose(gzfh);
                *err = 1;
                ERROR (OUTPUT_PREFIX_JSONRPC_CB_TOPPS "'%s' : Could not read a line (%s:%d)", filename, __FILE__, __LINE__);
                return(NULL);
        }
        for( l = strlen(line) -1 ; l>0; l--) {
                if(line[l] == '\n') line[l] = '\0';
                else if(line[l] == '\r') line[l] = '\0';
                else break;
        }
        if(!strcmp(line, "Version 1.0")) {
                time_t tm_current, tm_prev, tm_last;
                enum { top_ps_state_tm, top_ps_state_nb_lines, top_ps_state_line } state;
                long n;
                long nb_lines;
                short record_lines = 0;
                short record_last = 0;
                /* Read 2nd line : last tm */
                if(NULL == gzgets(gzfh, line, sizeof(line))) {
                        gzclose(gzfh);
                        *err = 1;
                        ERROR (OUTPUT_PREFIX_JSONRPC_CB_TOPPS "'%s' : Could not read a line (%s:%d)", filename, __FILE__, __LINE__);
                        return(NULL);
                }
                /* Check if the last one is the one we want.
                 * If yes, optimize and remember that when we reach it, we
                 * record it.
                 */
                tm_last = strtol(line, NULL, 10);
                if(0 != errno) {
                        gzclose(gzfh);
                        *err = 1;
                        ERROR (OUTPUT_PREFIX_JSONRPC_CB_TOPPS "'%s' : Could not convert '%s' to integer (%s:%d)", filename, line, __FILE__, __LINE__);
                        return(NULL);
                }
                if((0 == take_next) && (tm > tm_last)) {
                        record_last = 1;
                }


                state = top_ps_state_tm;
                tm_current = 0;
                tm_prev = 0;
                nb_lines = 0;
                n = 0;
                while(
                                ((record_lines != 0) || (NULL == top_ps_array)) && 
                                (NULL != gzgets(gzfh, line, sizeof(line)))
                     ) {
                        json_object *json_string;

                        switch(state) {
                                case top_ps_state_tm :
                                        errno=0;
                                        tm_prev = tm_current;
                                        tm_current = strtol(line, NULL, 10);
                                        if(0 != errno) {
                                                gzclose(gzfh);
                                                *err = 1;
                                                ERROR (OUTPUT_PREFIX_JSONRPC_CB_TOPPS "'%s' : Could not convert '%s' to integer (%s:%d)", filename, line, __FILE__, __LINE__);
                                                return(NULL);
                                        }
                                        if(tm_current == tm) {
                                                /* We fould the one we are looking for.
                                                 * Start recording. */
                                                *data_tm = tm_current;
                                                record_lines = 1;
                                        } else if((tm_current == tm_last) && (record_last)) {
                                                *data_tm = tm_current;
                                                record_lines = 1;
                                        } else if(take_next && (tm > tm_prev) && (tm < tm_current)) {
                                                /* The one we are looking for does not exist. The one
                                                 * starting now is the best we can find.
                                                 * Start recording. */
                                                *data_tm = tm_current;
                                                record_lines = 1;
                                        } else if((0 == take_next) && (tm_current > tm)) {
                                                /* We wanted the previous one and we just missed it */
                                                gzclose(gzfh);
                                                if(tm_prev) {
                                                        *data_tm = tm_prev;
                                                        *err = 0; /* no error : try again with exact tm */
                                                        return(NULL);
                                                } else {
                                                        /* this one is not the one we want. And there is no
                                                         * previous one. Error. */
                                                        *err = 1;
                                                        ERROR (OUTPUT_PREFIX_JSONRPC_CB_TOPPS "'%s' : Could not find '%d' before '%ld' (%s:%d)", filename, tm, tm_current, __FILE__, __LINE__);
                                                        return(NULL);
                                                }
                                        }
                                        state = top_ps_state_nb_lines;
                                        break;
                                case top_ps_state_nb_lines :
                                        errno=0;
                                        nb_lines = strtol(line, NULL, 10);
                                        if(0 != errno) {
                                                gzclose(gzfh);
                                                *err = 1;
                                                ERROR (OUTPUT_PREFIX_JSONRPC_CB_TOPPS "'%s' : Could not convert '%s' to integer (%s:%d)", filename, line, __FILE__, __LINE__);
                                                return(NULL);
                                        }
                                        n = 0;
                                        state = top_ps_state_line;
                                        break;
                                case top_ps_state_line :
                                        if(record_lines) {
                                                /* record the line */
                                                if(NULL == top_ps_array) {
                                                        if(NULL == (top_ps_array = json_object_new_array())) {
                                                                gzclose(gzfh);
                                                                *err = 1;
                                                                ERROR (OUTPUT_PREFIX_JSONRPC_CB_TOPPS "'%s' : Could not create a new JSON array (%s:%d)", filename, __FILE__, __LINE__);
                                                                return(NULL);
                                                        }
                                                }
                                                /* Remove CR and LF at the end of the line */
                                                l = strlen(line) - 1;
                                                while(l > 0 && ((line[l] == '\r' ) || (line[l] == '\r' ))) {
                                                        line[l] = '\0';
                                                        l -= 1;
                                                }
                                                if(NULL == (json_string = json_object_new_string(line))) {
                                                        json_object_put(top_ps_array);
                                                        gzclose(gzfh);
                                                        *err = 1;
                                                        ERROR (OUTPUT_PREFIX_JSONRPC_CB_TOPPS "'%s' : Could not create a new JSON string (%s:%d)", filename, __FILE__, __LINE__);
                                                        return(NULL);
                                                }
                                                json_object_array_add(top_ps_array,json_string);
                                        }
                                        n++;
                                        if(n >= nb_lines) {
                                                state = top_ps_state_tm;
                                                record_lines = 0; /* End recoding */
                                        }
                                        break;
                        }
                }
                gzerror(gzfh, &errnum);
                gzclose(gzfh);
                if(errnum < 0) {
                        *err = 1;
                        ERROR (OUTPUT_PREFIX_JSONRPC_CB_TOPPS "'%s' : Could not read a line (%s:%d)", filename, __FILE__, __LINE__);
                        return(NULL);
                }
        }
        if(NULL == top_ps_array) {
                ERROR (OUTPUT_PREFIX_JSONRPC_CB_TOPPS "'%s' : Could not find '%d' before the end of the file (%s:%d)", filename, tm, __FILE__, __LINE__);
                return(NULL);
        }

        return(top_ps_array);
} /* }}} read_top_ps_file */
Example #27
0
static FILE* logs_open_file(char *path, int ff) {
	char fullname[PATH_MAX];
#ifdef HAVE_ZLIB
	int zlibmode = 0;
#endif
	if (ff != LOG_FORMAT_IRSSI && ff != LOG_FORMAT_SIMPLE && ff != LOG_FORMAT_XML && ff != LOG_FORMAT_RAW) {
		if (ff == LOG_FORMAT_NONE)
			debug("[logs] opening log file %s with ff == LOG_FORMAT_NONE CANCELLED\n", __(path), ff);
		else	debug("[logs] opening log file %s with ff == %d CANCELED\n", __(path), ff);
		return NULL;
	}

	debug("[logs] opening log file %s ff:%d\n", __(path), ff);

	if (!path) {
		errno = EACCES; /* = 0 ? */
		return NULL;
	}

	{	/* check if such file was already open SLOW :( */
		list_t l;

		for (l=log_logs; l; l = l->next) {
			logs_log_t *ll = l->data;
			log_window_t *lw;

			if (!ll || !(lw = ll->lw))
				continue;

/*			debug_error("here: %x [%s, %s] [%d %d]\n", lw->file, lw->path, path, lw->logformat, ff); */

			if (lw->file && lw->logformat == ff && !xstrcmp(lw->path, path)) {
				FILE *f = lw->file;
				lw->file = NULL;	/* simulate fclose() on this */
				return f;		/* simulate fopen() here */
			}
		}
	}

	if (mkdir_recursive(path, 0)) {
		print("directory_cant_create", path, strerror(errno));
		return NULL;
	}

	strlcpy(fullname, path, PATH_MAX);

	if (ff == LOG_FORMAT_IRSSI)		strlcat(fullname, ".log", PATH_MAX);
	else if (ff == LOG_FORMAT_SIMPLE)	strlcat(fullname, ".txt", PATH_MAX);
	else if (ff == LOG_FORMAT_XML)		strlcat(fullname, ".xml", PATH_MAX);
	else if (ff == LOG_FORMAT_RAW)		strlcat(fullname, ".raw", PATH_MAX);

#ifdef HAVE_ZLIB /* z log.c i starego ekg1. Wypadaloby zaimplementowac... */
	/* nawet je¶li chcemy gzipowane logi, a istnieje nieskompresowany log,
	 * olewamy kompresjê. je¶li loga nieskompresowanego nie ma, dodajemy
	 * rozszerzenie .gz i balujemy. */
	if (config_log & 4) {
		struct stat st;
		if (stat(fullname, &st) == -1) {
			gzFile f;

			if (!(f = gzopen(path, "a")))
				return NULL;

			gzputs(f, buf);
			gzclose(f);

			zlibmode = 1;
		}
	}
	if (zlibmode) {
		/* XXX, ustawic jakas flage... */
		strlcat(fullname, ".gz", PATH_MAX);
	}
#endif

	/* if xml, prepare xml file */
	if (ff == LOG_FORMAT_XML) {
		FILE *fdesc = fopen(fullname, "r+");
		if (!fdesc) {
			if (!(fdesc = fopen(fullname, "w+")))
				return NULL;
			fputs("<?xml version=\"1.0\"?>\n", fdesc);
			fputs("<!DOCTYPE ekg2log PUBLIC \"-//ekg2log//DTD ekg2log 1.0//EN\" ", fdesc);
			fputs("\"http://www.ekg2.org/DTD/ekg2log.dtd\">\n", fdesc);
			fputs("<ekg2log xmlns=\"http://www.ekg2.org/DTD/\">\n", fdesc);
			fputs("</ekg2log>\n", fdesc);
		} 
		return fdesc;
	}

	return fopen(fullname, "a+");
}
void read_trace_file(void)
{
  g_stream = gzopen((KNOB(KNOB_TRACE_FILE)->getValue()).c_str(), "r");
}
Example #29
0
/* Compress `src' into `dest' using gzip.  */
static int compress_with_gzip(const char *src, const char *dest)
{
#ifdef HAVE_ZLIB
    FILE *fdsrc;
    gzFile fddest;
    size_t len;

    fdsrc = fopen(dest, MODE_READ);
    if (fdsrc == NULL) {
        return -1;
    }

    fddest = gzopen(src, MODE_WRITE "9");
    if (fddest == NULL) {
        fclose(fdsrc);
        return -1;
    }

    do {
        char buf[256];
        len = fread((void *)buf, 256, 1, fdsrc);
        if (len > 0) {
            gzwrite(fddest, (void *)buf, (unsigned int)len);
        }
    } while (len > 0);

    gzclose(fddest);
    fclose(fdsrc);

    archdep_file_set_gzip(dest);

    ZDEBUG(("compress with zlib: OK."));

    return 0;
#else
    static char *argv[4];
    int exit_status;
    char *mdest;

    /* `exec*()' does not want these to be constant...  */
    argv[0] = lib_stralloc("gzip");
    argv[1] = lib_stralloc("-c");
    argv[2] = lib_stralloc(src);
    argv[3] = NULL;

    mdest = lib_stralloc(dest);

    ZDEBUG(("compress_with_gzip: spawning gzip -c %s", src));
    exit_status = archdep_spawn("gzip", argv, &mdest, NULL);

    lib_free(mdest);

    lib_free(argv[0]);
    lib_free(argv[1]);
    lib_free(argv[2]);

    if (exit_status == 0) {
        ZDEBUG(("compress_with_gzip: OK."));
        return 0;
    } else {
        ZDEBUG(("compress_with_gzip: failed."));
        return -1;
    }
#endif
}
Example #30
0
/*
    Load a normal file, or ZIP/GZ archive.
    Returns NULL if an error occured.
*/
uint8 *load_archive(char *filename, int *file_size)
{
    int size = 0;
    uint8 *buf = NULL;

    if(check_zip(filename))
    {
        unzFile *fd = NULL;
        unz_file_info info;
        int ret = 0;

        /* Attempt to open the archive */
        fd = unzOpen(filename);
        if(!fd) return (NULL);

        /* Go to first file in archive */
        ret = unzGoToFirstFile(fd);
        if(ret != UNZ_OK)
        {
            unzClose(fd);
            return (NULL);
        }

        ret = unzGetCurrentFileInfo(fd, &info, NULL, 0, NULL, 0, NULL, 0);
        if(ret != UNZ_OK)
        {
            unzClose(fd);
            return (NULL);
        }

        /* Open the file for reading */
        ret = unzOpenCurrentFile(fd);
        if(ret != UNZ_OK)
        {
            unzClose(fd);
            return (NULL);
        }

        /* Allocate file data buffer */
        size = info.uncompressed_size;
        buf = malloc(size);
        if(!buf)
        {
            unzClose(fd);
            return (NULL);
        }

        /* Read (decompress) the file */
        ret = unzReadCurrentFile(fd, buf, info.uncompressed_size);
        if(ret != info.uncompressed_size)
        {
            free(buf);
            unzCloseCurrentFile(fd);
            unzClose(fd);
            return (NULL);
        }

        /* Close the current file */
        ret = unzCloseCurrentFile(fd);
        if(ret != UNZ_OK)
        {
            free(buf);
            unzClose(fd);
            return (NULL);
        }

        /* Close the archive */
        ret = unzClose(fd);
        if(ret != UNZ_OK)
        {
            free(buf);
            return (NULL);
        }

        /* Update file size and return pointer to file data */
        *file_size = size;
        return (buf);
    }
    else
    {
        gzFile *gd = NULL;

        /* Open file */
        gd = gzopen(filename, "rb");
        if(!gd) return (0);

        /* Get file size */
        size = gzsize(gd);

        /* Allocate file data buffer */
        buf = malloc(size);
        if(!buf)
        {
            gzclose(gd);
            return (0);
        }

        /* Read file data */
        gzread(gd, buf, size);

        /* Close file */
        gzclose(gd);

        /* Update file size and return pointer to file data */
        *file_size = size;
        return (buf);
    }
}