Exemplo n.º 1
0
int
main(
     int argc,
     char **argv
     )
{
  int status = 0;
  char *buffer = NULL;
  char filename[256];
  size_t bufsize = BUFSIZE;
  FILE *ifp = NULL, *ofp = NULL, *tfp = NULL;
  char *infile = NULL, *outfile = NULL, *tempfile = NULL;
  int nr, lno;

  if ( argc != 3 ) { 
    fprintf(stderr, "Usage is txt_sub <infile> <outfile> \n");
    go_BYE(-1);
  }
  infile = argv[1];
  outfile = argv[2];
  if ( *infile == '\0' ) { go_BYE(-1); }
  if ( *outfile == '\0' ) { go_BYE(-1); }
  if ( strcmp(infile, outfile)== 0 ) { go_BYE(-1); }

  buffer = (char *)malloc(bufsize * sizeof(char));
  return_if_malloc_failed(buffer);
  zero_string(buffer, bufsize);
  zero_string(filename, 256);

  ifp = fopen(infile, "r");
  return_if_fopen_failed(ifp, infile, "r");
  ofp = fopen(outfile, "w");
  return_if_fopen_failed(ofp, outfile, "w");

  for ( lno = 1; ; lno++ ) {
    zero_string_to_nullc(buffer);
    nr = getline(&buffer, &bufsize, ifp);
    if ( nr <= 0 ) { break; } 
    if ( ( strcasecmp(buffer, "%% INCLUDE") == 0 ) || 
         ( strcasecmp(buffer, "// INCLUDE") == 0 ) ) {
      status = open_temp_file(&tfp, &tempfile); cBYE(status);
      fclose_if_non_null(tfp);
      fprintf(ofp, "\\verbatiminput{%s}\n", tempfile);
      free_if_non_null(tempfile);
    }
    else { // copy from input to output 
      fprintf(ofp, "%s", buffer); 
     }
  }
  fclose_if_non_null(ifp);
  fclose_if_non_null(ofp);
 BYE:
  free_if_non_null(buffer);
  return(status);
}
Exemplo n.º 2
0
off_t oggedit_write_opus_metadata(DB_FILE *in, const char *fname, const off_t offset, const off_t stream_size, const int output_gain, const uint32_t num_tags, char **tags)
{
    off_t res;
    char tempname[PATH_MAX] = "";
    char *vendor = NULL;
    ogg_sync_state oy;
    ogg_sync_init(&oy);

    /* Original file must be writable whichever way we update it */
    FILE *out = fopen(fname, "r+b");
    if (!out) {
        res = OGGEDIT_CANNOT_UPDATE_FILE;
        goto cleanup;
    }

    /* Should we write the tags packet directly into the existing file ... */
    const long tags_packet_size = check_opus_header(in, &oy, offset, &vendor);
    if (tags_packet_size <= OGGEDIT_EOF) {
        res = tags_packet_size;
        goto cleanup;
    }
    const int64_t metadata_size = strlen(TAGMAGIC) + vc_size(vendor, num_tags, tags);
    int64_t padding = tags_packet_size - metadata_size;
    const off_t file_size_k = in->vfs->getlength(in) / 1000;
    const off_t stream_size_k = stream_size ? stream_size / 1000 : file_size_k;
    if (file_size_k < 100 || padding < 0 || padding > file_size_k/10+stream_size_k+metadata_size) {
        res = open_temp_file(fname, tempname, &out);
        if (res) {
            goto cleanup;
        }
    }

    /* Re-pad if writing the whole file */
    if (*tempname)
        padding = stream_size_k < 90 ? 0 : stream_size_k < 1000 ? 128 : stream_size_k < 10000 ? 1024 : 8192;

    /* Write pages until we reach the correct OpusHead, then write OpusTags */
    ogg_page og;
    int64_t opus_serial = copy_up_to_codec(in, out, &oy, &og, *tempname ? 0 : offset, offset, OPUSNAME);
    if (opus_serial <= OGGEDIT_EOF) {
        res = opus_serial;
        goto cleanup;
    }
    if (output_gain > INT_MIN) {
        og.body[16] = output_gain & 0xFF;
        og.body[17] = output_gain >> 8 & 0xFF;
        ogg_page_checksum_set(&og);
    }
Exemplo n.º 3
0
FileHolder Database_ResourceStorage::get_file_path(const std::string& key)
{
	int ind = find_in_cache(key);
	if(ind >= 0)
		return FileCache[ind].file;

	guint32 entry_offset, entry_size;
	if(!ridx_file->lookup(key.c_str(), entry_offset, entry_size))
		return FileHolder(); // key not found
	gchar *data = dict->GetData(entry_offset, entry_size);
	if(!data)
		return FileHolder();

	std::string name_pattern; // in file name encoding
	if(!utf8_to_file_name(key, name_pattern))
		return FileHolder();
	std::string::size_type pos = name_pattern.find_last_of("."DB_DIR_SEPARATOR_S);
	if(pos != std::string::npos) {
		if(name_pattern[pos] == '.')
			name_pattern = name_pattern.substr(pos);
		else
			name_pattern.clear();
	} else
		name_pattern.clear();
	gint fd;
	FileHolder file(open_temp_file(name_pattern, fd));
	if(file.empty())
		return file;
	ssize_t write_size;
#ifdef _WIN32
	write_size = _write(fd, data+sizeof(guint32), entry_size);
	if (write_size == -1) {
		g_print("write error!\n");
	}
	_close(fd);
#else
	write_size = write(fd, data+sizeof(guint32), entry_size);
	if (write_size == -1) {
		g_print("write error!\n");
	}
	close(fd);
#endif
	ind = put_in_cache(key, file);
	return FileCache[ind].file;
}
Exemplo n.º 4
0
void csync_rs_sig(const char *filename)
{
	FILE *basis_file = 0, *sig_file = 0;
	rs_stats_t stats;
	rs_result result;
	char tmpfname[MAXPATHLEN];

	csync_debug(3, "Csync2 / Librsync: csync_rs_sig('%s')\n", filename);

	csync_debug(3, "Opening basis_file and sig_file..\n");

	sig_file = open_temp_file(tmpfname, prefixsubst(filename));
	if ( !sig_file ) goto io_error;
	if (unlink(tmpfname) < 0) goto io_error;

	basis_file = fopen(prefixsubst(filename), "rb");
	if ( !basis_file ) basis_file = fopen("/dev/null", "rb");

	csync_debug(3, "Running rs_sig_file() from librsync..\n");
	result = rs_sig_file(basis_file, sig_file,
			RS_DEFAULT_BLOCK_LEN, RS_DEFAULT_STRONG_LEN, &stats);
	if (result != RS_DONE)
		csync_fatal("Got an error from librsync, too bad!\n");

	csync_debug(3, "Sending sig_file to peer..\n");
	csync_send_file(sig_file);

	csync_debug(3, "Signature has been created successfully.\n");
	fclose(basis_file);
	fclose(sig_file);

	return;

io_error:
	csync_debug(0, "I/O Error '%s' in rsync-sig: %s\n",
			strerror(errno), prefixsubst(filename));

	if (basis_file) fclose(basis_file);
	if (sig_file) fclose(sig_file);
}
Exemplo n.º 5
0
// START FUNC DECL
int
mk_temp_file(
    char *opfile,
    size_t filesz
    )
// STOP FUNC DECL
{
  int status = 0;
  char cwd[MAX_LEN_DIR_NAME+1];

  if ( g_data_dir == NULL ) { go_BYE(-1); }
  zero_string(cwd, MAX_LEN_DIR_NAME+1);
  getcwd(cwd, MAX_LEN_DIR_NAME);
  if ( strlen(cwd) == 0 ) { go_BYE(-1); }

  status = open_temp_file(opfile, filesz); cBYE(status);
  status = chdir(g_data_dir); cBYE(status);
  status = mk_file(opfile, filesz); cBYE(status);
  status = chdir(cwd); cBYE(status);
BYE:
  return(status);
}
Exemplo n.º 6
0
//---------------------------------------------------------------
// START FUNC DECL
int 
parsort1(
       char *tbl,
       char *f1,
       char *f2,
       char *up_or_down /* not used right now */
       )
// STOP FUNC DECL
{
  int status = 0;
  char *f1_X = NULL; size_t f1_nX = 0;
  char *op_X = NULL; size_t op_nX = 0;
  char *cnt_X = NULL; size_t cnt_nX = 0;
  char *t2f2_X = NULL; size_t t2f2_nX = 0;
  FLD_TYPE *f1_meta = NULL; 
  FLD_TYPE *f2_meta = NULL; 
  FLD_TYPE *t2f2_meta = NULL; 
  FLD_TYPE *cnt_meta = NULL; 
  long long nR, nR2;
  int tbl_id = INT_MIN, f1_id = INT_MIN, f2_id = INT_MIN, cnt_id = INT_MIN;
  int t2f2_id = INT_MIN;
  char str_meta_data[1024];
  FILE *ofp = NULL; char *opfile = NULL;
  FILE *tfp = NULL; char *tempfile = NULL;
  char str_rslt[32]; zero_string(str_rslt, 32);
  char t2[MAX_LEN_TBL_NAME]; 
  int itemp;
  int *xxx = NULL, *f1lb = NULL, *f1ub = NULL; 
  long long *count = NULL, *chk_count = NULL;
  int **offsets = NULL, **bak_offsets = NULL;
  int *inptr = NULL;
  // For multi-threading 
  int rc; // result code for thread create 
  pthread_t threads[MAX_NUM_THREADS];
  pthread_attr_t attr;
  void *thread_status;
  // START: For timing
  struct timeval Tps;
  struct timezone Tpf;
  void *Tzp = NULL;
  long long t_before_sec = 0, t_before_usec = 0, t_before = 0;
  long long t_after_sec, t_after_usec, t_after;
  long long t_delta_usec;
  // STOP : For timing
  //----------------------------------------------------------------
  if ( ( tbl == NULL ) || ( *tbl == '\0' ) ) { go_BYE(-1); }
  if ( ( f1 == NULL ) || ( *f1 == '\0' ) ) { go_BYE(-1); }
  if ( ( f2 == NULL ) || ( *f2 == '\0' ) ) { go_BYE(-1); }
  zero_string(str_meta_data, 1024);
  /* t2 isa temporary table */
  zero_string(t2, MAX_LEN_TBL_NAME);
  status = qd_uq_str(t2, MAX_LEN_TBL_NAME);
  strcpy(t2, "t2"); // TODO DELETE THIS 
  g_offsets = NULL;
  g_count = NULL;
  //--------------------------------------------------------
  status = is_tbl(tbl, &tbl_id); cBYE(status);
  chk_range(tbl_id, 0, g_n_tbl);
  nR = g_tbl[tbl_id].nR;
  status = is_fld(NULL, tbl_id, f1, &f1_id); cBYE(status);
  chk_range(f1_id, 0, g_n_fld);
  f1_meta = &(g_fld[f1_id]);
  status = rs_mmap(f1_meta->filename, &f1_X, &f1_nX, 0); 
  cBYE(status);
  // Not implemented for following cases 
  if ( g_fld[f1_id].nn_fld_id >= 0 ) { go_BYE(-1); }
  if ( strcmp(f1_meta->fldtype, "int") != 0 ) { go_BYE(-1); }
  if ( nR <= 1048576 ) { go_BYE(-1); }
  //---------------------------------------------
  status = gettimeofday(&Tps, &Tpf); cBYE(status);
  t_before_sec  = (long long)Tps.tv_sec;
  t_before_usec = (long long)Tps.tv_usec;
  t_before = t_before_sec * 1000000 + t_before_usec;

  int reduction_factor = (int)(sqrt((double)nR));
  sprintf(str_rslt, "%d", reduction_factor);
  status = subsample(tbl, f1, str_rslt, t2, "f2"); cBYE(status);

  status = gettimeofday(&Tps, &Tpf); cBYE(status);
  t_after_sec  = (long long)Tps.tv_sec;
  t_after_usec = (long long)Tps.tv_usec;
  t_after = t_after_sec * 1000000 + t_after_usec;
  fprintf(stderr, "TIME0 = %lld \n", t_after - t_before); 
  t_before = t_after;


  // Must have sufficient diversity of values
  status = f1opf2(t2, "f2", "op=shift:val=-1", "nextf2"); cBYE(status);
  status = drop_nn_fld(t2, "nextf2"); cBYE(status);
  status = f1f2opf3(t2, "f2", "nextf2", "==", "x"); cBYE(status);
  status = f_to_s(t2, "x", "sum", str_rslt);
  char *endptr;
  long long lltemp = strtoll(str_rslt, &endptr, 10);
  if ( lltemp != 0 ) { go_BYE(-1); }
  //-------------------------------------------------
  // Get range of values of f1 
  status = f_to_s(tbl, f1, "max", str_rslt);
  int f1max = strtoll(str_rslt, &endptr, 10);
  status = f_to_s(tbl, f1, "min", str_rslt);
  int f1min = strtoll(str_rslt, &endptr, 10);
  //-------------------------------------------------
  // Now we sort the values that we sampled
  status = fop(t2, "f2", "sortA"); cBYE(status);
  // status = pr_fld(t2, "f2", "", stdout);
  status = get_nR(t2, &nR2);
  // Now each thread selects a range to work on
  int nT;
  for ( int i = 0; i < MAX_NUM_THREADS; i++ ) { 
    g_thread_id[i] = i;
  }
  status = get_num_threads(&nT);
  cBYE(status);
  //--------------------------------------------
#define MIN_ROWS_FOR_PARSORT1 1048576
  if ( nR <= MIN_ROWS_FOR_PARSORT1 ) {
    nT = 1;
  }
  /* Don't create more threads than you can use */
  if ( nT > nR ) { nT = nR; }
  //--------------------------------------------

  double block_size = (double)nR2 / (double)nT;
  status = is_fld(t2, -1, "f2", &t2f2_id); cBYE(status);
  chk_range(t2f2_id, 0, g_n_fld);
  t2f2_meta = &(g_fld[t2f2_id]);
  status = rs_mmap(t2f2_meta->filename, &t2f2_X, &t2f2_nX, 0); 
  cBYE(status);
  int *iptr = (int *)t2f2_X;
  xxx = malloc(nT * sizeof(int)); return_if_malloc_failed(xxx);
  f1lb = malloc(nT * sizeof(int)); return_if_malloc_failed(f1lb);
  f1ub = malloc(nT * sizeof(int)); return_if_malloc_failed(f1ub);
  /* FOR OLD_WAY 
  count = malloc(nT * sizeof(long long)); return_if_malloc_failed(count);
  */
  chk_count = malloc(nT * sizeof(long long));
  return_if_malloc_failed(chk_count);
  g_count = malloc(nT * sizeof(long long)); return_if_malloc_failed(g_count);

  for ( int i = 0; i < nT; i++ ) { 
    // FOR OLD_WAY count[i]= 0;
    chk_count[i]= 0;
    int j = i+1;
    long long idx = j * block_size;
    if ( idx >= nR2 ) { idx = nR2 -1 ; }
    int y = iptr[idx];
    xxx[i] = y;
    // fprintf(stdout,"idx = %lld: j = %d: y = %d \n", idx, j, y);
  }
  for ( int i = 0; i < nT; i++ ) { 
    if ( ( i == 0 ) && ( i == (nT - 1 ) ) ) {
      f1lb[i] = f1min;
      f1ub[i] = f1max;
    }
    else if ( i == 0 ) { 
      f1lb[i] = f1min;
      f1ub[i] = xxx[i];
    }
    else if ( i == (nT -1 ) ) {
      f1lb[i] = xxx[i-1] + 1;
      f1ub[i] = f1max;
    }
    else {
      f1lb[i] = xxx[i-1] + 1;
      f1ub[i] = xxx[i];
    }
  }
  // STOP: Each thread has now a range to work on
  // Create a temporary table t3 to store ranges
  char t3[MAX_LEN_TBL_NAME]; int t3_id;
  zero_string(t3, MAX_LEN_TBL_NAME);
  status = qd_uq_str(t3, MAX_LEN_TBL_NAME);
  strcpy(t3, "t3"); // TODO DELETE THIS 
  sprintf(str_rslt, "%d", nT);
  status = add_tbl(t3, str_rslt, &t3_id);

  // Add lower bound to t3
  status = open_temp_file(&tfp, &tempfile, -1); cBYE(status);
  fclose_if_non_null(tfp);
  tfp = fopen(tempfile, "wb"); return_if_fopen_failed(tfp, tempfile, "wb");
  fwrite(f1lb, sizeof(int),  nT, tfp); 
  fclose_if_non_null(tfp);
  sprintf(str_meta_data, "fldtype=%s:n_sizeof=%d:filename=%s",
      f1_meta->fldtype, f1_meta->n_sizeof, tempfile);
  status = add_fld(t3, "lb", str_meta_data, &itemp); cBYE(status);
  free_if_non_null(tempfile);

  // Add upper bound to t3
  status = open_temp_file(&tfp, &tempfile, -1); cBYE(status);
  fclose_if_non_null(tfp);
  tfp = fopen(tempfile, "wb"); return_if_fopen_failed(tfp, tempfile, "wb");
  fwrite(f1ub, sizeof(int),  nT, tfp); 
  fclose_if_non_null(tfp);
  sprintf(str_meta_data, "fldtype=%s:n_sizeof=%d:filename=%s",
      f1_meta->fldtype, f1_meta->n_sizeof, tempfile);
  status = add_fld(t3, "ub", str_meta_data, &itemp); cBYE(status);
  free_if_non_null(tempfile);

#undef OLD_WAY
#ifdef OLD_WAY
  // Now we count how much there is in each range 
  inptr = (int *)f1_X;
  for ( long long i = 0; i < nR; i++ ) { 
    int ival = *inptr++;
    int range_idx = INT_MIN;
    // TODO: Improve sequential search
    for ( int j = 0; j < nT; j++ ) { 
      if ( ival >= f1lb[j] && ( ival <= f1ub[j] ) ) {
	range_idx = j;
	break;
      }
    }
    count[range_idx]++;
  }
  /*
  for ( int i = 0; i < nT; i++ ) { 
    fprintf(stdout,"%d: (%d, %d) = %lld \n", i, f1lb[i], f1ub[i], count[i]);
  }
  */
#else
  status = num_in_range(tbl, f1, t3, "lb", "ub", "cnt"); cBYE(status);
  // Get a pointer to the count field 
  status = is_tbl(t3, &t3_id);
  chk_range(t3_id, 0, g_n_tbl);
  status = is_fld(NULL, t3_id, "cnt", &cnt_id);
  chk_range(cnt_id, 0, g_n_fld);
  cnt_meta = &(g_fld[cnt_id]); 
  status = rs_mmap(cnt_meta->filename, &cnt_X, &cnt_nX, 0); cBYE(status);
  count = (long long *)cnt_X;
#endif
  status = gettimeofday(&Tps, &Tpf); cBYE(status);
  t_after_sec  = (long long)Tps.tv_sec;
  t_after_usec = (long long)Tps.tv_usec;
  t_after = t_after_sec * 1000000 + t_after_usec;
  fprintf(stderr, "TIME1 = %lld \n", t_after - t_before); 
  t_before = t_after;


  bak_offsets = malloc(nT * sizeof(int *)); return_if_malloc_failed(bak_offsets);
  g_offsets = malloc(nT * sizeof(int *)); return_if_malloc_failed(g_offsets);
#ifdef OLD_WAY
  // Make space for output 
  long long filesz = nR * f1_meta->n_sizeof;
  status = open_temp_file(&ofp, &opfile, filesz); cBYE(status);
  status = mk_file(opfile, filesz); cBYE(status);
  status = rs_mmap(opfile, &op_X, &op_nX, 1); cBYE(status);
  offsets = malloc(nT * sizeof(int *)); return_if_malloc_failed(offsets);
  long long cum_count = 0;
  for ( int i = 0; i < nT; i++ ) {
    bak_offsets[i] = offsets[i] = (int *)op_X;
    if ( i > 0 ) {
      cum_count += count[i-1];
      offsets[i] += cum_count;
      bak_offsets[i] = offsets[i];
    }
  }

  inptr = (int *)f1_X;
  // Now we place each item into its thread bucket
  for ( long long i = 0; i < nR; i++ ) { 
    int ival = *inptr++;
    int range_idx = INT_MIN;
    // TODO: Improve sequential search
    for ( int j = 0; j < nT; j++ ) { 
      if ( ival >= f1lb[j] && ( ival <= f1ub[j] ) ) {
	range_idx = j;
	break;
      }
    }
    int *xptr = offsets[range_idx];
    *xptr = ival;
    offsets[range_idx]++;
    chk_count[range_idx]++;
    if ( chk_count[range_idx] > count[range_idx] ) {
      go_BYE(-1);
    }
  }
  cum_count = 0;
  for ( int i = 0; i < nT-1; i++ ) { 
    if ( offsets[i] != bak_offsets[i+1] ) { 
      go_BYE(-1);
    }
  }
#else
  status = mv_range(tbl, f1, f2, t3, "lb", "ub", "cnt"); 
  cBYE(status);
  status = is_fld(NULL, tbl_id, f2, &f2_id);
  chk_range(f2_id, 0, g_n_fld);
  f2_meta = &(g_fld[f2_id]); 
  status = rs_mmap(f2_meta->filename, &op_X, &op_nX, 1); cBYE(status);
#endif

  long long cum_count = 0;
  for ( int i = 0; i < nT; i++ ) {
    bak_offsets[i] = (int *)op_X;
    if ( i > 0 ) {
      cum_count += count[i-1];
      bak_offsets[i] += cum_count;
    }
  }

  status = gettimeofday(&Tps, &Tpf); cBYE(status);
  t_after_sec  = (long long)Tps.tv_sec;
  t_after_usec = (long long)Tps.tv_usec;
  t_after = t_after_sec * 1000000 + t_after_usec;
  fprintf(stderr, "TIME2 = %lld \n", t_after - t_before); 
  t_before = t_after;

  // Set up global variables
  g_nT = nT;
  for ( int i = 0; i < nT; i++ ) { 
    g_offsets[i] = bak_offsets[i];
    g_count[i] = count[i];
  }
  if ( g_nT == 1 ) { 
    core_parsort1(&(g_thread_id[0]));
  }
  else {
    pthread_attr_init(&attr);
    pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
    for ( int t = 0; t < g_nT; t++ ) { 
      rc = pthread_create(&threads[t], NULL, core_parsort1,
	  &(g_thread_id[t]));
      if ( rc ) { go_BYE(-1); }
    }
    /* Free attribute and wait for the other threads */
    pthread_attr_destroy(&attr);
    for ( int t = 0; t < g_nT; t++ ) { 
      rc = pthread_join(threads[t], &thread_status);
      if ( rc ) { go_BYE(-1); }
    }
  }
  /* SEQUENTIAL CODE 
  for ( int i = 0; i < nT; i++ ) { 
    qsort_asc_int(bak_offsets[i], count[i], sizeof(int), NULL);
  }
  */
  status = gettimeofday(&Tps, &Tpf); cBYE(status);
  t_after_sec  = (long long)Tps.tv_sec;
  t_after_usec = (long long)Tps.tv_usec;
  t_after = t_after_sec * 1000000 + t_after_usec;
  fprintf(stderr, "TIME3 = %lld \n", t_after - t_before); 

  // Indicate the dst_fld is sorted ascending
  status = set_fld_info(tbl, f2, "sort=1");


  rs_munmap(op_X, op_nX);
  status = del_tbl(t2, -1); cBYE(status);
  status = del_tbl(t3, -1); cBYE(status);
BYE:
  rs_munmap(op_X, op_nX);
  rs_munmap(cnt_X, cnt_nX);
  free_if_non_null(xxx);
  free_if_non_null(f1lb);
  free_if_non_null(f1ub);
  // Do not delete unless using OLD_WAY free_if_non_null(count);
  free_if_non_null(g_count);
  free_if_non_null(g_offsets);
  free_if_non_null(offsets);
  free_if_non_null(bak_offsets);
  free_if_non_null(chk_count);

  fclose_if_non_null(ofp);
  g_write_to_temp_dir = false;
  rs_munmap(f1_X, f1_nX);
  rs_munmap(op_X, op_nX);
  free_if_non_null(opfile);
  return(status);
}
Exemplo n.º 7
0
// START FUNC DECL
int
read_csv(
	 char *infile,
	 char fld_delim, /* double quote character */
	 char fld_sep, /* comma */
	 char rec_delim, /* new line character */
	 bool ignore_hdr, /* whether to ignore header or not */
	 char flds[MAX_NUM_FLDS][MAX_LEN_FLD_NAME+1],
	 int n_flds,
	 FLD_TYPE *fldtype,
	 bool *is_null_null, /* whether to convert "null" to null */
	 bool *is_load, /* to load or not */
	 bool *is_all_def, /* whether all values are defined */
	 char fnames[MAX_NUM_FLDS][MAX_LEN_FILE_NAME+1],
	 char nn_fnames[MAX_NUM_FLDS][MAX_LEN_FILE_NAME+1],
	 long long *ptr_num_rows
	 )
// STOP FUNC DECL
{
  int status = 0;
  char *X = NULL; size_t nX = 0;
  bool in_fld, is_fld_delim;
  long long num_rows = 0; long long num_cells = 0, fld_idx; 
  long long lb, ub;
  FILE *fps[MAX_NUM_FLDS];
  FILE *nn_fps[MAX_NUM_FLDS];
  bool in_escape_mode = false, is_fld_delim_escaped = false;
  
  g_num_rows = 0;
  g_buflen = 8192;
  g_buffer = (char *)malloc(g_buflen * sizeof(char));
  return_if_malloc_failed(g_buffer);
  zero_string(g_buffer, g_buflen);

  /*----------------------------------------------------------------------------*/
  if ( n_flds <= 0 ) { go_BYE(-1); }
  for ( int i = 0; i < MAX_NUM_FLDS; i++ ) { 
    fps[i] = NULL; 
    nn_fps[i] = NULL; 
  }
  /*----------------------------------------------------------------------------*/
  status = rs_mmap(infile, &X, &nX, 0);
  cBYE(status);
  lb = 0; fld_idx = 0; in_fld = false; is_fld_delim = false;
  for ( int i = 0; i < n_flds; i++ ) { 
    if ( is_load[i] ) {
      status = open_temp_file((fnames[i]), 0);
      cBYE(status);
      fps[i] = fopen(fnames[i], "wb");
      return_if_fopen_failed(fps[i], fnames[i],  "wb");
      if ( is_all_def[i] == false ) { 
        status = open_temp_file(nn_fnames[i], 0);
        cBYE(status);
        nn_fps[i] = fopen(nn_fnames[i], "wb");
        return_if_fopen_failed(nn_fps[i], nn_fnames[i],  "wb");
      }
    }
  }
  for ( long long i = 0; i < nX; i++ ) {
    // START: Error handling for escape conventions
    /* TODO: If I get an incorrect backslash, I simply replace it with a
     * space. This is incorrect. But I need it to read files created 
     * with jspool utility from Jay Thomas. The correct thing to do is
     * to fix the utility.
     */
    if ( fld_sep != '\t' ) {
      if ( in_escape_mode ) { 
	if ( ( X[i] != '\\' ) && ( X[i] != '"' ) ) {
	  fprintf(stderr, "Error. Bad backslash. \n");
	  fprintf(stderr, "num_rows  = %lld \n", num_rows);
	  go_BYE(-1);
	}
      }
    }
    // STOP: Error handling for escape conventions
    // START: Handling bslash and dquote 
    if ( fld_sep != '\t' ) {
      if ( X[i] == '\\' ) { 
	if ( in_escape_mode ) { 
	  in_escape_mode = false;
	}
	else {
	  in_escape_mode = true;
	}
      }
    }
    // STOP: Handling bslash and dquote 
    if ( fld_sep != '\t' ) {
      if ( X[i] == '"' ) { 
	if ( in_escape_mode ) {
	  is_fld_delim_escaped = true;
	  in_escape_mode = false;
	}
      }
    }
    if ( in_fld == false ) {
      in_fld = true;
      if ( X[i] == fld_delim ) { /* consume this character */
	is_fld_delim = true;
        lb = i+1;
      }
      else {
        lb = i; 
	if ( ( ( fld_idx < (n_flds-1) ) && ( X[i] == fld_sep ) ) || 
	     ( ( fld_idx == n_flds-1 ) && ( X[i] == rec_delim ) ) ) {
	  /* field has terminated */
	  ub = i-1;
	  if ( is_load[fld_idx] ) { 
	    if ( ( ignore_hdr ) & ( num_rows == 0 ) ) {
	      /* Do nothing */
	    }
	    else {
	      status = print_cell(X, lb, ub+1, fldtype[fld_idx], 
		  is_null_null[fld_idx], is_all_def[fld_idx], 
		  fps[fld_idx], nn_fps[fld_idx], fld_sep);
	      cBYE(status);
	    }
	  }
	  in_fld = false;
	  num_cells++;
          if ( ( fld_idx == n_flds-1 ) && ( X[i] == rec_delim ) ) {
	    num_rows++;
	    g_num_rows = num_rows;
	  }
	  fld_idx++;
	  if ( fld_idx == n_flds ) { fld_idx = 0; }
	}
      }
    }
    else { /* Now within field. Decide whether to continue or stop. */
      if ( is_fld_delim ) {
	/* if field started with a fld_delim, it must end with one */
	/* Note that we need to take care of escaping the dquote */
	if ( ( X[i] == fld_delim ) && ( !is_fld_delim_escaped ) ) {
	  /* next char must be fld_sep or rec_delim or we have come to eof */
	  if ( i+1 > nX ) { go_BYE(-1); }
	  if ( ( fld_idx < (n_flds-1) ) && ( X[i+1] != fld_sep ) ) { 
	    fprintf(stderr, "num_rows  = %lld \n", num_rows);
	    go_BYE(-1); 
	  }
	  if ( fld_idx == n_flds-1 ) {
	    if ( X[i+1] != rec_delim ) { 
	      go_BYE(-1); 
	    }
	    else {
	      num_rows++;
	      g_num_rows = num_rows;
	    }
	  }
	  ub = i-1;
	  if ( is_load[fld_idx] ) { 
	    if ( ( ignore_hdr ) & ( num_rows == 0 ) ) {
	      /* Do nothing */
	    }
	    else {
	      status = print_cell(X, lb, ub+1, fldtype[fld_idx], 
		  is_null_null[fld_idx], is_all_def[fld_idx], 
		  fps[fld_idx], nn_fps[fld_idx], fld_sep);
	      cBYE(status);
	    }
	  }
	  in_fld = false;
	  is_fld_delim = false;
	  fld_idx++;
	  if ( fld_idx == n_flds ) { fld_idx = 0; }
	  num_cells++;
	  i++; /* Consume next char which must be fld_sep or rec_delim */
	}
      }
      else {
	if ( ( i == nX ) ||  
	     ( ( fld_idx < (n_flds-1) ) && ( X[i] == fld_sep ) ) || 
	     ( ( fld_idx == n_flds-1 ) && ( X[i] == rec_delim ) ) ) {
	  /* field has terminated */
	  if ( i == nX ) { 
	    ub = i;
	  }
	  else {
	    ub = i-1;
	  }
	  if ( is_load[fld_idx] ) { 
	    if ( ( ignore_hdr ) & ( num_rows == 0 ) ) {
	      /* Do nothing */
	    }
	    else {
	      status = print_cell(X, lb, ub+1, fldtype[fld_idx], 
		  is_null_null[fld_idx], is_all_def[fld_idx], 
		  fps[fld_idx], nn_fps[fld_idx], fld_sep);
	      cBYE(status);
	    }
	  }
	  in_fld = false;
	  num_cells++;
          if ( ( fld_idx == n_flds-1 ) && ( X[i] == rec_delim ) ) {
	    num_rows++;
	    g_num_rows = num_rows;
	  }
	  fld_idx++;
	  if ( fld_idx == n_flds ) { fld_idx = 0; }
	}
      }
    }
    is_fld_delim_escaped = false;
  }
  if ( num_cells != ( n_flds * num_rows ) ) { 
    fprintf(stderr, "num_cells = %lld   \n", num_cells);
    fprintf(stderr, "n_flds    = %d   \n", n_flds   );
    fprintf(stderr, "num_rows  = %lld \n", num_rows);
    go_BYE(-1); 
  }
  if ( ignore_hdr ) { 
    num_rows--; // Because you have eliminated first row 
  }
  /* TODO: Copy information into meta data structure */
  for ( int i = 0; i < n_flds; i++ ) { 
    if ( fps != NULL ) { fclose_if_non_null(fps[i]); }
    if ( nn_fps != NULL ) { fclose_if_non_null(nn_fps[i]); }
  }
  *ptr_num_rows = num_rows;
 BYE:
  rs_munmap(X, nX);
  free_if_non_null(g_buffer);
  return(status);
}
Exemplo n.º 8
0
//---------------------------------------------------------------
// START FUNC DECL
int 
crossprod(
	  char *docroot,
	  sqlite3 *in_db,
	  char *t1,
	  char *f1,
	  char *t2,
	  char *f2,
	  char *t3
	  )
// STOP FUNC DECL
{
  int status = 0;
  sqlite3 *db = NULL;

  char *Y1 = NULL; size_t nY1 = 0;
  char *Y2 = NULL; size_t nY2 = 0;

  char *f1_X = NULL; size_t f1_nX = 0;
  char *f2_X = NULL; size_t f2_nX = 0;

  char *nn_f1_X = NULL; size_t nn_f1_nX = 0;
  char *nn_f2_X = NULL; size_t nn_f2_nX = 0;

  int t3_id, itemp; bool b_is_tbl;
  FLD_META_TYPE f1_meta, f2_meta;
  FLD_META_TYPE nn_f1_meta, nn_f2_meta;
  long long nR1, nR2, nn_nR1, nn_nR2, nR3;
  char str_meta_data[1024];
  char *t3f1_opfile = NULL, *t3f2_opfile = NULL;
  FILE *ofp = NULL;
  char buffer[32];
  //----------------------------------------------------------------
  zero_string(str_meta_data, 1024);
  zero_string(buffer, 32);
  zero_fld_meta(&f1_meta);
  zero_fld_meta(&f2_meta);
  zero_fld_meta(&nn_f1_meta);
  zero_fld_meta(&nn_f2_meta);
  status = open_db_if_needed(docroot, in_db, &db);
  cBYE(status);
  //----------------------------------------------------------------
  status = get_data(docroot, db, t1, f1, &nR1, &f1_meta, &f1_X, 
      &f1_nX, &nn_f1_meta, &nn_f1_X, &nn_f1_nX, &nn_nR1);
  cBYE(status);
  status = get_data(docroot, db, t2, f2, &nR2, &f2_meta, &f2_X, 
      &f2_nX, &nn_f2_meta, &nn_f2_X, &nn_f2_nX, &nn_nR2);
  cBYE(status);
  nR3 = nn_nR1 * nn_nR2;
  if ( nR3 == 0 ) {
    fprintf(stderr, "No data to create t3 \n");
    goto BYE;
  }

  status = open_temp_file(&ofp, &t3f1_opfile); cBYE(status);
  fclose_if_non_null(ofp);
  status = mk_file(t3f1_opfile, nR3 * f1_meta.n_sizeof); cBYE(status);
  status = rs_mmap(t3f1_opfile, &Y1, &nY1, 1); cBYE(status);

  status = open_temp_file(&ofp, &t3f2_opfile); cBYE(status);
  fclose_if_non_null(ofp);
  status = mk_file(t3f2_opfile, nR3 * f2_meta.n_sizeof); cBYE(status);
  status = rs_mmap(t3f2_opfile, &Y2, &nY2, 1); cBYE(status);
  //----------------------------------------------------------------
  if ( ( strcmp(f1_meta.fldtype, "int") == 0 ) && 
     ( strcmp(f2_meta.fldtype, "int") == 0 ) ) {
    status = core_crossprod_I_I(f1_X, nn_f1_X, f2_X, nn_f2_X, nR1, nR2, nR3, Y1, Y2);
  }
  else if ( ( strcmp(f1_meta.fldtype, "int") == 0 ) && 
     ( strcmp(f2_meta.fldtype, "long long") == 0 ) ) {
    status = core_crossprod_I_L(f1_X, nn_f1_X, f2_X, nn_f2_X, nR1, nR2, nR3, Y1, Y2);
  }
  else if ( ( strcmp(f1_meta.fldtype, "long long") == 0 ) && 
     ( strcmp(f2_meta.fldtype, "int") == 0 ) ) {
    status = core_crossprod_L_I(f1_X, nn_f1_X, f2_X, nn_f2_X, nR1, nR2, nR3, Y1, Y2);
  }
  else if ( ( strcmp(f1_meta.fldtype, "long long") == 0 ) && 
     ( strcmp(f2_meta.fldtype, "long long") == 0 ) ) {
    status = core_crossprod_L_L(f1_X, nn_f1_X, f2_X, nn_f2_X, nR1, nR2, nR3, Y1, Y2);
  }
  else { go_BYE(-1); }
  cBYE(status);
  //----------------------------------------------------------------
  // Add output fields to t3 meta data 
  status = is_tbl(docroot, db, t3, &b_is_tbl, &t3_id);
  cBYE(status);
  if ( b_is_tbl ) {
    status = del_tbl(docroot, db, t3);
    cBYE(status);
  }
  sprintf(buffer, "%lld", nR3);
  status = add_tbl(docroot, db, t3, buffer, &itemp);
  cBYE(status);

  sprintf(str_meta_data, "fldtype=int:n_sizeof=%u:filename=%s",
	  f1_meta.n_sizeof, t3f1_opfile);
  status = add_fld(docroot, db, t3, f1, str_meta_data);
  cBYE(status);
  zero_string(str_meta_data, 1024);
  sprintf(str_meta_data, "fldtype=int:n_sizeof=%u:filename=%s",
	  f2_meta.n_sizeof, t3f2_opfile);
  status = add_fld(docroot, db, t3, f2, str_meta_data);
  cBYE(status);
 BYE:
  fclose_if_non_null(ofp);
  if ( in_db == NULL ) { sqlite3_close(db); }
  rs_munmap(f1_X, f1_nX);
  rs_munmap(nn_f1_X, nn_f1_nX);
  rs_munmap(f2_X, f2_nX);
  rs_munmap(nn_f2_X, nn_f2_nX);

  rs_munmap(Y1, nY1);
  rs_munmap(Y2, nY2);

  free_if_non_null(t3f1_opfile);
  free_if_non_null(t3f2_opfile);
  return(status);
}
Exemplo n.º 9
0
//---------------------------------------------------------------
// START FUNC DECL
int 
count_vals(
	   char *src_tbl,
	   char *src_fld,
	   char *in_cnt_fld,
	   char *dst_tbl,
	   char *dst_fld,
	   char *out_cnt_fld
	   )
// STOP FUNC DECL
{
  int status = 0;
  char *src_fld_X = NULL; size_t src_fld_nX = 0;
  char *in_cnt_fld_X = NULL; size_t in_cnt_fld_nX = 0;
  FLD_TYPE *src_fld_meta = NULL, *in_cnt_fld_meta = NULL;
  FLD_TYPE *cnt_fld_meta = NULL, *nn_src_fld_meta = NULL;
  long long src_nR; 
  int src_tbl_id = INT_MIN, dst_tbl_id = INT_MIN;
  int src_fld_id = INT_MIN, nn_src_fld_id = INT_MIN;
  int dst_fld_id = INT_MIN;
  int cnt_fld_id = INT_MIN, nn_cnt_fld_id = INT_MIN;
  long long dst_nR, n_out;
  char buffer[32];
  char str_meta_data[256];
  char *nn_src_fld_X = NULL; size_t nn_src_fld_nX = 0;
  char *out_X = NULL; char *cnt_X = NULL; 
  char *opfile = NULL, *tmp_opfile = NULL; FILE *ofp = 0;
  char *cnt_opfile = NULL, *tmp_cnt_opfile = NULL; FILE *cnt_ofp = 0;
  size_t n_out_X = 0, n_cnt_X = 0;
  unsigned int n_sizeof = INT_MAX;
  //----------------------------------------------------------------
  if ( ( src_tbl == NULL ) || ( *src_tbl == '\0' ) ) { go_BYE(-1); }
  if ( ( src_fld == NULL ) || ( *src_fld == '\0' ) ) { go_BYE(-1); }
  if ( ( dst_tbl == NULL ) || ( *dst_tbl == '\0' ) ) { go_BYE(-1); }
  if ( ( dst_fld == NULL ) || ( *dst_fld == '\0' ) ) { go_BYE(-1); }
  if ( strcmp(src_tbl, dst_tbl) == 0 ) { go_BYE(-1); }
  zero_string(str_meta_data, 256);
  zero_string(buffer, 32);
  //--------------------------------------------------------
  status = is_tbl(src_tbl, &src_tbl_id); cBYE(status);
  chk_range(src_tbl_id, 0, g_n_tbl);
  src_nR = g_tbl[src_tbl_id].nR;
  //--------------------------------------------------------
  status = is_fld(NULL, src_tbl_id, src_fld, &src_fld_id); cBYE(status);
  chk_range(src_fld_id, 0, g_n_fld);
  src_fld_meta = &(g_fld[src_fld_id]);
  status = rs_mmap(src_fld_meta->filename, &src_fld_X, &src_fld_nX, 0); cBYE(status);
  nn_src_fld_id = src_fld_meta->nn_fld_id;
  if ( nn_src_fld_id >= 0 ) { 
    nn_src_fld_meta = &(g_fld[nn_src_fld_id]);
    status = rs_mmap(nn_src_fld_meta->filename, &nn_src_fld_X, &nn_src_fld_nX, 0); 
    cBYE(status);
  }
  // Get the count field if specified. It cannot have null values
  if ( *in_cnt_fld != '\0' ) { 
    status = is_fld(NULL, src_tbl_id, src_fld, &src_fld_id); cBYE(status);
    chk_range(src_fld_id, 0, g_n_fld);
    src_fld_meta = &(g_fld[src_fld_id]);
    status = rs_mmap(src_fld_meta->filename, &src_fld_X, &src_fld_nX, 0); cBYE(status);
    nn_cnt_fld_id = cnt_fld_meta->nn_fld_id;
    chk_range(nn_cnt_fld_id, 0, g_n_fld);
  }
  /* Make sure src_fld field is sorted ascending */
  zero_string(buffer, 32);
  status = f_to_s(src_tbl, src_fld, "is_sorted", buffer);
  cBYE(status);
  if ( strcmp(buffer, "ascending") != 0 ) {
    fprintf(stderr, "Field [%s] in Table [%s] not sorted ascending\n",
	    src_fld, src_tbl);
    go_BYE(-1);
  }
  //--------------------------------------------------------
  // Create 2 temporary files to store the results. We allocate space
  // differently based on field types
  g_write_to_temp_dir = true;
  status = open_temp_file(&ofp, &tmp_opfile, 0); cBYE(status);
  fclose_if_non_null(ofp);
  status = open_temp_file(&cnt_ofp, &tmp_cnt_opfile, 0); cBYE(status);
  fclose_if_non_null(cnt_ofp);
  //------------------------------------------------------
  n_out = src_nR * sizeof(int); // upper bound 

  if ( strcmp(src_fld_meta->fldtype, "int") == 0 ) {
    status = mk_file(tmp_cnt_opfile, sizeof(int) * src_nR);
    cBYE(status);
    rs_mmap(tmp_cnt_opfile, &cnt_X, &n_cnt_X, 1);
    cBYE(status);

    status = mk_file(tmp_opfile, sizeof(int) * src_nR);
    cBYE(status);
    rs_mmap(tmp_opfile, &out_X, &n_out_X, 1);
    cBYE(status);
    if ( nn_src_fld_X == NULL ) { 
      if ( *in_cnt_fld == '\0' ) { 
	uniq_alldef_int((int *)src_fld_X, src_nR, (int *)out_X, 
			&dst_nR, (int *)cnt_X);
      }
      else {
	uniq_cnt_alldef_i_i((int *)src_fld_X, (int *)in_cnt_fld_X, 
			    src_nR, (int *)out_X, &dst_nR, (int *)cnt_X);
      }
    }
    else {
      uniq_int((int *)src_fld_X, src_nR, nn_src_fld_X,
	       (int *)out_X, &dst_nR, (int *)cnt_X);
    }
  }
  else if ( strcmp(src_fld_meta->fldtype, "long long") == 0 ) {
    status = mk_file(tmp_opfile, sizeof(long long) * src_nR);
    cBYE(status);
    rs_mmap(tmp_opfile, &out_X, &n_out_X, 1);
    cBYE(status);
    if ( nn_src_fld_X == NULL ) { 
      if ( *in_cnt_fld == '\0' ) { 
        status = mk_file(tmp_cnt_opfile, sizeof(long long) * src_nR);
        cBYE(status);
        rs_mmap(tmp_cnt_opfile, &cnt_X, &n_cnt_X, 1);
        cBYE(status);
	uniq_alldef_longlong((long long *)src_fld_X, src_nR, 
			     (long long *)out_X, &dst_nR, (long long *)cnt_X);
      }
      else {
	if ( strcmp(in_cnt_fld_meta->fldtype, "long long") == 0 ) { 
          status = mk_file(tmp_cnt_opfile, sizeof(long long) * src_nR);
          cBYE(status);
          rs_mmap(tmp_cnt_opfile, &cnt_X, &n_cnt_X, 1);
          cBYE(status);
	  uniq_cnt_alldef_ll_ll((long long *)src_fld_X, 
	    (long long *)in_cnt_fld_X, src_nR, (long long *)out_X, &dst_nR, 
	    (long long *)cnt_X);
	}
	else if ( strcmp(in_cnt_fld_meta->fldtype, "double") == 0 ) { 
          status = mk_file(tmp_cnt_opfile, sizeof(double) * src_nR);
          cBYE(status);
          rs_mmap(tmp_cnt_opfile, &cnt_X, &n_cnt_X, 1);
          cBYE(status);
	  uniq_cnt_alldef_ll_dbl((long long *)src_fld_X, 
	    (double *)in_cnt_fld_X, src_nR, (long long *)out_X, &dst_nR, 
	    (double *)cnt_X);
	}
      }
    }
    else {
      if ( *in_cnt_fld == '\0' ) { 
        status = mk_file(tmp_cnt_opfile, sizeof(long long) * src_nR);
        cBYE(status);
        rs_mmap(tmp_cnt_opfile, &cnt_X, &n_cnt_X, 1);
        cBYE(status);
        uniq_longlong((long long *)src_fld_X, src_nR, nn_src_fld_X, 
		    (long long *)out_X, &dst_nR, (long long *)cnt_X);
      }
      else {
	fprintf(stderr, "NOT IMPLEMENTED\n"); go_BYE(-1);
      }
    }
  }
  else { go_BYE(-1); }
  if ( dst_nR == 0 ) { goto BYE; }
  //-----------------------------------------------------------
  status = is_tbl(dst_tbl, &dst_tbl_id); cBYE(status);
  if ( dst_tbl_id >= 0 ) { 
    status = del_tbl(NULL, dst_tbl_id);
  }
  sprintf(buffer, "%lld", dst_nR);
  status = add_tbl(dst_tbl, buffer, &dst_tbl_id);
  cBYE(status);
  //-----------------------------------------------------------
  // Now copy the temporary files over to real files
  status = open_temp_file(&ofp, &opfile, 0);
  cBYE(status);
  if ( strcmp(src_fld_meta->fldtype, "int") == 0 ) {
    fwrite(out_X, sizeof(int), dst_nR, ofp);
    n_sizeof = sizeof(int);
  }
  else if ( strcmp(src_fld_meta->fldtype, "long long") == 0 ) {
    fwrite(out_X, sizeof(long long), dst_nR, ofp);
    n_sizeof = sizeof(long long);
  }
  fclose_if_non_null(ofp);
  // Add output field to meta data 
  sprintf(str_meta_data, "filename=%s:fldtype=%s:n_sizeof=%u", opfile,
	  src_fld_meta->fldtype, n_sizeof);
  status = add_fld(dst_tbl, dst_fld, str_meta_data, &dst_fld_id);
  cBYE(status);
  // Since src_fld is sorted ascending, so is dst_fld 
  g_fld[dst_fld_id].sorttype = 1; 
  // Now copy the temporary count file to the real one
  status = open_temp_file(&cnt_ofp, &cnt_opfile, 0);
  cBYE(status);
  if ( *in_cnt_fld == '\0' ) { 
    if ( strcmp(src_fld_meta->fldtype, "int") == 0 ) {
      fwrite(cnt_X, sizeof(int), dst_nR, cnt_ofp);
      sprintf(str_meta_data, "filename=%s:fldtype=int:n_sizeof=%lu", 
        cnt_opfile, sizeof(int));
    }
    else if ( strcmp(src_fld_meta->fldtype, "long long") == 0 ) {
      fwrite(cnt_X, sizeof(long long), dst_nR, cnt_ofp);
      sprintf(str_meta_data, "filename=%s:fldtype=long long:n_sizeof=%lu",
        cnt_opfile, sizeof(long long));
    }
    else { go_BYE(-1); }
  }
  else {
    if ( strcmp(in_cnt_fld_meta->fldtype, "int") == 0 ) {
      fwrite(cnt_X, sizeof(int), dst_nR, cnt_ofp);
      sprintf(str_meta_data, "filename=%s:fldtype=int:n_sizeof=%lu",
        cnt_opfile, sizeof(int));
    }
    else if ( strcmp(in_cnt_fld_meta->fldtype, "long long") == 0 ) {
      fwrite(cnt_X, sizeof(long long), dst_nR, cnt_ofp);
      sprintf(str_meta_data, "filename=%s:fldtype=long long:n_sizeof=%lu",
        cnt_opfile, sizeof(long long));
    }
    else if ( strcmp(in_cnt_fld_meta->fldtype, "double") == 0 ) {
      fwrite(cnt_X, sizeof(double), dst_nR, cnt_ofp);
      sprintf(str_meta_data, "filename=%s:fldtype=double:n_sizeof=%lu",
        cnt_opfile, sizeof(double));
    }
    else { go_BYE(-1); }
  }
  fclose_if_non_null(cnt_ofp);
  // Add count field to meta data 
  status = add_fld(dst_tbl, out_cnt_fld, str_meta_data, &cnt_fld_id);
  cBYE(status);
  //-----------------------------------------------------------
 BYE:
  g_write_to_temp_dir = false;
  rs_munmap(src_fld_X, src_fld_nX);
  free_if_non_null(opfile);
  rs_munmap(src_fld_X, src_fld_nX);
  rs_munmap(in_cnt_fld_X, in_cnt_fld_nX);
  rs_munmap(cnt_X, n_cnt_X);
  rs_munmap(out_X, n_out_X);
  rs_munmap(nn_src_fld_X, nn_src_fld_nX);
  unlink(tmp_opfile);
  unlink(tmp_cnt_opfile);
  free_if_non_null(tmp_opfile);
  free_if_non_null(tmp_cnt_opfile);
  free_if_non_null(cnt_opfile);
  return(status);
}
Exemplo n.º 10
0
// START FUNC DECL
int
orphan_files(
    char *in_data_dir,
    char *action
    )
// STOP FUNC DECL
{
  int status = 0;
  DIR *dp = NULL; FILE *ofp = NULL;
  struct dirent *ep = NULL;     
  char Xopfile[MAX_LEN_FILE_NAME+1];
  char Yopfile[MAX_LEN_FILE_NAME+1];
  char *X = NULL; size_t nX = 0;
  char *Y = NULL; size_t nY = 0;
  char nullc = '\0';
  int Xnum_files = 0, Ynum_files = 0;
  int ddir_id;
  char data_dir[MAX_LEN_DIR_NAME+1];

  if ( ( in_data_dir == NULL ) || ( *in_data_dir == '\0' ) ) {
    status = chdir(g_data_dir); cBYE(status);
    ddir_id = -1;
    if ( strlen(g_data_dir) >= MAX_LEN_DIR_NAME ) { go_BYE(-1); }
    strcpy(data_dir, g_data_dir);
  }
  else {
    status = strip_trailing_slash(in_data_dir, data_dir, MAX_LEN_DIR_NAME+1);
    cBYE(status);
    status = get_ddir_id(data_dir, g_ddirs, g_n_ddir, false, &ddir_id);
    cBYE(status);
    status = chdir(data_dir); cBYE(status);
  }

  zero_string(Xopfile, MAX_LEN_FILE_NAME+1);
  zero_string(Yopfile, MAX_LEN_FILE_NAME+1);

  /* START: Create X. X = filenames listed in the meta data (whose 
   * ddir_id matches that requested) in sorted order */
  status = open_temp_file(g_cwd, g_cwd, Xopfile, 0); cBYE(status);
  ofp = fopen(Xopfile, "w"); 
  return_if_fopen_failed(ofp, Xopfile, "w");
  for ( int i = 0; i < g_n_fld; i++ ) { 
    if ( g_flds[i].name[0] == '\0' ) { continue; }
    if ( g_flds[i].ddir_id != ddir_id ) { continue; }
    Xnum_files++;
    fwrite(g_flds[i].filename, sizeof(char), MAX_LEN_FILE_NAME+1, ofp);
  }
  fclose_if_non_null(ofp);
  if ( Xnum_files > 0 ) { 
    status = rs_mmap(Xopfile, &X, &nX, 1);
    qsort(X, Xnum_files, MAX_LEN_FILE_NAME+1, str_compare);
  }
  /* STOP ---- */

  status = open_temp_file(g_cwd, g_cwd, Yopfile, 0); cBYE(status);
  ofp = fopen(Yopfile, "w"); 
  return_if_fopen_failed(ofp, Xopfile, "w");
  dp = opendir(data_dir);
  if ( dp == NULL) { go_BYE(-1); }
  for ( ; ; ) { 
    ep = readdir (dp);
    if ( ep == NULL ) { break; }
    char *filename = ep->d_name;
    if ( ( filename == NULL ) || ( *filename == '\0' ) )  { break; }
    if ( strncmp(filename, "docroot", 7) == 0  ) { continue; }
    if ( strncmp(filename, ".LDB_META", 9) == 0  ) { continue; }
    if ( strcmp(filename, "..") == 0  ) { continue; }
    if ( strcmp(filename, ".") == 0  ) { continue; }
    int len = strlen(filename);
    if ( len >= MAX_LEN_FILE_NAME ) {
      // This is clearly no good 
      fprintf(stdout, "%s\n", filename);
    }
    else {
      Ynum_files++;
      fwrite(filename, sizeof(char), len, ofp);
      for ( int i = len; i < MAX_LEN_FILE_NAME+1; i++ ) { 
        fwrite(&nullc, sizeof(char), 1, ofp);
      }
    }
  }
  fclose_if_non_null(ofp);
  (void) closedir (dp);
  if ( Ynum_files > 0 ) { 
    status = rs_mmap(Yopfile, &Y, &nY, 1);
    qsort(Y, Ynum_files, MAX_LEN_FILE_NAME+1, str_compare);
  }
  /* Now compare X against Y */
  char *xptr = X;
  char *yptr = Y;
  int seenX = 0, seenY = 0;
  for ( ; ( ( seenX < Xnum_files ) && ( seenY < Ynum_files ) ) ; ) {
    int cmpval = str_compare(xptr, yptr);
    switch ( cmpval ) { 
      case 0 : 
      /* File in meta data exists on disk. This is good */
      xptr += MAX_LEN_FILE_NAME+1;
      yptr += MAX_LEN_FILE_NAME+1;
      seenX++;
      seenY++;
      break;
      case -1 : 
      /* File exists in meta data but not on disk. This is horrendous */
      printf("HORRENDOUS %s\n", yptr);
      xptr += MAX_LEN_FILE_NAME+1;
      seenX++;

      break;
      case 1 : 
      /* File on disk but not in meta data. This is an "orphan" * */
      printf("%s\n", yptr);
      yptr += MAX_LEN_FILE_NAME+1;
      seenY++;
      break;
      default : 
      go_BYE(-1);
      break;
    }
  }
  if ( seenX < Xnum_files ) {
    for ( ; seenX < Xnum_files ; seenX++ ) {
      fprintf(stderr, "MISSING FILE %s\n", xptr);
      xptr += MAX_LEN_FILE_NAME;
      seenX++;
    }
  }
  if ( seenY < Ynum_files ) {
    for ( ; seenY < Ynum_files ; seenY++ ) {
      fprintf(stderr, "%s\n", yptr);
      yptr += MAX_LEN_FILE_NAME;
      seenY++;
    }
  }


BYE:
  fclose_if_non_null(ofp);
  unlink_if_non_null(Xopfile); 
  unlink_if_non_null(Yopfile);
  return(status);
}
Exemplo n.º 11
0
//---------------------------------------------------------------
int 
crossprod1(
      char *docroot,
      sqlite3 *in_db,
      char *t1,
      char *f1,
      char *f2
      )
{
  int status = 0;
  sqlite3 *db = NULL;

  char *f1_X = NULL; size_t f1_nX = 0;
  char *f2_X = NULL; size_t f2_nX = 0;

  char *nn_f1_X = NULL; size_t nn_f1_nX = 0;
  char *nn_f2_X = NULL; size_t nn_f2_nX = 0;

  int t1_id, t2_id, t3_id, itemp; bool b_is_tbl;
  FLD_META_TYPE f1_meta, f2_meta;
  FLD_META_TYPE nn_f1_meta, nn_f2_meta;
  int i1val, i2val; int *iptr1 = NULL, *iptr2 = NULL;
  long long nR1, nR2, nR3 = 0;
  char str_meta_data[1024];
  char *t3f1_opfile = NULL, *t3f2_opfile = NULL;
  FILE *ofp1 = NULL, *ofp2 = NULL;
  char str_nR3[32];
  //----------------------------------------------------------------
  zero_string(str_meta_data, 1024);
  zero_string(str_nR3, 32);
  zero_fld_meta(&f1_meta);
  zero_fld_meta(&f2_meta);
  zero_fld_meta(&nn_f1_meta);
  zero_fld_meta(&nn_f2_meta);
  //----------------------------------------------------------------
  status = open_temp_file(&ofp1, &t3f1_opfile);
  cBYE(status);
  status = open_temp_file(&ofp2, &t3f2_opfile);
  cBYE(status);
  status = open_db_if_needed(docroot, in_db, &db);
  cBYE(status);
  //----------------------------------------------------------------
  status = external_get_nR(docroot, db, t1, &nR1, &t1_id);
  cBYE(status);
  if ( nR1 <= 0 ) { go_BYE(-1); }
  status = fld_meta(docroot, db, t1, f1, -1, &f1_meta);
  cBYE(status);
  status = rs_mmap(f1_meta.filename, &f1_X, &f1_nX, 0);
  cBYE(status);

  // Get nn field for t1/f1 if it exists
  status = get_aux_field_if_it_exists(docroot, db, t1, f1_meta.id, 
      "nn", &nn_f1_meta, &nn_f1_X, &nn_f1_nX);
  //----------------------------------------------------------------
  status = external_get_nR(docroot, db, t2, &nR2, &t2_id);
  cBYE(status);
  if ( nR2 <= 0 ) { go_BYE(-1); }
  status = fld_meta(docroot, db, t2, f2, -1, &f2_meta);
  cBYE(status);
  status = rs_mmap(f2_meta.filename, &f2_X, &f2_nX, 0);
  cBYE(status);

  // Get nn field for t2/f2 if it exists
  status = get_aux_field_if_it_exists(docroot, db, t2, f2_meta.id, 
      "nn", &nn_f2_meta, &nn_f2_X, &nn_f2_nX);
  //----------------------------------------------------------------
  iptr1 = (int *)f1_X;
  iptr2 = (int *)f2_X;
  for ( long long i1 = 0; i1 < nR1; i1++ ) { 
    if ( ( nn_f1_X == NULL ) || ( nn_f1_X[i1] == 1 ) )  {
      i1val = iptr1[i1];
      for ( long long i2 = 0; i2 < nR2; i2++ ) { 
        if ( ( nn_f2_X == NULL ) || ( nn_f2_X[i2] == 1 ) )  {
          i2val = iptr2[i2];
	  fwrite(&i1val, sizeof(int), 1, ofp1);
	  fwrite(&i2val, sizeof(int), 1, ofp2);
	  nR3++;
	}
      }
    }
  }
  fclose_if_non_null(ofp1);
  fclose_if_non_null(ofp2);

  //----------------------------------------------------------------
  // Add output fields to t3 meta data 
  status = is_tbl(docroot, db, t3, &b_is_tbl, &t3_id, &itemp);
  cBYE(status);
  if ( b_is_tbl ) {
    status = del_tbl(docroot, db, t3);
    cBYE(status);
  }
  if ( nR3 == 0 ) {
    fprintf(stderr, "No data to create t3 \n");
    goto BYE;
  }
  sprintf(str_nR3, "%lld", nR3);
  status = add_tbl(docroot, db, t3, str_nR3, &itemp);
  cBYE(status);

  sprintf(str_meta_data, "fldtype=int:n_sizeof=%d:filename=%s",
      sizeof(int), t3f1_opfile);
  status = add_fld(docroot, db, t3, f1, str_meta_data);
  cBYE(status);
  zero_string(str_meta_data, 1024);
  sprintf(str_meta_data, "fldtype=int:n_sizeof=%d:filename=%s",
      sizeof(int), t3f2_opfile);
  status = add_fld(docroot, db, t3, f2, str_meta_data);
  cBYE(status);
 BYE:
  fclose_if_non_null(ofp1);
  fclose_if_non_null(ofp2);
  if ( in_db == NULL ) { sqlite3_close(db); }
  rs_munmap(f1_X, f1_nX);
  rs_munmap(nn_f1_X, nn_f1_nX);
  rs_munmap(f2_X, f2_nX);
  rs_munmap(nn_f2_X, nn_f2_nX);
  free_if_non_null(t3f1_opfile);
  free_if_non_null(t3f2_opfile);
  return(status);
}
Exemplo n.º 12
0
//---------------------------------------------------------------
// START FUNC DECL
int 
count_strings(
	   char *src_tbl,
	   char *src_fld,
	   char *str_trunc_len, 
	   // truncate strings beyond this len
	   char *dst_tbl
	   )
// STOP FUNC DECL
{
  int status = 0;
  char *X = NULL; size_t nX = 0;
  char *src_fld_X = NULL; size_t src_fld_nX = 0;
  char *nn_src_fld_X = NULL; size_t nn_src_fld_nX = 0;
  char *sz_src_fld_X = NULL; size_t sz_src_fld_nX = 0;
  FLD_TYPE *src_fld_meta = NULL; 
  FLD_TYPE *nn_src_fld_meta = NULL; 
  FLD_TYPE *sz_src_fld_meta = NULL;
  long long src_nR; 
  int src_tbl_id = INT_MIN, src_fld_id = INT_MIN;
  int dst_tbl_id = INT_MIN;
  int nn_src_fld_id = INT_MIN, sz_src_fld_id = INT_MIN;
  int itemp;
  char str_dst_nR[32]; long long dst_nR = 0;
  char str_meta_data[4096];
  char *out_X = NULL; char *cnt_X = NULL; 
  char *dst_fk_X = NULL; size_t dst_fk_nX = 0;
  FILE *ofp = NULL; char *opfile = NULL; 
  FILE *tmp_ofp = NULL; char *tmp_opfile = NULL; 
  FILE *nn_ofp = NULL; char *nn_opfile = NULL; 
  FILE *sz_ofp = NULL; char *sz_opfile = NULL; 
  FILE *cnt_ofp = NULL; char *cnt_opfile = NULL; 
  FILE *from_dst_ofp = NULL; char *from_dst_opfile = NULL; 
  FILE *dst_fk_ofp = NULL; char *dst_fk_opfile = NULL; int *idst_fk = NULL;
  FILE *to_dst_ofp = NULL; char *to_dst_opfile = NULL; 
  size_t out_nX = 0, cnt_nX = 0;
  int *szptr = NULL; char *nnptr = NULL;
  char  *cptr = NULL, *prev_cptr = NULL; 
  char *sz_src_fld = NULL; char *str_maxlen = NULL; 
  char *from_dst_fld = NULL, *to_dst_fld = NULL, *dst_fk_fld = NULL;
  char *null_str = NULL, *endptr = NULL;
  int maxlen, rec_size, trunc_len = -1; // no truncation by default
  long long cnt = 1; int icnt = 1;
  bool is_any_null = false;
      char is_nn; int sz;
  //----------------------------------------------------------------
  if ( str_trunc_len != NULL ) { 
    trunc_len = strtol(str_trunc_len, &endptr, 10);
    if ( *endptr != '\0' ) { go_BYE(-1); }
    if ( trunc_len <= 1 ) { go_BYE(-1); } 
    trunc_len++; // to include null character
  }
  if ( ( src_tbl == NULL ) || ( *src_tbl == '\0' ) ) { go_BYE(-1); }
  if ( ( src_fld == NULL ) || ( *src_fld == '\0' ) ) { go_BYE(-1); }
  if ( ( dst_tbl == NULL ) || ( *dst_tbl == '\0' ) ) { go_BYE(-1); }
  zero_string(str_meta_data, 4096);
  zero_string(str_dst_nR, 32);
  //--------------------------------------------------------
  status = is_tbl(src_tbl, &src_tbl_id); cBYE(status);
  chk_range(src_tbl_id, 0, g_n_tbl);
  status = is_fld(NULL, src_tbl_id, src_fld, &src_fld_id); cBYE(status);
  chk_range(src_fld_id, 0, g_n_fld);
  src_nR = g_tbl[src_tbl_id].nR;
  if ( src_nR > INT_MAX ) { 
    fprintf(stderr, "NOT IMPLEMENTED\n"); go_BYE(-1);
  }
  //--------------------------------------------------------
  src_fld_meta = &(g_fld[src_fld_id]);
  status = rs_mmap(src_fld_meta->filename, &src_fld_X, &src_fld_nX, 0);
  cBYE(status);
  //--------------------------------------------------------

  if ( strcmp(src_fld_meta->fldtype, "char string") != 0 ) { go_BYE(-1); }
  cptr = src_fld_X;
  // Get nn field for source if if it exists
  nn_src_fld_id = g_fld[src_fld_id].nn_fld_id;
  if ( nn_src_fld_id >= 0 ) { 
    nn_src_fld_meta = &(g_fld[nn_src_fld_id]);
    status = rs_mmap(nn_src_fld_meta->filename, &nn_src_fld_X, &nn_src_fld_nX, 0);
    cBYE(status);
  }
  nnptr = (char *)nn_src_fld_X;
  // Get sz field for source 
  sz_src_fld_id = g_fld[src_fld_id].sz_fld_id;
  chk_range(sz_src_fld_id, 0, g_n_fld);
  sz_src_fld_meta = &(g_fld[sz_src_fld_id]);
  status = rs_mmap(sz_src_fld_meta->filename, &sz_src_fld_X, &sz_src_fld_nX, 0);
  cBYE(status);
  szptr = (int *)sz_src_fld_X;
  //------------------------------------
  mcr_alloc_null_str(from_dst_fld, (MAX_LEN_FILE_NAME+16));
  mcr_alloc_null_str(to_dst_fld, (MAX_LEN_FILE_NAME+16));
  mcr_alloc_null_str(dst_fk_fld, (MAX_LEN_FILE_NAME+16));
  mcr_alloc_null_str(sz_src_fld, (MAX_LEN_FILE_NAME+16));
  mcr_alloc_null_str(str_maxlen, 32);

  strcpy(sz_src_fld, "_sz_"); strcat(sz_src_fld, src_fld);

  strcpy(from_dst_fld, src_tbl); strcat(from_dst_fld, "_from");
  strcpy(to_dst_fld,   dst_tbl); strcat(to_dst_fld,   "_to");
  strcpy(dst_fk_fld,   dst_tbl); strcat(dst_fk_fld,   "_id");
  
  // Get max length of src field 
  status = f_to_s(src_tbl, sz_src_fld, "max", str_maxlen);
  cBYE(status);
  maxlen = strtol(str_maxlen, &endptr, 10);
  if ( *endptr != '\0' ) { go_BYE(-1); }
  if ( maxlen <= 0 ) { go_BYE(-1); }
  if ( trunc_len < 0 ) { 
    trunc_len = maxlen;
  }
  mcr_alloc_null_str(null_str, trunc_len);
  //--------------------------------------------------------
  // Write out src field as constant length field 
  status = open_temp_file(&tmp_ofp, &tmp_opfile, 0);
  cBYE(status);
  for ( int i = 0; i < src_nR; i++ ) { 
    int sz, fill_sz; 
    sz = szptr[i]; // includes terminating null character
    if ( sz <= trunc_len ) { 
      fwrite(cptr, sizeof(char), sz, tmp_ofp);
      fill_sz = trunc_len - sz;
      fwrite(null_str, sizeof(char), fill_sz, tmp_ofp);
    }
    else {
      fwrite(cptr, sizeof(char), trunc_len-1, tmp_ofp);
      fwrite(null_str, sizeof(char), 1, tmp_ofp);
    }
    // Append origin of this string 
    fwrite(&i, sizeof(int), 1, tmp_ofp);
    cptr += sz;
    // TODO: Check about nullc and sz 
  }
  fclose_if_non_null(tmp_ofp);
  //---------------------------------------------------------
  // Sort output file so that similar titles occur together
  status = rs_mmap(tmp_opfile, &X, &nX, 1); // 1 for writing
  cBYE(status);
  // Note that we added the origin of the string. Hence the size of a
  // record is trunc_len + sizeof(int)
  rec_size = trunc_len + sizeof(int);
  qsort(X, src_nR, rec_size, str_compare);
  rs_munmap(X, nX);
  //--------------------------------------------------------
  // Now we need to compact the sorted output so that it is not padded
  // Open 4 files for 
  // output string, sz(output string), nn(output string), cnt 
  status = open_temp_file(&ofp,     &opfile, 0);     cBYE(status);
  status = open_temp_file(&nn_ofp,  &nn_opfile, 0);  cBYE(status);
  status = open_temp_file(&sz_ofp,  &sz_opfile, 0);  cBYE(status);
  status = open_temp_file(&cnt_ofp, &cnt_opfile, 0); cBYE(status);
  status = open_temp_file(&from_dst_ofp, &from_dst_opfile, 0); cBYE(status);
  status = open_temp_file(&to_dst_ofp, &to_dst_opfile, 0); cBYE(status);

  status = open_temp_file(&dst_fk_ofp, &dst_fk_opfile, 0); cBYE(status);
  fclose_if_non_null(dst_fk_ofp);
  mk_file(dst_fk_opfile, src_nR * sizeof(int)); cBYE(status);
  rs_mmap(dst_fk_opfile,  &dst_fk_X, &dst_fk_nX, 1); cBYE(status);
  idst_fk = (int *)dst_fk_X;


  status = rs_mmap(tmp_opfile, &X, &nX, 0); 
  cBYE(status);
  cptr = X;
  prev_cptr = cptr;
  cptr += rec_size;
  cnt = icnt = 1;
  for ( int i = 0; i < src_nR; i++ ) { 
    char *xptr; int *iptr; 
    xptr = prev_cptr + trunc_len;
    iptr = (int *)xptr;
    if ( ( *iptr < 0 ) || ( *iptr >= src_nR ) ) { go_BYE(-1); }
    fwrite(iptr,    sizeof(int), 1, from_dst_ofp);
    fwrite(&dst_nR, sizeof(int), 1, to_dst_ofp);
    idst_fk[*iptr] = dst_nR;
    if ( strcmp(cptr, prev_cptr) != 0 ) { 

      sz = strlen(prev_cptr);
      if ( sz == 0 ) { is_nn = FALSE; } else { is_nn = TRUE; }
      sz++; // for null character termination

      fwrite(prev_cptr,   sizeof(char), sz, ofp);
      fwrite(&sz,         sizeof(int),  1, sz_ofp);
      fwrite(&is_nn,      sizeof(char), 1,  nn_ofp);

      if ( cnt > INT_MAX ) { go_BYE(-1); }
      icnt = cnt;
      fwrite(&icnt,   sizeof(int), 1,  cnt_ofp);
      cnt = 1;
      dst_nR++;
    }
    else {
      cnt++;
    }
    prev_cptr = cptr;
    cptr += rec_size;
  }
  // START: Now for the last guy
  cnt--; // this is needed
  sz = strlen(prev_cptr);
  if ( sz == 0 ) { is_nn = FALSE; } else { is_nn = TRUE; }
  if ( is_nn == FALSE ) { 
    is_any_null = true;
  }
  sz++; // for null character termination

  fwrite(prev_cptr,   sizeof(char), sz, ofp);
  fwrite(&sz,         sizeof(int),  1, sz_ofp);
  fwrite(&is_nn,      sizeof(char), 1,  nn_ofp);

  if ( cnt > INT_MAX ) { go_BYE(-1); }
  icnt = cnt;
  fwrite(&icnt,   sizeof(int), 1,  cnt_ofp);
  dst_nR++;
  // STOP: Now for the last guy
  //---------------------------
  fclose_if_non_null(ofp);
  fclose_if_non_null(nn_ofp);
  fclose_if_non_null(sz_ofp);
  fclose_if_non_null(cnt_ofp);
  fclose_if_non_null(from_dst_ofp);
  fclose_if_non_null(to_dst_ofp);
  //-----------------------------------------------
  // Delete temp file
  unlink(tmp_opfile);
  //-----------------------------------------------------------
  status = is_tbl(dst_tbl, &dst_tbl_id); cBYE(status);
  if ( dst_tbl_id >= 0 ) { /* Delete table */
    status = del_tbl(NULL, dst_tbl_id);
  }
  sprintf(str_dst_nR, "%lld", dst_nR);
  status = add_tbl(dst_tbl, str_dst_nR, &dst_tbl_id);
  cBYE(status);
  //-----------------------------------------------------------
  /* TODO I think we can dispense with from_dst_fld and to_dst_fld */
  sprintf(str_meta_data, "filename=%s:fldtype=int:n_sizeof=4", from_dst_opfile);
  status = add_fld(src_tbl, from_dst_fld, str_meta_data, &itemp);
  cBYE(status);

  sprintf(str_meta_data, "filename=%s:fldtype=int:n_sizeof=4", to_dst_opfile);
  status = add_fld(src_tbl, to_dst_fld, str_meta_data, &itemp);
  cBYE(status);

  sprintf(str_meta_data, "filename=%s:fldtype=int:n_sizeof=4", dst_fk_opfile);
  status = add_fld(src_tbl, dst_fk_fld, str_meta_data, &itemp);
  cBYE(status);

  // Add output field to meta data 
  sprintf(str_meta_data, "filename=%s:fldtype=char string:n_sizeof=0", opfile);
  status = add_fld(dst_tbl, src_fld, str_meta_data, &itemp);
  cBYE(status);
  fclose_if_non_null(cnt_ofp);
  // Add nn field if needed
  if ( is_any_null ) { 
    status = add_aux_fld(dst_tbl, src_fld, nn_opfile, "nn", &itemp);
    cBYE(status);
  }
  else {
    unlink(nn_opfile);
  }
  // Add sz field 
  status = add_aux_fld(dst_tbl, src_fld, sz_opfile, "sz", &itemp);
  cBYE(status);
  // Add count field to meta data 
  sprintf(str_meta_data, "filename=%s:fldtype=int:n_sizeof=4",
      cnt_opfile);
  status = add_fld(dst_tbl, "cnt", str_meta_data, &itemp);
  cBYE(status);
  //-----------------------------------------------------------
 BYE:
  free_if_non_null(null_str);
  rs_munmap(X, nX);
  rs_munmap(src_fld_X, src_fld_nX);
  rs_munmap(cnt_X, cnt_nX);
  rs_munmap(out_X, out_nX);
  rs_munmap(dst_fk_X, dst_fk_nX);
  rs_munmap(nn_src_fld_X, nn_src_fld_nX);
  fclose_if_non_null(ofp);     free_if_non_null(opfile);
  fclose_if_non_null(nn_ofp);  free_if_non_null(nn_opfile);
  fclose_if_non_null(sz_ofp);  free_if_non_null(sz_opfile);
  fclose_if_non_null(cnt_ofp); free_if_non_null(cnt_opfile);
  fclose_if_non_null(tmp_ofp); free_if_non_null(tmp_opfile);
  fclose_if_non_null(from_dst_ofp); free_if_non_null(from_dst_opfile);
  fclose_if_non_null(to_dst_ofp); free_if_non_null(to_dst_opfile);
  free_if_non_null(sz_src_fld);
  free_if_non_null(dst_fk_fld);
  free_if_non_null(str_maxlen);
  return(status);
}
Exemplo n.º 13
0
// START FUNC DECL
int
read_csv(
	 FLD_META_TYPE *flds,
	 int n_flds,
	 char *infile,
	 char *str_load_fld,
	 char fld_delim, /* double quote character */
	 char fld_sep, /* comma */
	 char rec_delim, /* new line character */
	 bool ignore_hdr, /* whether to ignore header or not */
	 bool is_null_null, /* whether to convert "null" to null */
	 char ***ptr_nn_fnames,
	 char ***ptr_sz_fnames,
	 long long *ptr_num_rows
	 )
// STOP FUNC DECL
{
  int status = 0;
  int *load_fld = NULL; int n_load_fld = 0;
  char *X = NULL; size_t nX = 0;
  bool in_fld, is_fld_delim;
  long long num_rows = 0; long long num_cells = 0, fld_idx; 
  long long lb, ub;
  FILE **fps = NULL; char **fnames = NULL;
  FILE **nn_fps = NULL; char **nn_fnames = NULL;
  FILE **sz_fps = NULL; char **sz_fnames = NULL;
  bool in_escape_mode = false, is_fld_delim_escaped = false;
  
  g_num_rows = 0;
  g_buflen = 8192;
  g_buffer = (char *)malloc(g_buflen * sizeof(char));
  return_if_malloc_failed(g_buffer);
  zero_string(g_buffer, g_buflen);

  /*----------------------------------------------------------------------------*/
  if ( n_flds <= 0 ) { go_BYE(-1); }
  fps = (FILE **)malloc(n_flds * sizeof(FILE *));
  return_if_malloc_failed(fps);
  fnames = (char **)malloc(n_flds * sizeof(char *));
  return_if_malloc_failed(fnames);

  nn_fps = (FILE **)malloc(n_flds * sizeof(FILE *));
  return_if_malloc_failed(nn_fps);
  nn_fnames = (char **)malloc(n_flds * sizeof(char *));
  return_if_malloc_failed(nn_fnames);

  sz_fps = (FILE **)malloc(n_flds * sizeof(FILE *));
  return_if_malloc_failed(sz_fps);
  sz_fnames = (char **)malloc(n_flds * sizeof(char *));
  return_if_malloc_failed(sz_fnames);

  for ( int i = 0; i < n_flds; i++ ) { 
    fps[i] = NULL;
    nn_fps[i] = NULL;
    sz_fps[i] = NULL;
    fnames[i] = nn_fnames[i] = sz_fnames[i] = NULL;
  }
  /*----------------------------------------------------------------------------*/

  if ( ( str_load_fld == NULL ) || ( *str_load_fld == '\0' ) ) {
    load_fld = (int *)malloc(n_flds * sizeof(int));
    return_if_malloc_failed(load_fld);
    for ( int i = 0; i < n_flds; i++ ) { 
      load_fld[i]= 1;
    }
  }
  status = parse_scalar_vals(str_load_fld, &load_fld, &n_load_fld);
  cBYE(status);
  if ( n_load_fld != n_flds ) { go_BYE(-1); }
  status = rs_mmap(infile, &X, &nX, 0);
  cBYE(status);
  lb = 0; fld_idx = 0; in_fld = false; is_fld_delim = false;
  for ( int i = 0; i < n_flds; i++ ) { 
    if ( load_fld[i] == TRUE ) { 
      status = open_temp_file(&(fps[i]), &(fnames[i]));
      cBYE(status);
      status = open_temp_file(&(nn_fps[i]), &(nn_fnames[i]));
      cBYE(status);
      if ( strcmp(flds[i].fldtype, "char string" ) == 0 ) {
        status = open_temp_file(&(sz_fps[i]), &(sz_fnames[i]));
	cBYE(status);
      }
    }
  }
  for ( long long i = 0; i < nX; i++ ) {
    // START: Error handling for escape conventions
    /* TODO: If I get an incorrect backslash, I simply replace it with a
     * space. This is incorrect. But I need it to read files created 
     * with jspool utility from Jay Thomas. The correct thing to do is
     * to fix the utility.
     */
    if ( in_escape_mode ) { 
      if ( ( X[i] != '\\' ) && ( X[i] != '"' ) ) {
	/* replace the backslash that put is into escape mode wih a space */
	X[i-1] = ' '; 
      }
    }
    // STOP: Error handling for escape conventions
    // START: Handling bslash and dquote 
    if ( X[i] == '\\' ) { 
      if ( in_escape_mode ) { 
	in_escape_mode = false;
      }
      else {
	in_escape_mode = true;
      }
    }
    // STOP: Handling bslash and dquote 
    if ( X[i] == '"' ) { 
      if ( in_escape_mode ) {
	is_fld_delim_escaped = true;
	in_escape_mode = false;
      }
    }
    if ( in_fld == false ) {
      in_fld = true;
      if ( X[i] == fld_delim ) { /* consume this character */
	is_fld_delim = true;
        lb = i+1;
      }
      else {
        lb = i; 
	if ( ( ( fld_idx < (n_flds-1) ) && ( X[i] == fld_sep ) ) || 
	     ( ( fld_idx == n_flds-1 ) && ( X[i] == rec_delim ) ) ) {
	  /* field has terminated */
	  ub = i-1;
	  if ( load_fld[fld_idx] == 1 ) { 
	    if ( ( ignore_hdr ) & ( num_rows == 0 ) ) {
	      /* Do nothing */
	    }
	    else {
	      status = print_cell(X, lb, ub+1, flds[fld_idx].fldtype,
				  flds[fld_idx].n_sizeof, is_null_null,
				  fps[fld_idx], nn_fps[fld_idx],
				  sz_fps[fld_idx]);
	      cBYE(status);
	    }
	  }
	  in_fld = false;
	  num_cells++;
          if ( ( fld_idx == n_flds-1 ) && ( X[i] == rec_delim ) ) {
	    num_rows++;
	    g_num_rows = num_rows;
	  }
	  fld_idx++;
	  if ( fld_idx == n_flds ) { fld_idx = 0; }
	}
      }
    }
    else { /* Now within field. Decide whether to continue or stop. */
      if ( is_fld_delim ) {
	/* if field started with a fld_delim, it must end with one */
	/* Note that we need to take care of escaping the dquote */
	if ( ( X[i] == fld_delim ) && ( !is_fld_delim_escaped ) ) {
	  /* next char must be fld_sep or rec_delim or we have come to eof */
	  if ( i+1 > nX ) { go_BYE(-1); }
	  if ( ( fld_idx < (n_flds-1) ) && ( X[i+1] != fld_sep ) ) { 
	    go_BYE(-1); 
	  }
	  if ( fld_idx == n_flds-1 ) {
	    if ( X[i+1] != rec_delim ) { 
	      go_BYE(-1); 
	    }
	    else {
	      num_rows++;
	      g_num_rows = num_rows;
	    }
	  }
	  ub = i-1;
	  if ( load_fld[fld_idx] == 1 ) { 
	    if ( ( ignore_hdr ) & ( num_rows == 0 ) ) {
	      /* Do nothing */
	    }
	    else {
	      status = print_cell(X, lb, ub+1, flds[fld_idx].fldtype,
				  flds[fld_idx].n_sizeof, is_null_null,
				  fps[fld_idx], nn_fps[fld_idx], sz_fps[fld_idx]);
	      cBYE(status);
	    }
	  }
	  in_fld = false;
	  is_fld_delim = false;
	  fld_idx++;
	  if ( fld_idx == n_flds ) { fld_idx = 0; }
	  num_cells++;
	  i++; /* Consume next char which must be fld_sep or rec_delim */
	}
      }
      else {
	if ( ( i == nX ) ||  
	     ( ( fld_idx < (n_flds-1) ) && ( X[i] == fld_sep ) ) || 
	     ( ( fld_idx == n_flds-1 ) && ( X[i] == rec_delim ) ) ) {
	  /* field has terminated */
	  if ( i == nX ) { 
	    ub = i;
	  }
	  else {
	    ub = i-1;
	  }
	  if ( load_fld[fld_idx] == 1 ) { 
	    if ( ( ignore_hdr ) & ( num_rows == 0 ) ) {
	      /* Do nothing */
	    }
	    else {
	      status = print_cell(X, lb, ub+1, flds[fld_idx].fldtype,
				  flds[fld_idx].n_sizeof, is_null_null,
				  fps[fld_idx], nn_fps[fld_idx], sz_fps[fld_idx]);
	      cBYE(status);
	    }
	  }
	  in_fld = false;
	  num_cells++;
          if ( ( fld_idx == n_flds-1 ) && ( X[i] == rec_delim ) ) {
	    num_rows++;
	    g_num_rows = num_rows;
	  }
	  fld_idx++;
	  if ( fld_idx == n_flds ) { fld_idx = 0; }
	}
      }
    }
    is_fld_delim_escaped = false;
  }
  if ( num_cells != ( n_flds * num_rows ) ) { 
    fprintf(stderr, "num_cells = %lld   \n", num_cells);
    fprintf(stderr, "n_flds    = %d   \n", n_flds   );
    fprintf(stderr, "num_rows  = %lld \n", num_rows);
    go_BYE(-1); 
  }
  if ( ignore_hdr ) { 
    num_rows--; // Because you have eliminated first row 
  }
  /* TODO: Copy information into meta data structure */
  for ( int i = 0; i < n_flds; i++ ) { 
    if ( fnames[i] != NULL ) { // since not all fields are loaded
      strcpy(flds[i].filename, fnames[i]);
    }
  }
  for ( int i = 0; i < n_flds; i++ ) { 
    if ( fnames != NULL ) { free_if_non_null(fnames[i]); }
    if ( fps != NULL ) { fclose_if_non_null(fps[i]); }
    if ( nn_fps != NULL ) { fclose_if_non_null(nn_fps[i]); }
    if ( sz_fps != NULL ) { fclose_if_non_null(sz_fps[i]); }
  }
  *ptr_nn_fnames = nn_fnames;
  *ptr_sz_fnames = sz_fnames;
  *ptr_num_rows = num_rows;
 BYE:
  rs_munmap(X, nX);
  free_if_non_null(load_fld);
  free_if_non_null(nn_fps);
  free_if_non_null(sz_fps);
  free_if_non_null(fnames);
  free_if_non_null(fps);
  free_if_non_null(g_buffer);
  return(status);
}
Exemplo n.º 14
0
int csync_rs_patch(const char *filename)
{
	FILE *basis_file = 0, *delta_file = 0, *new_file = 0;
	int backup_errno;
	rs_stats_t stats;
	rs_result result;
	char *errstr = "?";
	char tmpfname[MAXPATHLEN], newfname[MAXPATHLEN];

	csync_debug(3, "Csync2 / Librsync: csync_rs_patch('%s')\n", filename);

	csync_debug(3, "Receiving delta_file from peer..\n");
	delta_file = open_temp_file(tmpfname, prefixsubst(filename));
	if ( !delta_file ) { errstr="creating delta temp file"; goto io_error; }
	if (unlink(tmpfname) < 0) { errstr="removing delta temp file"; goto io_error; }
	if ( csync_recv_file(delta_file) ) goto error;

	csync_debug(3, "Opening to be patched file on local host..\n");
	basis_file = fopen(prefixsubst(filename), "rb");
	if ( !basis_file ) {
		basis_file = open_temp_file(tmpfname, prefixsubst(filename));
		if ( !basis_file ) { errstr="opening data file for reading"; goto io_error; }
		if (unlink(tmpfname) < 0) { errstr="removing data temp file"; goto io_error; }
	}

	csync_debug(3, "Opening temp file for new data on local host..\n");
	new_file = open_temp_file(newfname, prefixsubst(filename));
	if ( !new_file ) { errstr="creating new data temp file"; goto io_error; }

	csync_debug(3, "Running rs_patch_file() from librsync..\n");
	result = rs_patch_file(basis_file, delta_file, new_file, &stats);
	if (result != RS_DONE) {
		csync_debug(0, "Internal error from rsync library!\n");
		goto error;
	}

	csync_debug(3, "Renaming tmp file to data file..\n");
	fclose(basis_file);

#ifdef __CYGWIN__

/* TODO: needed? */
	// This creates the file using the native windows API, bypassing
	// the cygwin wrappers and so making sure that we do not mess up the
	// permissions..
	{
		char winfilename[MAX_PATH];
		HANDLE winfh;

		cygwin_conv_to_win32_path(prefixsubst(filename), winfilename);

		winfh = CreateFile(TEXT(winfilename),
				GENERIC_WRITE,          // open for writing
				0,                      // do not share
				NULL,                   // default security
				CREATE_ALWAYS,          // overwrite existing
				FILE_ATTRIBUTE_NORMAL | // normal file
				FILE_FLAG_OVERLAPPED,   // asynchronous I/O
				NULL);                  // no attr. template

		if (winfh == INVALID_HANDLE_VALUE) {
			csync_debug(0, "Win32 I/O Error %d in rsync-patch: %s\n",
					(int)GetLastError(), winfilename);
			errno = EACCES;
			goto error;
		}
		CloseHandle(winfh);
	}
#endif

	if (rename(newfname, prefixsubst(filename)) < 0) { errstr="renaming tmp file to to be patched file"; goto io_error; }

	csync_debug(3, "File has been patched successfully.\n");
	fclose(delta_file);
	fclose(new_file);

	return 0;

io_error:
	csync_debug(0, "I/O Error '%s' while %s in rsync-patch: %s\n",
			strerror(errno), errstr, prefixsubst(filename));

error:;
	backup_errno = errno;
	if ( delta_file ) fclose(delta_file);
	if ( basis_file ) fclose(basis_file);
	if ( new_file )   fclose(new_file);
	errno = backup_errno;
	return -1;
}
Exemplo n.º 15
0
static int get_data(
		    char *tbl, 
		    char *fld,
		    long long *ptr_nR,
		    char **ptr_X,
		    size_t *ptr_nX,
		    char **ptr_opfile,
		    int *ptr_fldtype,
		    int *ptr_fldsz
		    )
{
  int status = 0;
  int tbl_id = INT_MIN, fld_id = INT_MIN, nn_fld_id = INT_MIN;
  char *X = NULL;    size_t nX = 0;
  char *nn_X = NULL; size_t nn_nX = 0;
  char *op_X = NULL; size_t op_nX = 0;
  char *endptr = NULL;
  char buffer[32];
  long long nR = 0;
  FLD_TYPE *fld_meta = NULL, *nn_fld_meta = NULL;
  FILE *ofp = NULL; char *opfile = NULL;

  zero_string(buffer, 32);

  status = is_tbl(tbl, &tbl_id); cBYE(status);
  chk_range(tbl_id, 0, g_n_tbl);
  status = is_fld(NULL, tbl_id, fld, &fld_id); cBYE(status);
  chk_range(fld_id, 0, g_n_fld);
  nR = g_tbl[tbl_id].nR;
  fld_meta = &(g_fld[fld_id]);
  nn_fld_id = g_fld[fld_id].nn_fld_id;
  if ( nn_fld_id >= 0 ) { 
    nn_fld_meta = &(g_fld[nn_fld_id]);
    status = rs_mmap(nn_fld_meta->filename, &nn_X, &nn_nX, 0);
    cBYE(status);
  }
  *ptr_fldsz = fld_meta->n_sizeof;
  status = mk_ifldtype(fld_meta->fldtype, ptr_fldtype); cBYE(status);

  switch ( *ptr_fldtype ) { 
  case FLDTYPE_INT : 
  case FLDTYPE_LONGLONG : 
    /* all is well */
    break;
  default : 
    go_BYE(-1);
  }
  if ( nn_X == NULL ) {  /* no nn field */
    status = rs_mmap(fld_meta->filename, &X, &nX, 0);
    cBYE(status);
    *ptr_nR = nR;
    *ptr_X  = X;
    *ptr_nX = nX;
    *ptr_opfile = NULL;
  }
  else {
    long long nn_nR = 0;
    status = f_to_s(tbl, nn_fld_meta->name, "sum", buffer);
    cBYE(status);
    nn_nR = strtol(buffer, &endptr, 10);
    if ( *endptr != '\0' ) { go_BYE(-1); }
    if ( ( nn_nR <= 0 ) || ( nn_nR >= nR ) ) { go_BYE(-1); }
    /* Now make a copy of just the non-null values */
    status = rs_mmap(fld_meta->filename, &X, &nX, 0);
    cBYE(status);
    status = rs_mmap(nn_fld_meta->filename, &nn_X, &nn_nX, 0);
    cBYE(status);
    long long filesz = nn_nR * fld_meta->n_sizeof;
    status = open_temp_file(&ofp, &opfile, filesz); cBYE(status);
    fclose_if_non_null(ofp);
    status = mk_file(opfile, filesz); cBYE(status);
    status = rs_mmap(opfile, &op_X, &op_nX, 1); cBYE(status);
    status = copy_nn_vals(X, nR, nn_X, op_X, nn_nR, fld_meta->n_sizeof);
    cBYE(status);
    /* Cleanup and return values */
    rs_munmap(X, nX);
    rs_munmap(nn_X, nn_nX);
    *ptr_nR = nn_nR;
    *ptr_X  = op_X;
    *ptr_nX = op_nX;
    *ptr_opfile = opfile;
  }
 BYE:
  return(status);
}
Exemplo n.º 16
0
int csync_rs_delta(const char *filename)
{
	FILE *sig_file = 0, *new_file = 0, *delta_file = 0;
	rs_result result;
	rs_signature_t *sumset;
	rs_stats_t stats;
	char tmpfname[MAXPATHLEN];

	csync_debug(3, "Csync2 / Librsync: csync_rs_delta('%s')\n", filename);

	csync_debug(3, "Receiving sig_file from peer..\n");
	sig_file = open_temp_file(tmpfname, prefixsubst(filename));
	if ( !sig_file ) goto io_error;
	if (unlink(tmpfname) < 0) goto io_error;

	if ( csync_recv_file(sig_file) ) {
		fclose(sig_file);
		return -1;
	}
	result = rs_loadsig_file(sig_file, &sumset, &stats);
	if (result != RS_DONE)
		csync_fatal("Got an error from librsync, too bad!\n");
	fclose(sig_file);

	csync_debug(3, "Opening new_file and delta_file..\n");
	new_file = fopen(prefixsubst(filename), "rb");
	if ( !new_file ) {
		int backup_errno = errno;
		csync_debug(0, "I/O Error '%s' while %s in rsync-delta: %s\n",
				strerror(errno), "opening data file for reading", filename);
		csync_send_error();
		fclose(new_file);
		errno = backup_errno;
		return -1;
	}

	delta_file = open_temp_file(tmpfname, prefixsubst(filename));
	if ( !delta_file ) goto io_error;
	if (unlink(tmpfname) < 0) goto io_error;

	csync_debug(3, "Running rs_build_hash_table() from librsync..\n");
	result = rs_build_hash_table(sumset);
	if (result != RS_DONE)
		csync_fatal("Got an error from librsync, too bad!\n");

	csync_debug(3, "Running rs_delta_file() from librsync..\n");
	result = rs_delta_file(sumset, new_file, delta_file, &stats);
	if (result != RS_DONE)
		csync_fatal("Got an error from librsync, too bad!\n");

	csync_debug(3, "Sending delta_file to peer..\n");
	csync_send_file(delta_file);

	csync_debug(3, "Delta has been created successfully.\n");
	rs_free_sumset(sumset);
	fclose(delta_file);
	fclose(new_file);

	return 0;

io_error:
	csync_debug(0, "I/O Error '%s' in rsync-delta: %s\n",
			strerror(errno), prefixsubst(filename));

	if (new_file) fclose(new_file);
	if (delta_file) fclose(delta_file);
	if (sig_file) fclose(sig_file);

	return -1;
}
Exemplo n.º 17
0
//---------------------------------------------------------------
// START FUNC DECL
int
f1opf2(
    char *tbl,
    char *f1,
    char *str_op_spec,
    char *f2
)
// STOP FUNC DECL
{
    int status = 0;
    char *f1_X = NULL;
    size_t f1_nX = 0;
    char *nn_f1_X = NULL;
    size_t nn_f1_nX = 0;
    char *sz_f1_X = NULL;
    size_t sz_f1_nX = 0;
    FLD_TYPE *f1_meta = NULL, *nn_f1_meta = NULL, *sz_f1_meta = NULL;
    long long nR;
    int tbl_id = INT_MIN, f1_id = INT_MIN, nn_f1_id = INT_MIN, sz_f1_id = INT_MIN;
    int         f2_id = INT_MIN, nn_f2_id = INT_MIN, sz_f2_id = INT_MIN;
    char str_meta_data[1024];
    int f1type;
    char *op = NULL, *str_val = NULL;
    FILE *ofp = NULL;
    char *opfile = NULL;
    FILE *nn_ofp = NULL;
    char *nn_opfile = NULL;
    FILE *sz_ofp = NULL;
    char *sz_opfile = NULL;
    char *xform_enum = NULL, *hash_algo = NULL;
    //----------------------------------------------------------------
    if ( ( tbl == NULL ) || ( *tbl == '\0' ) ) {
        go_BYE(-1);
    }
    if ( ( f1 == NULL ) || ( *f1 == '\0' ) ) {
        go_BYE(-1);
    }
    if ( ( f2 == NULL ) || ( *f2 == '\0' ) ) {
        go_BYE(-1);
    }
    if ( ( str_op_spec == NULL ) || ( *str_op_spec == '\0' ) ) {
        go_BYE(-1);
    }
    zero_string(str_meta_data, 1024);
    //--------------------------------------------------------
    status = extract_name_value(str_op_spec, "op=", ":", &op);
    cBYE(status);
    if ( op == NULL ) {
        go_BYE(-1);
    }
    //--------------------------------------------------------
    status = chk_if_ephemeral(&f2);
    cBYE(status);
    status = is_tbl(tbl, &tbl_id);
    cBYE(status);
    chk_range(tbl_id, 0, g_n_tbl);
    nR = g_tbl[tbl_id].nR;
    status = is_fld(NULL, tbl_id, f1, &f1_id);
    cBYE(status);
    chk_range(f1_id, 0, g_n_fld);
    f1_meta = &(g_fld[f1_id]);
    status = rs_mmap(f1_meta->filename, &f1_X, &f1_nX, 0);
    cBYE(status);
    // Get nn field for f1 if if it exists
    nn_f1_id = g_fld[f1_id].nn_fld_id;
    if ( nn_f1_id >= 0 ) {
        nn_f1_meta = &(g_fld[nn_f1_id]);
        status = rs_mmap(nn_f1_meta->filename, &nn_f1_X, &nn_f1_nX, 0);
        cBYE(status);
    }
    //---------------------------------------------
    status = mk_ifldtype(f1_meta->fldtype, &f1type);
    /* TODO: Why do we get an error for 'char string' ? */
    // If f1 type is 'char string', then get sz field
    if ( strcmp(f1_meta->fldtype, "char string") == 0 ) {
        sz_f1_id = g_fld[f1_id].sz_fld_id;
        chk_range(sz_f1_id, 0, g_n_fld);
        sz_f1_meta = &(g_fld[sz_f1_id]);
        status = rs_mmap(sz_f1_meta->filename, &sz_f1_X, &sz_f1_nX, 0);
        cBYE(status);
    }
    //--------------------------------------------------------
    if ( strcmp(op, "xform") == 0 ) {
        bool is_some_null;
        status = open_temp_file(&ofp, &opfile, 0);
        cBYE(status);
        status = open_temp_file(&nn_ofp, &nn_opfile, 0);
        cBYE(status);
        status = open_temp_file(&sz_ofp, &sz_opfile, 0);
        cBYE(status);
        status = str_xform(nR, f1_X, (int *)sz_f1_X, nn_f1_X, str_op_spec,
                           ofp, nn_ofp, sz_ofp, &is_some_null);
        cBYE(status);
        fclose_if_non_null(ofp);
        fclose_if_non_null(nn_ofp);
        fclose_if_non_null(sz_ofp);
        sprintf(str_meta_data, "filename=%s:fldtype=char string:n_sizeof=0",
                opfile);
        status = add_fld(tbl, f2, str_meta_data, &f2_id);
        cBYE(status);
        status = add_aux_fld(tbl, f2, sz_opfile, "sz", &sz_f2_id);
        cBYE(status);
        if ( is_some_null ) {
            status = add_aux_fld(tbl, f2, nn_opfile, "nn", &nn_f2_id);
            cBYE(status);
        }
        else {
            unlink(nn_opfile);
        }
    }
    else if ( strcmp(op, "hash") == 0 ) {
        int hash_len;
        char *endptr = NULL;

        status = extract_name_value(str_op_spec, "len=", ":", &str_val);
        cBYE(status);
        if  ( str_val == NULL ) {
            go_BYE(-1);
        }
        hash_len = strtol(str_val, &endptr, 10);
        if ( *endptr != '\0' ) {
            go_BYE(-1);
        }

        status = extract_name_value(str_op_spec, "hash_algo=", ":", &hash_algo);
        cBYE(status);

        status = open_temp_file(&ofp, &opfile, 0);
        cBYE(status);
        status = f1opf2_hash(hash_algo, nR, f1_X, f1_nX, (int *)sz_f1_X,
                             f1_meta->n_sizeof, nn_f1_X, hash_len, ofp);
        cBYE(status);
        fclose_if_non_null(ofp);
        sprintf(str_meta_data, "filename=%s", opfile);
        switch ( hash_len ) {
        case 4 :
            strcat(str_meta_data, ":fldtype=int:n_sizeof=4");
            break;
        case 8 :
            strcat(str_meta_data, ":fldtype=long long:n_sizeof=8");
            break;
        default :
            go_BYE(-1);
            break;
        }
        fclose_if_non_null(ofp);
        status = add_fld(tbl, f2, str_meta_data, &f2_id);
        cBYE(status);
    }
    else if ( ( strcmp(op, "conv") == 0 ) ||
              ( strcmp(op, "dateconv" ) == 0 ) ||
              ( strcmp(op, "bitcount" ) == 0 ) ||
              ( strcmp(op, "sqrt" ) == 0 ) ||
              ( strcmp(op, "!" ) == 0 ) ||
              ( strcmp(op, "++" ) == 0 ) ||
              ( strcmp(op, "--" ) == 0 ) ||
              ( strcmp(op, "~" ) == 0 )
            ) {
        status = vec_f1opf2(nR, f1type, str_meta_data,
                            f1_X, nn_f1_X, sz_f1_X, op, str_op_spec, &opfile, &nn_opfile);
        cBYE(status);
        status = add_fld(tbl, f2, str_meta_data, &f2_id);
        cBYE(status);
        if ( nn_opfile != NULL ) {
            status = add_aux_fld(tbl, f2, nn_opfile, "nn", &nn_f2_id);
            cBYE(status);
        }
    }
    else if ( strcmp(op, "shift") == 0 ) {
        int ival;
        char *endptr = NULL;
        status = extract_name_value(str_op_spec, "val=", ":", &str_val);
        cBYE(status);
        ival = strtol(str_val, &endptr, 10);
        if ( *endptr != '\0' ) {
            fprintf(stderr, "invalid shift specifier = [%s] \n", str_val);
            go_BYE(-1);
        }
        if ( ( ival == 0 ) || ( ival > MAX_SHIFT ) ||
                (  ival < -MAX_SHIFT ) || (ival >= nR )) {
            fprintf(stderr, "shift = [%d] is out of bounds \n", ival);
            go_BYE(-1);
        }

        status = open_temp_file(&ofp, &opfile, 0);
        cBYE(status);
        status = open_temp_file(&nn_ofp, &nn_opfile, 0);
        cBYE(status);
        status = f1opf2_shift(f1_X, nn_f1_X, nR, f1_meta->fldtype, ival,
                              ofp, nn_ofp);
        cBYE(status);
        fclose_if_non_null(ofp);
        fclose_if_non_null(nn_ofp);
        sprintf(str_meta_data, "filename=%s:fldtype=%s:n_sizeof=%d",opfile,
                f1_meta->fldtype, f1_meta->n_sizeof);
        status = add_fld(tbl, f2, str_meta_data, &f2_id);
        cBYE(status);
        status = add_aux_fld(tbl, f2, nn_opfile, "nn", &nn_f2_id);
        cBYE(status);
    }
    else if ( strcmp(op, "cum") == 0 ) {
        /* TODO: Document. If you do a cum, resultant field is all def */
        char *new_fld_type = NULL;
        int optype, n_sizeof;
        char *str_reset_on = NULL, *str_reset_to = NULL, *endptr = NULL;
        long long reset_on = 0, reset_to = 0;
        bool is_reset = false;
        status = extract_name_value(str_op_spec, "newtype=", ":", &new_fld_type);
        cBYE(status);
        status = extract_name_value(str_op_spec, "reset_on=", ":",
                                    &str_reset_on);
        cBYE(status);
        if ( str_reset_on != NULL ) {
            reset_on = strtoll(str_reset_on, &endptr, 10);
            if ( *endptr != '\0' ) {
                go_BYE(-1);
            }
            is_reset = true;
            status = extract_name_value(str_op_spec, "reset_to=", ":",
                                        &str_reset_to);
            cBYE(status);
            if ( str_reset_to == NULL ) {
                go_BYE(-1);
            }
            reset_to = strtoll(str_reset_to, &endptr, 10);
            if ( *endptr != '\0' ) {
                go_BYE(-1);
            }
        }
        if ( new_fld_type == NULL ) {
            optype = f1type;
            new_fld_type = strdup(f1_meta->fldtype);
            n_sizeof = f1_meta->n_sizeof;
        }
        else {
            if ( strcmp(new_fld_type, "int") == 0 ) {
                optype = FLDTYPE_INT;
                n_sizeof = sizeof(int);
            }
            else if ( strcmp(new_fld_type, "long long") == 0 ) {
                optype = FLDTYPE_LONGLONG;
                n_sizeof = sizeof(long long);
            }
            else {
                go_BYE(-1);
            }
        }
        status = f1opf2_cum(f1_X, nR, f1type, optype, is_reset,
                            reset_on, reset_to, &opfile);
        cBYE(status);
        sprintf(str_meta_data, "filename=%s:fldtype=%s:n_sizeof=%d",opfile,
                new_fld_type, n_sizeof);
        status = add_fld(tbl, f2, str_meta_data, &f2_id);
        cBYE(status);
        free_if_non_null(new_fld_type);
        free_if_non_null(str_reset_on);
        free_if_non_null(str_reset_to);
    }
    else {
        fprintf(stderr, "Invalid op = [%s] \n", op);
        go_BYE(-1);
    }
BYE:
    fclose_if_non_null(ofp);
    fclose_if_non_null(nn_ofp);
    g_write_to_temp_dir = false;
    rs_munmap(f1_X, f1_nX);
    rs_munmap(nn_f1_X, nn_f1_nX);
    rs_munmap(sz_f1_X, sz_f1_nX);
    free_if_non_null(op);
    free_if_non_null(xform_enum);
    free_if_non_null(hash_algo);
    free_if_non_null(str_val);
    free_if_non_null(opfile);
    free_if_non_null(nn_opfile);
    free_if_non_null(sz_opfile);
    return(status);
}
Exemplo n.º 18
0
int csync_rs_check(const char *filename, int isreg)
{
	FILE *basis_file = 0, *sig_file = 0;
	char buffer1[512], buffer2[512];
	int rc, chunk, found_diff = 0;
	int backup_errno;
	rs_stats_t stats;
	rs_result result;
	long size;
	char tmpfname[MAXPATHLEN];

	csync_debug(3, "Csync2 / Librsync: csync_rs_check('%s', %d [%s])\n",
		filename, isreg, isreg ? "regular file" : "non-regular file");

	csync_debug(3, "Opening basis_file and sig_file..\n");

	sig_file = open_temp_file(tmpfname, prefixsubst(filename));
	if ( !sig_file ) goto io_error;
	if (unlink(tmpfname) < 0) goto io_error;

	basis_file = fopen(prefixsubst(filename), "rb");
	if ( !basis_file ) {  /* ?? why a tmp file? */
		basis_file = open_temp_file(tmpfname, prefixsubst(filename));
		if ( !basis_file ) goto io_error;
		if (unlink(tmpfname) < 0) goto io_error;
	}

	if ( isreg ) {
		csync_debug(3, "Running rs_sig_file() from librsync....\n");
		result = rs_sig_file(basis_file, sig_file,
				RS_DEFAULT_BLOCK_LEN, RS_DEFAULT_STRONG_LEN, &stats);
		if (result != RS_DONE) {
			csync_debug(0, "Internal error from rsync library!\n");
			goto error;
		}
	}

	fclose(basis_file);
	basis_file = 0;

	{
		char line[100];
		csync_debug(3, "Reading signature size from peer....\n");
		if ( !conn_gets(line, 100) || sscanf(line, "octet-stream %ld\n", &size) != 1 )
			csync_fatal("Format-error while receiving data.\n");
	}

	fflush(sig_file);
	if ( size != ftell(sig_file) ) {
		csync_debug(2, "Signature size differs: local=%d, peer=%d\n",
				ftell(sig_file), size);
		found_diff = 1;
	}
	rewind(sig_file);

	csync_debug(3, "Receiving %ld bytes ..\n", size);

	while ( size > 0 ) {
		chunk = size > 512 ? 512 : size;
		rc = conn_read(buffer1, chunk);

		if ( rc <= 0 )
			csync_fatal("Read-error while receiving data.\n");
		chunk = rc;

		if ( fread(buffer2, chunk, 1, sig_file) != 1 ) {
			csync_debug(2, "Found EOF in local sig file.\n");
			found_diff = 1;
		}
		if ( memcmp(buffer1, buffer2, chunk) ) {
			csync_debug(2, "Found diff in sig at -%d:-%d\n",
					size, size-chunk);
			found_diff = 1;
		}

		size -= chunk;
		csync_debug(3, "Got %d bytes, %ld bytes left ..\n",
				chunk, size);
	}

	csync_debug(3, "File has been checked successfully (%s).\n",
		found_diff ? "difference found" : "files are equal");
	fclose(sig_file);
	return found_diff;

io_error:
	csync_debug(0, "I/O Error '%s' in rsync-check: %s\n",
			strerror(errno), prefixsubst(filename));

error:;
	backup_errno = errno;
	if ( basis_file ) fclose(basis_file);
	if ( sig_file )   fclose(sig_file);
	errno = backup_errno;
	return -1;
}
Exemplo n.º 19
0
//---------------------------------------------------------------
// START FUNC DECL
int 
srt_join(
	 char *docroot,
	 sqlite3 *in_db,
	 char *src_tbl,
	 char *src_lnk,
	 char *src_val,
	 char *dst_tbl,
	 char *dst_lnk,
	 char *dst_fld,
	 char *op
	 )
// STOP FUNC DECL
{
  int status = 0;
  sqlite3 *db = NULL;
  char *src_val_X = NULL; size_t src_val_nX = 0;
  char *src_lnk_X = NULL; size_t src_lnk_nX = 0;

  char *nn_dst_val_X = NULL; size_t nn_dst_val_nX = 0;
  char *dst_val_X = NULL; size_t dst_val_nX = 0;
  char *dst_lnk_X = NULL; size_t dst_lnk_nX = 0;
  FLD_META_TYPE src_val_meta, src_lnk_meta;
  FLD_META_TYPE               dst_lnk_meta;
  long long src_nR, dst_nR;  // counts all rows 
  char str_meta_data[1024]; 
  int ijoin_op; bool is_any_null = false;
  int src_tbl_id, dst_tbl_id; bool b_is_tbl;
  FILE *ofp = NULL, *nn_ofp = NULL;
  char *opfile = NULL, *nn_opfile = NULL;
  //----------------------------------------------------------------
  zero_string(str_meta_data, 1024);
  status = mk_mjoin_op(op, &ijoin_op);
  cBYE(status);
  status = open_db_if_needed(docroot, in_db, &db);
  cBYE(status);
  zero_fld_meta(&src_val_meta);
  zero_fld_meta(&src_lnk_meta);
  zero_fld_meta(&dst_lnk_meta);
  //----------------------------------------------------------------
  // Get meta-data for all necessary fields 
  status = is_tbl(docroot, db, src_tbl, &b_is_tbl, &src_tbl_id);
  cBYE(status);
  if ( b_is_tbl == false ) { 
    fprintf(stderr, "tbl [%s] not found\n", src_tbl); go_BYE(-1);
  }
  status = is_tbl(docroot, db, dst_tbl, &b_is_tbl, &dst_tbl_id);
  cBYE(status);
  if ( b_is_tbl == false ) { 
    fprintf(stderr, "tbl [%s] not found\n", src_tbl); go_BYE(-1);
  }

  status = fld_meta(docroot, db, src_tbl, src_lnk, -1, &src_lnk_meta);
  cBYE(status);
  if ( ( src_val != NULL ) && ( *src_val != '\0' ) ) { 
    status = fld_meta(docroot, db, src_tbl, src_val, -1, &src_val_meta);
    cBYE(status);
  }
  status = fld_meta(docroot, db, dst_tbl, dst_lnk, -1, &dst_lnk_meta);
  cBYE(status);
  status = internal_get_nR(db, src_tbl_id, &src_nR);
  cBYE(status);
  status = internal_get_nR(db, dst_tbl_id, &dst_nR);
  cBYE(status);
  //----------------------------------------------------------------
  // Get pointer access to all necessary fields
  status = rs_mmap(src_lnk_meta.filename, &src_lnk_X, &src_lnk_nX, 0); 
  cBYE(status);
  if ( ( src_val != NULL ) && ( *src_val != '\0' ) ) { 
    status = rs_mmap(src_val_meta.filename, &src_val_X, &src_val_nX, 0); 
    cBYE(status);
  }
  status = rs_mmap(dst_lnk_meta.filename, &dst_lnk_X, &dst_lnk_nX, 0); 
  cBYE(status);

  //--------------------------------------------------------
  // Create output data files
  status = open_temp_file(&ofp, &opfile); cBYE(status); 
  fclose_if_non_null(ofp);
  status = open_temp_file(&nn_ofp, &nn_opfile); cBYE(status); 
  fclose_if_non_null(nn_ofp);
  if ( ( src_val != NULL ) && ( *src_val != '\0' ) ) { 
    status = mk_file(opfile, src_val_meta.n_sizeof * dst_nR);
    cBYE(status);
    status = rs_mmap(opfile,    &dst_val_X,    &dst_val_nX,    1); cBYE(status);
  }
  status = mk_file(nn_opfile, sizeof(char)       * dst_nR);
  cBYE(status);
  status = rs_mmap(nn_opfile,    &nn_dst_val_X,    &nn_dst_val_nX,    1); cBYE(status);
  //--------------------------------------------------------
  // Core join 
  if ( ( strcmp(src_lnk_meta.fldtype, "int") == 0 ) && 
       ( strcmp(src_lnk_meta.fldtype, "int") == 0 ) && 
       ( strcmp(dst_lnk_meta.fldtype, "int") == 0 ) ) {
    status = core_srt_join_I_I_I(
				 (int *)src_lnk_X, (int *)src_val_X, src_nR,
				 (int *)dst_lnk_X, (int *)dst_val_X, nn_dst_val_X, 
				 0, dst_nR, ijoin_op, &is_any_null);
  }
  else if ( ( strcmp(src_lnk_meta.fldtype, "int") == 0 ) && 
	    ( strcmp(src_lnk_meta.fldtype, "int") == 0 ) && 
	    ( strcmp(dst_lnk_meta.fldtype, "long long") == 0 ) ) {
    status = core_srt_join_I_I_L(
				 (int *)src_lnk_X, (int *)src_val_X, src_nR,
				 (long long *)dst_lnk_X, (int *)dst_val_X, nn_dst_val_X, 
				 0, dst_nR, ijoin_op, &is_any_null);
  }
  else if ( ( strcmp(src_lnk_meta.fldtype, "int") == 0 ) && 
	    ( strcmp(src_lnk_meta.fldtype, "long long") == 0 ) && 
	    ( strcmp(dst_lnk_meta.fldtype, "int") == 0 ) ) {
    status = core_srt_join_I_L_I(
				 (int *)src_lnk_X, (long long
				   *)src_val_X, src_nR,
				 (int *)dst_lnk_X, (long long *)dst_val_X, nn_dst_val_X, 
				 0, dst_nR, ijoin_op, &is_any_null);
  }
  else if ( ( strcmp(src_lnk_meta.fldtype, "int") == 0 ) && 
	    ( strcmp(src_lnk_meta.fldtype, "long long") == 0 ) && 
	    ( strcmp(dst_lnk_meta.fldtype, "long long") == 0 ) ) {
    status = core_srt_join_I_L_L((int *)src_lnk_X, 
	(long long *)src_val_X, src_nR, (long long *)dst_lnk_X, 
	(long long *)dst_val_X, nn_dst_val_X, 0, dst_nR, ijoin_op, 
	&is_any_null);
  }
  else if ( ( strcmp(src_lnk_meta.fldtype, "long long") == 0 ) && 
	    ( strcmp(src_lnk_meta.fldtype, "int") == 0 ) && 
	    ( strcmp(dst_lnk_meta.fldtype, "int") == 0 ) ) {
    status = core_srt_join_L_I_I(
				 (long long *)src_lnk_X, (int
				   *)src_val_X, src_nR,
				 (int *)dst_lnk_X, (int *)dst_val_X, nn_dst_val_X, 
				 0, dst_nR, ijoin_op, &is_any_null);
  }
  else if ( ( strcmp(src_lnk_meta.fldtype, "long long") == 0 ) && 
	    ( strcmp(src_lnk_meta.fldtype, "int") == 0 ) && 
	    ( strcmp(dst_lnk_meta.fldtype, "long long") == 0 ) ) {
    status = core_srt_join_L_I_L(
				 (long long *)src_lnk_X, (int
				   *)src_val_X, src_nR,
				 (long long *)dst_lnk_X, (int *)dst_val_X, nn_dst_val_X, 
				 0, dst_nR, ijoin_op, &is_any_null);
  }
  else if ( ( strcmp(src_lnk_meta.fldtype, "long long") == 0 ) && 
	    ( strcmp(src_lnk_meta.fldtype, "long long") == 0 ) && 
	    ( strcmp(dst_lnk_meta.fldtype, "int") == 0 ) ) {
    status = core_srt_join_L_L_I(
				 (long long *)src_lnk_X, (long long
				   *)src_val_X, src_nR,
				 (int *)dst_lnk_X, (long long *)dst_val_X, nn_dst_val_X, 
				 0, dst_nR, ijoin_op, &is_any_null);
  }
  else if ( ( strcmp(src_lnk_meta.fldtype, "long long") == 0 ) && 
	    ( strcmp(src_lnk_meta.fldtype, "long long") == 0 ) && 
	    ( strcmp(dst_lnk_meta.fldtype, "long long") == 0 ) ) {
    status = core_srt_join_L_L_L(
				 (long long *)src_lnk_X, (long long
				   *)src_val_X, src_nR,
				 (long long *)dst_lnk_X, (long long *)dst_val_X, nn_dst_val_X, 
				 0, dst_nR, ijoin_op, &is_any_null);
  }
  else { go_BYE(-1); }
  //--------------------------------------------------------
  // Add output field to meta data 
  if ( ( src_val != NULL ) && ( *src_val != '\0' ) ) { 
    sprintf(str_meta_data,"filename=%s:n_sizeof=%d:fldtype=%s", opfile,
	    src_val_meta.n_sizeof, src_val_meta.fldtype);
    status = add_fld(docroot, db, dst_tbl, dst_fld, str_meta_data);
    cBYE(status);
    if ( is_any_null == true ) {
      status = add_aux_fld(docroot, db, dst_tbl, dst_fld, nn_opfile, "nn");
      cBYE(status);
    }
    else {
      unlink(nn_opfile);
      free_if_non_null(nn_opfile);
    }
  }
  else {
    sprintf(str_meta_data,"filename=%s:n_sizeof=%ld:fldtype=bool", 
	nn_opfile, sizeof(char));
    status = add_fld(docroot, db, dst_tbl, dst_fld, str_meta_data);
    cBYE(status);
  }
 BYE:
  if ( in_db == NULL ) { sqlite3_close(db); }
  rs_munmap(src_val_X, src_val_nX);
  rs_munmap(src_lnk_X, src_lnk_nX);
  rs_munmap(dst_val_X, dst_val_nX);
  rs_munmap(nn_dst_val_X, nn_dst_val_nX);
  rs_munmap(dst_lnk_X, dst_lnk_nX);
  free_if_non_null(nn_opfile);
  free_if_non_null(opfile);
  return(status);
}
Exemplo n.º 20
0
off_t oggedit_write_flac_metadata(DB_FILE *in, const char *fname, const off_t offset, const int num_tags, char **tags)
{
    off_t res;
    char tempname[PATH_MAX] = "";
    ogg_packet **headers = NULL;
    char *vendor = NULL;
    ogg_sync_state oy;
    ogg_sync_init(&oy);

    /* Original file must be writable whichever way we update it */
    FILE *out = fopen(fname, "r+b");
    if (!out) {
        res = OGGEDIT_CANNOT_UPDATE_FILE;
        goto cleanup;
    }

    /* See if we can write the tags packet directly into the existing file ... */
    if (!(headers = metadata_block_packets(in, &oy, offset, &vendor, (int *)&res)))
        goto cleanup;
    const off_t stream_size_k = in->vfs->getlength(in) / 1000; // use file size for now
    const size_t metadata_size = 4 + vc_size(vendor, num_tags, tags);
    ptrdiff_t padding = headers[0]->bytes - metadata_size;
    if (stream_size_k < 1000 || padding < 0 || (headers[1] && padding > 0) || padding > stream_size_k+metadata_size) {
        res = open_temp_file(fname, tempname, &out);
        if (res) {
            goto cleanup;
        }
    }

    /* Re-pad if writing the whole file */
    if (*tempname) {
        size_t i = 0, j = 0;
        while (headers[i]) {
            headers[j++] = headers[i];
            while (headers[++i] && (headers[i]->packet[0] & 0x3F) == PADTYPE) {
                ogg_packet_clear(headers[i]);
                free(headers[i]);
            }
        }
        headers[j] = NULL;
        padding = headers[1] || stream_size_k < 900 ? 0 : stream_size_k < 10000 ? 1024 : stream_size_k < 100000 ? 8192 : 65536;
    }

    /* Write pages until we reach the correct comment header */
    ogg_page og;
    const int flac_serial = copy_up_to_codec(in, out, &oy, &og, *tempname ? 0 : offset, offset, FLACNAME);
    if (flac_serial <= OGGEDIT_EOF) {
        res = flac_serial;
        goto cleanup;
    }
    if ((res = copy_up_to_header(in, out, &oy, &og, flac_serial)) <= OGGEDIT_EOF)
        goto cleanup;
    const long pageno = write_metadata_block_packets(out, flac_serial, vendor, num_tags, tags, padding, headers);
    if (pageno < OGGEDIT_EOF) {
        res = pageno;
        goto cleanup;
    }

    /* If we have tempfile, copy the remaining pages */
    if (*tempname) {
        if ((res = copy_remaining_pages(in, out, &oy, flac_serial, pageno)) <= OGGEDIT_EOF)
            goto cleanup;
        if (rename(tempname, fname)) {
            res = OGGEDIT_RENAME_FAILED;
            goto cleanup;
        }
    }

    res = file_size(fname);

cleanup:
    clear_header_list(headers);
    cleanup(in, out, &oy, vendor);
    if (res < OGGEDIT_OK)
        unlink(tempname);
    return res;
}
Exemplo n.º 21
0
/* TODO: This routine needs more work to deal with escaping dquotes and
 * backslashes and dealing with eoln within the quotes */
int
main(
    int argc,
    char **argv
    )
{
  int status = 0;
  char *infile = NULL;
  char *outfile_str = NULL;
  char *outfile_sz = NULL;
  char *outfile_nn = NULL;
  FILE *ofp = NULL, *szfp = NULL, *nnfp = NULL;
  char *X = NULL; size_t nX = 0;
  long long nR = 0, start_loc = -1, stop_loc =-1, len,fld_len;
  char nullc = '\0';
  bool is_in_str, pr_str;
  char c_one = 1; char c_zero = 0;

  if ( argc != 2 ) { go_BYE(-1); }
  infile      = argv[1];

  status = open_temp_file(&ofp, &outfile_str, 0); cBYE(status);
  status = open_temp_file(&szfp, &outfile_sz, 0); cBYE(status);
  status = open_temp_file(&nnfp, &outfile_nn, 0); cBYE(status);

  status = rs_mmap(infile, &X, &nX, 0);
  cBYE(status);
  nR = 0;
  is_in_str = false;
  pr_str = false;
  for ( size_t i = 0; i < nX; i++ ) {
    if ( X[i] == '"' ) {
      if ( is_in_str == false ) { 
        is_in_str = true;
        start_loc = i+1;
      }
      else {
        is_in_str = false;
        stop_loc = i-1;
	pr_str = true;
      }
    }
    if ( pr_str == true ) { 
      if ( ( stop_loc < 0 ) || ( start_loc < 0 ) || 
	  ( ( start_loc - stop_loc ) > 1 ) ) { 
	  fprintf(stderr, "Error at line %lld, position %lld of %lld \n", 
	      nR, (long long)i, (long long)nX);
	  go_BYE(-1);
      }
      nR++;
      pr_str = false;
      len = stop_loc - start_loc + 1; // + 1 bcos both are inclusive
      if ( len > 0 ) { 
        fwrite(X+start_loc, sizeof(char), len, ofp);
      }
      if ( len > 0 ) { 
        fwrite(&c_one, sizeof(char), 1, nnfp);
      }
      else {
        fwrite(&c_zero, sizeof(char), 1, nnfp);
      }
      fwrite(&nullc, sizeof(char), 1, ofp);
      fld_len = len + 1;
      fwrite(&fld_len, sizeof(int), 1, szfp);
      // Consume the eoln if it exists
      if ( i+1 < nX ) {
	if ( X[i+1] != '\n' ) { 
	  fprintf(stderr, "Error at line %lld, position %lld of %lld \n", 
	      nR, (long long)i, (long long)nX);
	  go_BYE(-1); 
	} 
	i++;
      }
    }
  }
  fprintf(stdout, "%lld:%s:%s:%s", nR, outfile_str, outfile_sz, outfile_nn);
BYE:
  fclose_if_non_null(ofp);
  fclose_if_non_null(szfp);
  fclose_if_non_null(nnfp);
  rs_munmap(X, nX);
  return(status);
}
Exemplo n.º 22
0
//---------------------------------------------------------------
// START FUNC DECL
int 
xfer_string(
	    FLD_TYPE *dst_idx_meta,
	    char *src_fld_X,
	    char *sz_src_fld_X,
	    long long src_nR,
	    char *dst_tbl,
	    char *dst_fld,
	    char *dst_idx_X,
	    char *nn_dst_idx_X,
	    long long dst_nR
	    )
// STOP FUNC DECL
{
  int status = 0;
  char str_meta_data[256];

  char *szX = NULL; size_t n_szX= 0; int *szptr = NULL;
  char *nnX = NULL; size_t n_nnX= 0; char *nnptr = NULL;

  int dst_fld_id, nn_dst_fld_id, sz_dst_fld_id;

  FILE *ofp    = NULL; char *opfile    = NULL;
  FILE *nn_ofp = NULL; char *nn_opfile = NULL;
  FILE *sz_ofp = NULL; char *sz_opfile = NULL;

  long long *offset_src_fld = NULL;
  long long *ll_dst_idx = NULL;
  bool is_any_null = false; 

  //--------------------------------------------------------
  status = open_temp_file(&ofp, &opfile, 0); cBYE(status);
  /* We cannot allocate storage for this since we do not know how much
   * to allocate a-priori */
  /* Open output fields for size and nn in mmmap mode */
  long long filesz = dst_nR * sizeof(int);
  status = open_temp_file(&sz_ofp, &sz_opfile, filesz); cBYE(status);
  fclose_if_non_null(sz_ofp);
  status = mk_file(sz_opfile, filesz); cBYE(status);
  status = rs_mmap(sz_opfile, &szX, &n_szX, 1); cBYE(status);
  szptr = (int *)szX;

  long long nnfilesz = dst_nR * sizeof(char);
  status = open_temp_file(&nn_ofp, &nn_opfile, nnfilesz); cBYE(status);
  fclose_if_non_null(nn_ofp);
  status = mk_file(nn_opfile, nnfilesz); cBYE(status);
  status = rs_mmap(nn_opfile, &nnX, &n_nnX, 1); cBYE(status);
  nnptr = (char *)nnX;
  /*-------------------------------------------------------------*/
  status = mk_offset((int *)sz_src_fld_X, src_nR, &offset_src_fld);
  cBYE(status);
  /* Convert from int to longlong if needed */
  if ( strcmp(dst_idx_meta->fldtype, "int") == 0 ) {
    ll_dst_idx = malloc(dst_nR * sizeof(long long));
    return_if_malloc_failed(ll_dst_idx);
    if ( dst_nR >= INT_MAX ) {
      fprintf(stderr, "TO BE IMPLEMENTED\n"); go_BYE(-1);
    }
    conv_int_to_longlong((int *)dst_idx_X, (int)dst_nR, ll_dst_idx);
  }
  else {
    ll_dst_idx = (long long *)dst_idx_X;
  }
  /*-------------------------------------------------------------*/
  for ( long long i = 0; i < dst_nR; i++ ) {
    int *i_sz_src_fld_ptr = (int *)sz_src_fld_X;
    long long idx;
    size_t nw;
    char *testptr;
    char nullc = 0;
    long long offset; int sz;
    if ( ( nn_dst_idx_X != NULL ) && ( nn_dst_idx_X[i] == FALSE ) ) { 
      // Output null value
      is_any_null = true;
      nnptr[i] = FALSE;
      szptr[i] = 1;
      nw = fwrite(&nullc, sizeof(char), 1,  ofp); // Null terminate
      if ( nw != 1 ) { go_BYE(-1); }
    }
    else { 
      idx = ll_dst_idx[i];
      if ( ( idx < 0 ) || ( idx >= src_nR ) ) { go_BYE(-1); }
      sz = i_sz_src_fld_ptr[idx];
      offset = offset_src_fld[idx];
      // This checks that strings are null terminated
      testptr = src_fld_X + offset + sz - 1;
      if ( *testptr != '\0' ) {
	go_BYE(-1);
      }
      szptr[i] = sz;
      nnptr[i] = TRUE;
      nw = fwrite(src_fld_X + offset, sizeof(char), sz, ofp);
      if ( nw != sz ) { go_BYE(-1); }
    }
  }
  fclose_if_non_null(ofp);
  rs_munmap(szX, n_szX);
  rs_munmap(nnX, n_nnX);


  // Add output field to meta data 
  zero_string(str_meta_data, 256);
  strcpy(str_meta_data, "fldtype=char string:n_sizeof=0");
  strcat(str_meta_data, ":filename="); 
  strcat(str_meta_data, opfile);
  status = add_fld(dst_tbl, dst_fld, str_meta_data, &dst_fld_id); cBYE(status);
  // Add size field 
  status = add_aux_fld(dst_tbl, dst_fld, sz_opfile, "sz", &sz_dst_fld_id);
  cBYE(status);
  if ( is_any_null ) {
    status = add_aux_fld(dst_tbl, dst_fld, nn_opfile, "nn", &nn_dst_fld_id);
    cBYE(status);
  }
  else {
    unlink(nn_opfile);
  }
 BYE:
  if ( strcmp(dst_idx_meta->fldtype, "int") == 0 ) {
    free_if_non_null(ll_dst_idx);
  }
  fclose_if_non_null(ofp);
  fclose_if_non_null(nn_ofp);
  fclose_if_non_null(sz_ofp);

  free_if_non_null(opfile);
  free_if_non_null(nn_opfile);
  free_if_non_null(sz_opfile);
  free_if_non_null(offset_src_fld);
  return(status);
}
Exemplo n.º 23
0
//---------------------------------------------------------------
// START FUNC DECL
int 
crossprod(
	  char *t1,
	  char *f1,
	  char *t2,
	  char *f2,
	  char *t3
	  )
// STOP FUNC DECL
{
  int status = 0;

  char *Y1 = NULL; size_t nY1 = 0;
  char *Y2 = NULL; size_t nY2 = 0;

  char *f1_X = NULL; size_t f1_nX = 0; char *f1_opfile = NULL; 
  int f1type, f2type;
  char *f2_X = NULL; size_t f2_nX = 0; char *f2_opfile = NULL; 
  long long f1size, f2size;

  int t2f1_fld_id = INT_MIN, t2f2_fld_id = INT_MIN;
  int t3_id = INT_MIN, itemp; 
  long long chk_nR1 = 0, nR1, nR2, nR3;
  char str_meta_data[1024];
  char *t3f1_opfile = NULL, *t3f2_opfile = NULL;
  FILE *ofp = NULL;
  char buffer[32];
  // For multi-threading 
  int rc; // result code for thread create 
  pthread_t threads[MAX_NUM_THREADS];
  pthread_attr_t attr;
  void *thread_status;
  //----------------------------------------------------------------
  zero_string(str_meta_data, 1024);
  zero_string(buffer, 32);
  if ( strcmp(f1, f2) == 0 ) { go_BYE(-1); } 
  /* Remove f1 != f2 restriction later. To do so, we need to specify
   * fields of t3 explicitly */
  //----------------------------------------------------------------
  status = get_data(t1, f1, &nR1, &f1_X, &f1_nX, &f1_opfile, &f1type, &f1size);
  cBYE(status);
  status = get_data(t2, f2, &nR2, &f2_X, &f2_nX, &f2_opfile,  &f2type, &f2size);
  cBYE(status);
  nR3 = nR1 * nR2;
  if ( nR3 == 0 ) {
    fprintf(stderr, "No data to create t3 \n");
    goto BYE;
  }
  // Create storage for field 1 in Table t3 */
  long long  filesz = nR3 * f1size;
  status = open_temp_file(&ofp, &t3f1_opfile, filesz); cBYE(status);
  fclose_if_non_null(ofp);
  status = mk_file(t3f1_opfile, filesz); cBYE(status);
  status = rs_mmap(t3f1_opfile, &Y1, &nY1, 1); cBYE(status);
  // Create storage for field 2 in Table t3 */
  filesz = nR3 * f2size;
  status = open_temp_file(&ofp, &t3f2_opfile, filesz); cBYE(status);
  fclose_if_non_null(ofp);
  status = mk_file(t3f2_opfile, filesz); cBYE(status);
  status = rs_mmap(t3f2_opfile, &Y2, &nY2, 1); cBYE(status);
  //----------------------------------------------------------------
  /* Set up parallelism computations. Parallelization strategy is
   * simple. Partition field 1 (nR1 rows) among the threads */
  g_nR1 = nR1;
  g_nR2 = nR2;
  g_nR3 = nR3;
  g_f1type = f1type;
  g_f2type = f2type;
  g_f1size = f1size;
  g_f2size = f2size;
  g_f1_X = f1_X;
  g_f2_X = f2_X;
  g_Y1 = Y1;
  g_Y2 = Y2;
 
  for ( int i = 0; i < MAX_NUM_THREADS; i++ ) { 
    g_thread_id[i] = i;
    g_num_rows_processed[i] = 0;
  }
  status = get_num_threads(&g_nT);
  cBYE(status);
  //--------------------------------------------
#define MIN_ROWS_FOR_CROSSPROD 4 // 1024
  if ( nR1 <= MIN_ROWS_FOR_CROSSPROD ) {
    g_nT = 1;
  }
  /* Don't create more threads than you can use */
  if ( g_nT > nR1 ) { g_nT = nR1; }

  if ( g_nT == 1 ) { 
    core_crossprod(&(g_thread_id[0]));
    chk_nR1 = g_num_rows_processed[0];
  }
  else { /* Create threads */
    pthread_attr_init(&attr);
    pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
    for ( int t = 0; t < g_nT; t++ ) { 
      rc = pthread_create(&threads[t], NULL, core_crossprod,
	  &(g_thread_id[t]));
      if ( rc ) { go_BYE(-1); }
    }
    /* Free attribute and wait for the other threads */
    pthread_attr_destroy(&attr);
    for ( int t = 0; t < g_nT; t++ ) { 
      rc = pthread_join(threads[t], &thread_status);
      if ( rc ) { go_BYE(-1); }
      chk_nR1 += g_num_rows_processed[t];
    }
  }
  if ( chk_nR1 != nR1 ) { go_BYE(-1); }
  //----------------------------------------------------------------
  // Add output fields to t3 meta data 
  status = is_tbl(t3, &t3_id); cBYE(status);
  if ( t3_id >= 0 ) { 
    status = del_tbl(NULL, t3_id);
    cBYE(status);
  }
  sprintf(buffer, "%lld", nR3);
  status = add_tbl(t3, buffer, &itemp); cBYE(status);

  sprintf(str_meta_data, "fldtype=int:n_sizeof=%u:filename=%s",
	  f1size, t3f1_opfile);
  status = add_fld(t3, f1, str_meta_data, &t2f1_fld_id); cBYE(status);
  zero_string(str_meta_data, 1024);
  sprintf(str_meta_data, "fldtype=int:n_sizeof=%u:filename=%s", 
	  f2size, t3f2_opfile);
  status = add_fld(t3, f2, str_meta_data, &t2f2_fld_id); cBYE(status);
 BYE:
  fclose_if_non_null(ofp);
  rs_munmap(f1_X, f1_nX);
  rs_munmap(f2_X, f2_nX);
  if ( f1_opfile != NULL ) { 
    unlink(f1_opfile); free_if_non_null(f1_opfile);
  }
  if ( f2_opfile != NULL ) { 
    unlink(f2_opfile); free_if_non_null(f2_opfile);
  }
  free_if_non_null(t3f1_opfile);
  free_if_non_null(t3f2_opfile);
  return(status);
}
Exemplo n.º 24
0
//---------------------------------------------------------------
// START FUNC DECL
int 
dup_fld(
       char *tbl,
       char *f1,
       char *f2
       )
// STOP FUNC DECL
{
  int status = 0;
  char *Y = NULL; size_t nY = 0;
  char *X = NULL; size_t nX = 0;
  char *nn_X = NULL; size_t nn_nX = 0;
  char *sz_X = NULL; size_t sz_nX = 0;
  FILE *ofp = NULL; char *opfile = NULL;
  FILE *nn_ofp = NULL; char *nn_opfile = NULL;
  FILE *sz_ofp = NULL; char *sz_opfile = NULL;
  int tbl_id, f1_fld_id, f2_fld_id, nn_fld_id, sz_fld_id;
  //----------------------------------------------------------------
  if ( ( tbl == NULL ) || ( *tbl == '\0' ) ) { go_BYE(-1); }
  if ( ( f1 == NULL ) || ( *f1 == '\0' ) ) { go_BYE(-1); }
  if ( ( f2 == NULL ) || ( *f2 == '\0' ) ) { go_BYE(-1); }
  if ( strcmp(f1, f2) == 0 ) { go_BYE(-1); } 
  //-------------------------------------------------------- 
  status = is_tbl(tbl, &tbl_id); cBYE(status); 
  chk_range(tbl_id, 0, g_n_tbl); 
  status = is_fld(NULL, tbl_id, f1, &f1_fld_id); cBYE(status); 
  chk_range(f1_fld_id, 0, g_n_fld); 
  status = is_fld(NULL, tbl_id, f2, &f2_fld_id); cBYE(status); 
  if ( f2_fld_id >= 0 ) { 
    status = del_fld(NULL, tbl_id, NULL, f2_fld_id); cBYE(status);
  }
  // Open input file 
  status = rs_mmap(g_fld[f1_fld_id].filename, &X, &nX, 0); cBYE(status); 
  // Create a copy of the data 
  status = open_temp_file(&ofp, &opfile, nX); cBYE(status); 
  fclose_if_non_null(ofp); 
  status = mk_file(opfile, nX); cBYE(status);
  status = rs_mmap(opfile, &Y, &nY, 1);
  memcpy(Y, X, nX);
  rs_munmap(Y, nY);
  // Get empty spot for f2
  status = get_empty_fld(&f2_fld_id); cBYE(status);
  // f2 is same as f1 except for data and "is_external"
  g_fld[f2_fld_id] = g_fld[f1_fld_id];
  g_fld[f2_fld_id].is_external = false;
  g_fld[f2_fld_id].nn_fld_id = -1;
  g_fld[f2_fld_id].sz_fld_id = -1;
  status = chk_fld_name(f2, 0);
  zero_string(g_fld[f2_fld_id].name, MAX_LEN_FLD_NAME+1);
  zero_string(g_fld[f2_fld_id].filename, MAX_LEN_FILE_NAME+1);
  strcpy(g_fld[f2_fld_id].name, f2);
  strcpy(g_fld[f2_fld_id].filename, opfile);
  // Add to hash table 
  //--------------------------------------------------------
  // Add auxiliary field nn if if it exists
  nn_fld_id = g_fld[f1_fld_id].nn_fld_id;
  if ( nn_fld_id >= 0 ) { 
    // open input file 
    status = rs_mmap(g_fld[nn_fld_id].filename, &nn_X, &nn_nX, 0); cBYE(status); 
    // create copy of data 
    status = open_temp_file(&nn_ofp, &nn_opfile, nn_nX); cBYE(status);
    fclose_if_non_null(nn_ofp);
    status = mk_file(nn_opfile, nn_nX); cBYE(status);
    status = rs_mmap(nn_opfile, &Y, &nY, 1);
    memcpy(Y, nn_X, nn_nX);
    rs_munmap(Y, nY);
    rs_munmap(nn_X, nn_nX);
    nn_fld_id = INT_MIN;
    status = add_aux_fld(tbl, f2, nn_opfile, "nn", &nn_fld_id);
    cBYE(status);
  }
  //--------------------------------------------------------
  // Add auxiliary field sz if if it exists
  sz_fld_id = g_fld[f1_fld_id].sz_fld_id;
  if ( sz_fld_id >= 0 ) { 
    status = open_temp_file(&sz_ofp, &sz_opfile, sz_nX); cBYE(status);
    fclose_if_non_null(sz_ofp);
    status = mk_file(sz_opfile, sz_nX); cBYE(status);
    status = rs_mmap(sz_opfile, &Y, &nY, 1);
    status = rs_mmap(g_fld[sz_fld_id].filename, &sz_X, &sz_nX, 0);
    memcpy(Y, sz_X, sz_nX);
    rs_munmap(Y, nY);
    rs_munmap(sz_X, sz_nX);
    status = add_aux_fld(tbl, f2, sz_opfile, "sz", &sz_fld_id);
    cBYE(status);
  }
  //--------------------------------------------------------
BYE:
  rs_munmap(X, nX);
  rs_munmap(nn_X, nn_nX);
  rs_munmap(sz_X, sz_nX);
  rs_munmap(Y, nY);
  fclose_if_non_null(ofp);
  fclose_if_non_null(nn_ofp);
  fclose_if_non_null(sz_ofp);
  free_if_non_null(opfile);
  free_if_non_null(nn_opfile);
  free_if_non_null(sz_opfile);
  return(status);
}
Exemplo n.º 25
0
off_t oggedit_write_vorbis_metadata(DB_FILE *in, const char *fname, const off_t offset, const size_t stream_size, const int num_tags, char **tags)
{
    off_t res;
    char tempname[PATH_MAX] = "";
    char *vendor = NULL;
    ogg_packet codebooks;
    memset(&codebooks, '\0', sizeof(codebooks));
    ogg_sync_state oy;
    ogg_sync_init(&oy);

    /* Original file must be writable whichever way we update it */
    FILE *out = fopen(fname, "r+b");
    if (!out) {
        res = OGGEDIT_CANNOT_UPDATE_FILE;
        goto cleanup;
    }

    /* See if we can write the tags packet directly into the existing file ... */
    const ptrdiff_t tags_packet_size = check_vorbis_headers(in, &oy, offset, &vendor, &codebooks);
    if (tags_packet_size <= OGGEDIT_EOF) {
        res = tags_packet_size;
        goto cleanup;
    }
    const size_t metadata_size = strlen(VCMAGIC) + vc_size(vendor, num_tags, tags) + 1;
    ptrdiff_t padding = tags_packet_size - metadata_size;
    const off_t file_size_k = in->vfs->getlength(in) / 1000;
    const size_t stream_size_k = stream_size ? stream_size / 1000 : file_size_k;
    if (file_size_k < 100 || padding < 0 || padding > file_size_k/10+stream_size_k+metadata_size) {
        res = open_temp_file(fname, tempname, &out);
        if (res) {
            goto cleanup;
        }
    }

    /* Re-pad if writing the whole file */
    if (*tempname)
        padding = stream_size_k < 90 ? 0 : stream_size_k < 1000 ? 128 : stream_size_k < 10000 ? 1024 : 8192;

    /* Write pages until the correct comment header */
    ogg_page og;
    int64_t vorbis_serial = copy_up_to_codec(in, out, &oy, &og, *tempname ? 0 : offset, offset, VORBISNAME);
    if (vorbis_serial <= OGGEDIT_EOF) {
        res = vorbis_serial;
        goto cleanup;
    }
    vorbis_serial = copy_up_to_header(in, out, &oy, &og, vorbis_serial);
    if (vorbis_serial <= OGGEDIT_EOF) {
        res = vorbis_serial;
        goto cleanup;
    }
    const long pageno = write_vorbis_tags(out, vorbis_serial, vendor, num_tags, tags, padding, &codebooks);
    if (pageno < OGGEDIT_EOF) {
        res = pageno;
        goto cleanup;
    }

    /* If we have tempfile, copy the remaining pages */
    if (*tempname) {
        vorbis_serial = copy_remaining_pages(in, out, &oy, vorbis_serial, pageno);
        if (vorbis_serial <= OGGEDIT_EOF) {
            res = vorbis_serial;
            goto cleanup;
        }
        if (rename(tempname, fname)) {
            res = OGGEDIT_RENAME_FAILED;
            goto cleanup;
        }
    }

    res = file_size(fname);

cleanup:
    ogg_packet_clear(&codebooks);
    cleanup(in, out, &oy, vendor);
    if (res < OGGEDIT_OK)
        unlink(tempname);
    return res;
}
Exemplo n.º 26
0
//---------------------------------------------------------------
// START FUNC DECL
int 
join(
      char *docroot,
      sqlite3 *in_db,
      char *src_tbl,
      char *src_lnk,
      char *src_val,
      char *dst_tbl,
      char *dst_lnk,
      char *dst_fld,
      char *op
      )
// STOP FUNC DECL
{
  int status = 0;
  sqlite3 *db = NULL;
  char *src_val_X = NULL; size_t src_val_nX = 0;
  char *src_lnk_X = NULL; size_t src_lnk_nX = 0;
  char *src_idx_X = NULL; size_t src_idx_nX = 0;

  char *nn_dst_val_X = NULL; size_t nn_dst_val_nX = 0;
  char *dst_val_X = NULL; size_t dst_val_nX = 0;
  char *dst_lnk_X = NULL; size_t dst_lnk_nX = 0;
  char *dst_idx_X = NULL; size_t dst_idx_nX = 0;
  FLD_META_TYPE src_val_meta, src_lnk_meta, src_idx_meta;
  FLD_META_TYPE dst_val_meta, dst_lnk_meta, dst_idx_meta;
  long long src_nR, dst_nR;  // counts all rows 
  char str_meta_data[1024]; 
  int sizeof_src_idx, sizeof_dst_idx;
  int *iptr;
  int ijoin_op;
  char *x_src_tbl = NULL, *x_dst_tbl = NULL;
  FILE *ofp = NULL, *nn_ofp = NULL;
  char *opfile = NULL, *nn_opfile = NULL;
  //----------------------------------------------------------------
  zero_string(str_meta_data, 1024);
  status = mk_mjoin_op(op, &ijoin_op);
  cBYE(status);
  status = open_db_if_needed(docroot, in_db, &db);
  cBYE(status);
  zero_fld_meta(&src_val_meta);
  zero_fld_meta(&src_lnk_meta);
  zero_fld_meta(&src_idx_meta);
  zero_fld_meta(&dst_val_meta);
  zero_fld_meta(&dst_lnk_meta);
  zero_fld_meta(&dst_idx_meta);
  //----------------------------------------------------------------
  /* Given the src_tbl, we create x_src_tbl which has no null 
     values in the link fields and which has an index field called "id"
     with values 0, 1, .... |x_src_tbl| -1 */
  status = elim_null_vals(docroot, db, src_tbl, src_lnk, &x_src_tbl);
  cBYE(status);
  /* Similarly for dst_tbl */
  status = elim_null_vals(docroot, db, dst_tbl, dst_lnk, &x_dst_tbl);
  cBYE(status);
  status = sort_if_necessary(docroot, db, x_src_tbl, src_lnk, "id");
  cBYE(status);
  status = sort_if_necessary(docroot, db, x_dst_tbl, dst_lnk, "id");
  cBYE(status);
  // Get meta-data for all necessary fields 
  status = fld_meta(docroot, db, x_src_tbl, src_lnk, -1, &src_lnk_meta);
  cBYE(status);
  status = fld_meta(docroot, db, x_src_tbl, XXXXXXX, -1, &src_idx_meta);
  cBYE(status);
  status = fld_meta(docroot, db, x_src_tbl, src_val, -1, &src_val_meta);
  cBYE(status);

  status = fld_meta(docroot, db, x_dst_tbl, dst_lnk, -1, &dst_lnk_meta);
  cBYE(status);
  status = fld_meta(docroot, db, x_dst_tbl, XXXXXXX, -1, &dst_idx_meta);
  cBYE(status);
  // Get pointer access to all necessary fields
  status = rs_mmap(src_lnk_meta.filename, &src_lnk_X, &src_lnk_nX, 0); 
  cBYE(status);
  status = rs_mmap(src_idx_meta.filename, &src_idx_X, &src_idx_nX, 0); 
  cBYE(status);
  status = rs_mmap(src_val_meta.filename, &src_val_X, &src_val_nX, 0); 
  cBYE(status);

  status = rs_mmap(dst_lnk_meta.filename, &dst_lnk_X, &dst_lnk_nX, 0); 
  cBYE(status);
  status = rs_mmap(dst_idx_meta.filename, &dst_idx_X, &dst_idx_nX, 0); 
  cBYE(status);

  //--------------------------------------------------------
  // Create output data files
  status = open_temp_file(&ofp, &opfile); cBYE(status); 
  fclose_if_non_null(ofp);
  status = open_temp_file(&nn_ofp, &nn_opfile); cBYE(status); 
  fclose_if_non_null(nn_ofp);
  status = mk_file(opfile, src_val_meta.n_sizeof * dst_nR);
  cBYE(status);
  status = mk_file(nn_opfile, sizeof(char)       * dst_nR);
  cBYE(status);
  status = rs_mmap(opfile, &dst_val_X, &dst_val_nX, 1); cBYE(status);
  status = rs_mmap(nn_opfile, &nn_dst_val_X, &nn_dst_val_nX, 1); cBYE(status);
  //--------------------------------------------------------
  // Core join 
  status = core_join_I_I_I_I_I_I(
      (int *)cpy_src_lnk_X, (int *)cpy_src_idx_X, 
      (int *)src_val_X, 
      src_nR, nn_src_nR,
      (int *)cpy_dst_lnk_X, (int *)cpy_dst_idx_X, 
      (int *)dst_val_X, nn_dst_val_X, 
      dst_nR, nn_dst_nR,
      ijoin_op, &is_any_null);
  //--------------------------------------------------------

  // Clean up aux data structures
  rs_munmap(cpy_src_lnk_X, cpy_src_lnk_nX);
  rs_munmap(cpy_src_idx_X, cpy_src_idx_nX);
  rs_munmap(cpy_dst_lnk_X, cpy_dst_lnk_nX);
  rs_munmap(cpy_dst_idx_X, cpy_dst_idx_nX);

  if ( cpy_src_lnk_file == NULL ) { 
    status = del_fld(docroot, db, src_tbl,"_cpy_lnk"); cBYE(status);
  }
  else {
    unlink(cpy_src_lnk_file); free_if_non_null(cpy_src_lnk_file);
  }
  if ( cpy_src_idx_file == NULL ) { 
    status = del_fld(docroot, db, src_tbl,"_cpy_idx"); cBYE(status);
  }
  else {
    unlink(cpy_src_idx_file); free_if_non_null(cpy_src_idx_file);
  }
  if ( cpy_dst_lnk_file == NULL ) { 
    status = del_fld(docroot, db, dst_tbl,"_cpy_lnk"); cBYE(status);
  }
  else {
    unlink(cpy_dst_lnk_file); free_if_non_null(cpy_dst_lnk_file);
  }
  if ( cpy_dst_idx_file == NULL ) { 
    status = del_fld(docroot, db, dst_tbl,"_cpy_idx"); cBYE(status);
  }
  else {
    unlink(cpy_dst_idx_file); free_if_non_null(cpy_dst_idx_file);
  }
  // Add output field to meta data 
  sprintf(str_meta_data,"filename=%s:n_sizeof=%d:fldtype=%s", opfile,
      src_val_meta.n_sizeof, src_val_meta.fldtype);
  status = add_fld(docroot, db, dst_tbl, dst_fld, str_meta_data);
  cBYE(status);
  if ( is_any_null == true ) {
    status = add_aux_fld(docroot, db, dst_tbl, dst_fld, nn_opfile, "nn");
    cBYE(status);
  }
  else {
    unlink(nn_opfile);
    free_if_non_null(nn_opfile);
  }
 BYE:
  if ( in_db == NULL ) { sqlite3_close(db); }
  rs_munmap(dst_val_X, dst_nX);
  rs_munmap(nn_dst_val_X, nn_dst_nX);

  rs_munmap(src_val_X, src_val_nX);
  rs_munmap(src_lnk_X, src_lnk_nX);
  rs_munmap(src_lnk_X, src_lnk_nX);

  rs_munmap(dst_lnk_X, dst_lnk_nX);
  rs_munmap(dst_idx_X, dst_idx_nX);

  free_if_non_null(nn_opfile);
  free_if_non_null(opfile);

  return(status);
}
Exemplo n.º 27
0
//---------------------------------------------------------------
// START FUNC DECL
int 
num_in_range(
	       char *t1,
	       char *f1,
	       char *t2,
	       char *lb,
	       char *ub,
	       char *cnt
	       )
// STOP FUNC DECL
{
  int status = 0;
  char *f1_X = NULL; size_t f1_nX = 0;
  char *lb_X = NULL; size_t lb_nX = 0;
  char *ub_X = NULL; size_t ub_nX = 0;
  char *cnt_X = NULL; size_t cnt_nX = 0;
  int t1_id = INT_MIN, t2_id = INT_MIN;
  int f1_id = INT_MIN, lb_id = INT_MIN, ub_id = INT_MIN, cnt_id = INT_MIN;
  FLD_TYPE *f1_meta = NULL, *lb_meta = NULL, *ub_meta = NULL;
  long long nR1 = INT_MIN, nR2 = INT_MIN, chk_nR1 = INT_MIN;
  long long **cntptrs = NULL;
  // For multi-threading 
  int nT;
  int rc; // result code for thread create 
  pthread_t threads[MAX_NUM_THREADS];
  pthread_attr_t attr;
  void *thread_status;

  char str_meta_data[1024];
  char *opfile = NULL; FILE *ofp = NULL;
  //----------------------------------------------------------------
  if ( ( t1 == NULL ) || ( *t1 == '\0' ) ) { go_BYE(-1); }
  if ( ( f1 == NULL ) || ( *f1 == '\0' ) ) { go_BYE(-1); }
  if ( ( t2 == NULL ) || ( *t2 == '\0' ) ) { go_BYE(-1); }
  if ( ( lb == NULL ) || ( *lb == '\0' ) ) { go_BYE(-1); }
  if ( ( ub == NULL ) || ( *ub == '\0' ) ) { go_BYE(-1); }
  if ( ( cnt  == NULL ) || ( *cnt == '\0' ) ) { go_BYE(-1); }
  if ( strcmp(t1, t2) == 0 ) { go_BYE(-1); }
  zero_string(str_meta_data, 1024);
  //--------------------------------------------------------
  status = is_tbl(t1, &t1_id); cBYE(status);
  chk_range(t1_id, 0, g_n_tbl);
  nR1 = g_tbl[t1_id].nR;
  //--------------------------------------------------------
  status = is_fld(NULL, t1_id, f1, &f1_id); cBYE(status);
  chk_range(f1_id, 0, g_n_fld);
  f1_meta = &(g_fld[f1_id]);
  status = rs_mmap(f1_meta->filename, &f1_X, &f1_nX, 0); cBYE(status);
  // Have not implemented case where f1 has null field 
  if ( f1_meta->nn_fld_id >= 0 ) { go_BYE(-1); }
  // Have implemented only for int 
  if ( strcmp(f1_meta->fldtype, "int") != 0 ) { cBYE(-1); } 
  //--------------------------------------------------------
  status = is_tbl(t2, &t2_id); cBYE(status);
  chk_range(t2_id, 0, g_n_tbl);
  nR2 = g_tbl[t2_id].nR;
  //--------------------------------------------------------
  status = is_fld(NULL, t2_id, lb, &lb_id); cBYE(status);
  chk_range(lb_id, 0, g_n_fld);
  lb_meta = &(g_fld[lb_id]);
  status = rs_mmap(lb_meta->filename, &lb_X, &lb_nX, 0); cBYE(status);
  // Have not implemented case where lb has null field 
  if ( lb_meta->nn_fld_id >= 0 ) { go_BYE(-1); }
  // Have implemented only for int 
  if ( strcmp(lb_meta->fldtype, "int") != 0 ) { cBYE(-1); } 
  //--------------------------------------------------------
  status = is_fld(NULL, t2_id, ub, &ub_id); cBYE(status);
  chk_range(ub_id, 0, g_n_fld);
  ub_meta = &(g_fld[ub_id]);
  status = rs_mmap(ub_meta->filename, &ub_X, &ub_nX, 0); cBYE(status);
  // Have not implemented case where ub has null field 
  if ( ub_meta->nn_fld_id >= 0 ) { go_BYE(-1); }
  // Have implemented only for int 
  if ( strcmp(ub_meta->fldtype, "int") != 0 ) { cBYE(-1); } 
  //--------------------------------------------------------
  // Set up access to input
  int *inptr = (int *)f1_X;
  int *lbptr = (int *)lb_X;
  int *ubptr = (int *)ub_X;
  //--------------------------------------------------------
  //--- Decide on how much parallelism to use
  for ( int i = 0; i < MAX_NUM_THREADS; i++ ) { 
    g_thread_id[i] = i;
    g_num_rows[i] = 0;
  }
  status = get_num_threads(&nT);
  cBYE(status);
  //--------------------------------------------
#define MIN_ROWS_FOR_SUBSAMPLE 10000 // 1048576
  if ( nR1 <= MIN_ROWS_FOR_SUBSAMPLE ) {
    nT = 1;
  }
  /* Don't create more threads than you can use */
  if ( nT > nR1 ) { nT = nR1; }
  //--------------------------------------------
  /* Make space for output */
  long long filesz = nR2 * sizeof(long long);
  status = open_temp_file(&ofp, &opfile, filesz); cBYE(status);
  fclose_if_non_null(ofp);
  status = mk_file(opfile, filesz); cBYE(status);
  status = rs_mmap(opfile, &cnt_X, &cnt_nX, 1);
  long long *cntptr = (long long *)cnt_X;
  /* Make a holding tank for partial results */
  cntptrs = malloc(nT * sizeof(long long *));
  return_if_malloc_failed(cntptrs);
  for ( int i = 0; i < nT; i++ ) {
    cntptrs[i] = malloc(nR2 * sizeof(long long));
    return_if_malloc_failed(cntptrs[i]);
    for ( long long j = 0; j <nR2; j++ ) { 
      cntptrs[i][j] = 0;
    }
  }

  // Add count field to meta data 
  sprintf(str_meta_data, "fldtype=long long:n_sizeof=8:filename=%s", opfile);
  status = add_fld(t2, cnt, str_meta_data, &cnt_id); cBYE(status);
  chk_range(cnt_id, 0, g_n_fld);
  //-----------------------------------------------------------
  // Now we count how much there is in each range 
  // Set up global variables
  g_nT = nT;
  g_inptr = inptr;
  g_lbptr = lbptr;
  g_ubptr = ubptr;
  g_cntptrs = cntptrs;
  g_nR1 = nR1;
  g_nR2 = nR2;
  if ( g_nT == 1 ) { 
    core_num_in_range(&(g_thread_id[0]));
    chk_nR1 = g_num_rows[0];
  }
  else {
    chk_nR1 = 0;
    pthread_attr_init(&attr);
    pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
    for ( int t = 0; t < g_nT; t++ ) { 
      rc = pthread_create(&threads[t], NULL, core_num_in_range,
	  &(g_thread_id[t]));
      if ( rc ) { go_BYE(-1); }
    }
    /* Free attribute and wait for the other threads */
    pthread_attr_destroy(&attr);
    for ( int t = 0; t < g_nT; t++ ) { 
      rc = pthread_join(threads[t], &thread_status);
      if ( rc ) { go_BYE(-1); }
      chk_nR1 += g_num_rows[t];
    }
  }
  if ( chk_nR1 != nR1 ) { go_BYE(-1); }
  // Accumulate partial results
  for ( long long i = 0; i < nR2; i++ ) { 
    cntptr[i] = 0;
    for ( int j= 0;  j < nT; j++ ) { 
      cntptr[i] += cntptrs[j][i];
    }
  }

 BYE:
  g_write_to_temp_dir = false;
  rs_munmap(f1_X, f1_nX);
  rs_munmap(lb_X, lb_nX);
  rs_munmap(ub_X, ub_nX);
  rs_munmap(cnt_X, cnt_nX);
  free_if_non_null(opfile);
  return(status);
}
Exemplo n.º 28
0
/*
   Example multipart form-data request content format:

   --AaB03x
   Content-Disposition: form-data; name="submit-name"

   Larry
   --AaB03x
   Content-Disposition: form-data; name="file"; filename="file1.txt"
   Content-Type: text/plain

   ... contents of file1.txt ...
   --AaB03x--
*/
static evhtp_res
upload_read_cb (evhtp_request_t *req, evbuf_t *buf, void *arg)
{
    RecvFSM *fsm = arg;
    char *line;
    size_t len;
    gboolean no_line = FALSE;
    int res = EVHTP_RES_OK;

    if (fsm->state == RECV_ERROR)
        return EVHTP_RES_OK;

    /* Update upload progress. */
    fsm->progress->uploaded += (gint64)evbuffer_get_length(buf);

    seaf_debug ("progress: %lld/%lld\n",
                fsm->progress->uploaded, fsm->progress->size);

    evbuffer_add_buffer (fsm->line, buf);
    /* Drain the buffer so that evhtp don't copy it to another buffer
     * after this callback returns. 
     */
    evbuffer_drain (buf, evbuffer_get_length (buf));

    while (!no_line) {
        switch (fsm->state) {
        case RECV_INIT:
            line = evbuffer_readln (fsm->line, &len, EVBUFFER_EOL_CRLF_STRICT);
            if (line != NULL) {
                seaf_debug ("[upload] boundary line: %s.\n", line);
                if (!strstr (line, fsm->boundary)) {
                    seaf_warning ("[upload] no boundary found in the first line.\n");
                    free (line);
                    res = EVHTP_RES_BADREQ;
                    goto out;
                } else {
                    fsm->state = RECV_HEADERS;
                    free (line);
                }
            } else {
                no_line = TRUE;
            }
            break;
        case RECV_HEADERS:
            line = evbuffer_readln (fsm->line, &len, EVBUFFER_EOL_CRLF_STRICT);
            if (line != NULL) {
                seaf_debug ("[upload] mime header line: %s.\n", line);
                if (len == 0) {
                    /* Read an blank line, headers end. */
                    free (line);
                    if (g_strcmp0 (fsm->input_name, "file") == 0) {
                        if (open_temp_file (fsm) < 0) {
                            seaf_warning ("[upload] Failed open temp file.\n");
                            res = EVHTP_RES_SERVERR;
                            goto out;
                        }
                    }
                    seaf_debug ("[upload] Start to recv %s.\n", fsm->input_name);
                    fsm->state = RECV_CONTENT;
                } else if (parse_mime_header (line, fsm) < 0) {
                    free (line);
                    res = EVHTP_RES_BADREQ;
                    goto out;
                } else {
                    free (line);
                }
            } else {
                no_line = TRUE;
            }
            break;
        case RECV_CONTENT:
            if (g_strcmp0 (fsm->input_name, "file") == 0)
                res = recv_file_data (fsm, &no_line);
            else
                res = recv_form_field (fsm, &no_line);

            if (res != EVHTP_RES_OK)
                goto out;

            break;
        }
    }

out:
    if (res != EVHTP_RES_OK) {
        /* Don't receive any data before the connection is closed. */
        evhtp_request_pause (req);

        /* Set keepalive to 0. This will cause evhtp to close the
         * connection after sending the reply.
         */
        req->keepalive = 0;

        fsm->state = RECV_ERROR;
    }

    if (res == EVHTP_RES_BADREQ) {
        evhtp_send_reply (req, EVHTP_RES_BADREQ);
    } else if (res == EVHTP_RES_SERVERR) {
        evbuffer_add_printf (req->buffer_out, "Internal server error\n");
        evhtp_send_reply (req, EVHTP_RES_SERVERR);
    }
    return EVHTP_RES_OK;
}
Exemplo n.º 29
0
//---------------------------------------------------------------
// START FUNC DECL
int 
f1f2f3f4opf5(
	     char *tbl,
	     char *f1,
	     char *f2,
	     char *f3,
	     char *f4,
	     char *str_op_spec,
	     char *f5
	     )
// STOP FUNC DECL
{
  int status = 0;

  char *f1_X = NULL; size_t f1_nX = 0;
  char *nn_f1_X = NULL; size_t nn_f1_nX = 0;
  char *f2_X = NULL; size_t f2_nX = 0;
  char *nn_f2_X = NULL; size_t nn_f2_nX = 0;
  char *f3_X = NULL; size_t f3_nX = 0;
  char *nn_f3_X = NULL; size_t nn_f3_nX = 0;
  char *f4_X = NULL; size_t f4_nX = 0;
  char *nn_f4_X = NULL; size_t nn_f4_nX = 0;

  char *out_X = NULL; size_t out_nX = 0;
  char *nn_out_X = NULL; size_t nn_out_nX = 0;

  char *opfile = NULL, *nn_opfile = NULL; FILE *ofp = NULL;

  FLD_TYPE *f1_meta = NULL, *nn_f1_meta = NULL;
  FLD_TYPE *f2_meta = NULL, *nn_f2_meta = NULL;
  FLD_TYPE *f3_meta = NULL, *nn_f3_meta = NULL;
  FLD_TYPE *f4_meta = NULL, *nn_f4_meta = NULL;
  long long nR; 
  int tbl_id = INT_MIN;
  int f1_id = INT_MIN, f2_id = INT_MIN, f3_id = INT_MIN, 
      f4_id = INT_MIN, f5_id = INT_MIN;
  int nn_f1_id = INT_MIN, nn_f2_id = INT_MIN, nn_f3_id = INT_MIN, 
      nn_f4_id = INT_MIN, nn_f5_id = INT_MIN;
  char str_meta_data[1024];
  bool is_any_null = false;
  float *f1ptr = NULL, *f2ptr = NULL, *f3ptr = NULL, *f4ptr = NULL; 
  float *foutptr = NULL; char *nn_out = NULL;
  char *op = NULL;
  //----------------------------------------------------------------
  if ( ( tbl == NULL ) || ( *tbl == '\0' ) ) { go_BYE(-1); }
  if ( ( f1 == NULL ) || ( *f1 == '\0' ) ) { go_BYE(-1); }
  if ( ( f2 == NULL ) || ( *f2 == '\0' ) ) { go_BYE(-1); }
  if ( ( f3 == NULL ) || ( *f3 == '\0' ) ) { go_BYE(-1); }
  if ( ( f4 == NULL ) || ( *f4 == '\0' ) ) { go_BYE(-1); }
  if ( ( f5 == NULL ) || ( *f5 == '\0' ) ) { go_BYE(-1); }
  if ( ( str_op_spec == NULL ) || ( *str_op_spec == '\0' ) ) { go_BYE(-1); }


  zero_string(str_meta_data, 1024);
  //--------------------------------------------------------
  status = extract_name_value(str_op_spec, "op=", ":", &op);
  cBYE(status);
  if ( op == NULL ) { go_BYE(-1); }
  //--------------------------------------------------------
  status = is_tbl(tbl, &tbl_id); cBYE(status);
  chk_range(tbl_id, 0, g_n_tbl);
  nR = g_tbl[tbl_id].nR;
  //--------------------------------------------------------
  /* Get information for field f1 */
  status = is_fld(NULL, tbl_id, f1, &f1_id);
  f1_meta = &(g_fld[f1_id]);
  status = rs_mmap(f1_meta->filename, &f1_X, &f1_nX, 0); cBYE(status);
  nn_f1_id = f1_meta->nn_fld_id;
  if ( nn_f1_id >= 0 ) {
    nn_f1_meta = &(g_fld[nn_f1_id]);
    status = rs_mmap(nn_f1_meta->filename, &nn_f1_X, &nn_f1_nX, 0); 
    cBYE(status);
  }
  //--------------------------------------------------------
  /* Get information for field f2 */
  status = is_fld(NULL, tbl_id, f2, &f2_id);
  f2_meta = &(g_fld[f2_id]);
  status = rs_mmap(f2_meta->filename, &f2_X, &f2_nX, 0); cBYE(status);
  nn_f2_id = f2_meta->nn_fld_id;
  if ( nn_f2_id >= 0 ) {
    nn_f2_meta = &(g_fld[nn_f2_id]);
    status = rs_mmap(nn_f2_meta->filename, &nn_f2_X, &nn_f2_nX, 0); 
    cBYE(status);
  }
  //--------------------------------------------------------
  /* Get information for field f3 */
  status = is_fld(NULL, tbl_id, f3, &f3_id);
  f3_meta = &(g_fld[f3_id]);
  status = rs_mmap(f3_meta->filename, &f3_X, &f3_nX, 0); cBYE(status);
  nn_f3_id = f3_meta->nn_fld_id;
  if ( nn_f3_id >= 0 ) {
    nn_f3_meta = &(g_fld[nn_f3_id]);
    status = rs_mmap(nn_f3_meta->filename, &nn_f3_X, &nn_f3_nX, 0); 
    cBYE(status);
  }
  //--------------------------------------------------------
  /* Get information for field f4 */
  status = is_fld(NULL, tbl_id, f4, &f4_id);
  f4_meta = &(g_fld[f4_id]);
  status = rs_mmap(f4_meta->filename, &f4_X, &f4_nX, 0); cBYE(status);
  nn_f4_id = f4_meta->nn_fld_id;
  if ( nn_f4_id >= 0 ) {
    nn_f4_meta = &(g_fld[nn_f4_id]);
    status = rs_mmap(nn_f4_meta->filename, &nn_f4_X, &nn_f4_nX, 0); 
    cBYE(status);
  }
  //--------------------------------------------------------
  if ( strcmp(f1_meta->fldtype, "float") != 0 ) { go_BYE(-1); }
  if ( strcmp(f2_meta->fldtype, "float") != 0 ) { go_BYE(-1); }
  if ( strcmp(f3_meta->fldtype, "float") != 0 ) { go_BYE(-1); }
  if ( strcmp(f4_meta->fldtype, "float") != 0 ) { go_BYE(-1); }
  //--------------------------------------------------------
  f1ptr = (float *)f1_X;
  f2ptr = (float *)f2_X;
  f3ptr = (float *)f3_X;
  f4ptr = (float *)f4_X;
  //--------------------------------------------------------
  // Create output file 
  long long filesz = sizeof(float) * nR;
  status = open_temp_file(&ofp, &opfile, filesz); cBYE(status); fclose_if_non_null(ofp);
  status = mk_file(opfile, filesz); cBYE(status);
  status = rs_mmap(opfile, &out_X, &out_nX, 1); cBYE(status);
  foutptr = (float*)out_X;
  //--------------------------------------------------------
  // Create nn field for output file 
  long long nnfilesz = sizeof(char) * nR;
  status = open_temp_file(&ofp, &nn_opfile, nnfilesz); cBYE(status); 
  fclose_if_non_null(ofp);
  status = mk_file(nn_opfile, filesz); cBYE(status);
  status = rs_mmap(nn_opfile, &nn_out_X, &nn_out_nX, 1); cBYE(status);
  nn_out = (char*)nn_out_X;
  //--------------------------------------------------------
  if ( strcmp(op, "lat_long_dist") == 0 ) {
    for ( long long i = 0; i < nR; i++ ) { 
      if ( ( nn_f1_X != NULL ) && ( nn_f1_X[i] == 0 ) ) {
	is_any_null = true; foutptr[i] = 0; nn_out[i] = 0; continue;
      }
      if ( ( nn_f2_X != NULL ) && ( nn_f2_X[i] == 0 ) ) {
	is_any_null = true; foutptr[i] = 0; nn_out[i] = 0; continue;
      }
      if ( ( nn_f3_X != NULL ) && ( nn_f3_X[i] == 0 ) ) {
	is_any_null = true; foutptr[i] = 0; nn_out[i] = 0; continue;
      }
      if ( ( nn_f4_X != NULL ) && ( nn_f4_X[i] == 0 ) ) {
	is_any_null = true; foutptr[i] = 0; nn_out[i] = 0; continue;
      }
      if ( ( f1ptr[i] < -90  ) || ( f1ptr[i] > 90  ) ) { go_BYE(-1); }
      if ( ( f2ptr[i] < -180 ) || ( f2ptr[i] > 180 ) ) { go_BYE(-1); }
      if ( ( f3ptr[i] < -90  ) || ( f3ptr[i] > 90  ) ) { go_BYE(-1); }
      if ( ( f4ptr[i] < -180 ) || ( f4ptr[i] > 180 ) ) { go_BYE(-1); }
      status = latlong_distance(f1ptr[i], f2ptr[i], f3ptr[i], f4ptr[i],
				&(foutptr[i]));
      cBYE(status);
      nn_out[i] = TRUE;
    }
  }
  sprintf(str_meta_data, "filename=%s:fldtype=float:n_sizeof=4", opfile);
  status = add_fld(tbl, f5, str_meta_data, &f5_id);
  cBYE(status);
  if ( is_any_null ) {
    status = add_aux_fld(tbl, f5, nn_opfile, "nn", &nn_f5_id);
    cBYE(status);
  }
  else {
    unlink(nn_opfile);
    free_if_non_null(nn_opfile);
  }
BYE:
  rs_munmap(f1_X, f1_nX); rs_munmap(nn_f1_X, nn_f1_nX);
  rs_munmap(f2_X, f2_nX); rs_munmap(nn_f2_X, nn_f2_nX);
  rs_munmap(f3_X, f3_nX); rs_munmap(nn_f3_X, nn_f3_nX);
  rs_munmap(f4_X, f4_nX); rs_munmap(nn_f4_X, nn_f4_nX);
  rs_munmap(out_X, out_nX); rs_munmap(nn_out_X, nn_out_nX);
  free_if_non_null(opfile);
  free_if_non_null(nn_opfile);
  free_if_non_null(op);
  return(status);
}
Exemplo n.º 30
0
int
collect_lnk_idx_vals_int(
    char *src_lnk_X,
    char *nn_src_lnk_X,
    char *nn_src_fld_X,
    int src_nR,
    char *dst_lnk_X,
    char *nn_dst_lnk_X,
    int dst_nR,
      INT_INT_TYPE **ptr_src_pair,
      int *ptr_nn_src_nR,
      INT_INT_TYPE **ptr_dst_pair,
      int *ptr_nn_dst_nR,
      char **ptr_temp_src_X,
      size_t *ptr_temp_src_nX,
      char **ptr_temp_dst_X,
      size_t *ptr_temp_dst_nX
      )
  {
    int status = 0;
    int nn_src_nR, nn_dst_nR;
    INT_INT_TYPE *src_pair = NULL, *dst_pair = NULL;
  int rec_size = sizeof(int) + sizeof(int);
  char *temp_src_file = NULL, *temp_dst_file = NULL;
  FILE *tfp = NULL;
  int *ll_dst_lnk_ptr = NULL, *ll_src_lnk_ptr = NULL;
  char *temp_src_X = NULL; size_t temp_src_nX = 0;
  char *temp_dst_X = NULL; size_t temp_dst_nX = 0;

  /* START: Collect nn source values and sort them on link field */
  status = open_temp_file(&tfp, &temp_src_file, 0);
  cBYE(status);
  nn_src_nR = 0;
  ll_src_lnk_ptr = (int*)src_lnk_X;
  for ( int i = 0; i < src_nR; i++ ) { 
    if ( ( nn_src_fld_X != NULL ) && ( nn_src_fld_X[i] == FALSE ) ) { continue; }
    if ( ( nn_src_lnk_X != NULL ) && ( nn_src_lnk_X[i] == FALSE ) ) { continue; }
    nn_src_nR++;
    fwrite(ll_src_lnk_ptr+i, 1, sizeof(int), tfp);
    fwrite(&i, 1, sizeof(int), tfp);
  }
  fclose_if_non_null(tfp);
  if ( nn_src_nR == 0 ) { /* No values to process */
    go_BYE(-1);
  }
  status = rs_mmap(temp_src_file, &temp_src_X, &temp_src_nX, 1); 
  cBYE(status);
  if ( ( ( temp_src_nX/ sizeof(INT_INT_TYPE) ) * sizeof(INT_INT_TYPE) ) != temp_src_nX) {
    go_BYE(-1);
  }
  qsort(temp_src_X, nn_src_nR, rec_size, sort2_asc_int);
  src_pair = (INT_INT_TYPE *)temp_src_X;
  /* STOP: Collect nn source values and sort them on link field */
  /*------------------------------------------------*/
  /* START: Sort destination link fields */
  status = open_temp_file(&tfp, &temp_dst_file, 0);
  cBYE(status);
  nn_dst_nR = 0;
  ll_dst_lnk_ptr = (int *)dst_lnk_X;
  for ( int  i = 0; i < dst_nR; i++ ) { 
    if ( ( nn_dst_lnk_X != NULL ) && ( nn_dst_lnk_X[i] == FALSE ) ) { continue; }
    nn_dst_nR++;
    fwrite(ll_dst_lnk_ptr+i, 1, sizeof(int), tfp);
    fwrite(&i, 1, sizeof(int), tfp);
  }
  fclose_if_non_null(tfp);
  if ( nn_dst_nR == 0 ) { /* No values to process */
    go_BYE(-1);
  }
  status = rs_mmap(temp_dst_file, &temp_dst_X, &temp_dst_nX, 1); 
  cBYE(status);
  if ( ( ( temp_dst_nX/ sizeof(INT_INT_TYPE) ) * sizeof(INT_INT_TYPE) ) != temp_dst_nX) {
    go_BYE(-1);
  }
  qsort(temp_dst_X, nn_dst_nR, rec_size, sort2_asc_int);
  dst_pair = (INT_INT_TYPE *)temp_dst_X;
  /*------------------------------------------------*/
  /* STOP : Sort destination link fields */
  //--------------------------------------------------------
  *ptr_nn_src_nR = nn_src_nR;
  *ptr_nn_dst_nR = nn_dst_nR;

  *ptr_src_pair = src_pair;
  *ptr_dst_pair = dst_pair;
  *ptr_temp_src_X = temp_src_X;
  *ptr_temp_src_nX = temp_src_nX;
  *ptr_temp_dst_X = temp_dst_X;
  *ptr_temp_dst_nX = temp_dst_nX;
BYE:
  unlink(temp_src_file);
  unlink(temp_dst_file);
  free_if_non_null(temp_src_file);
  free_if_non_null(temp_dst_file);
  return(status);
  }