int main( int argc, char **argv ) { int status = 0; char *buffer = NULL; char filename[256]; size_t bufsize = BUFSIZE; FILE *ifp = NULL, *ofp = NULL, *tfp = NULL; char *infile = NULL, *outfile = NULL, *tempfile = NULL; int nr, lno; if ( argc != 3 ) { fprintf(stderr, "Usage is txt_sub <infile> <outfile> \n"); go_BYE(-1); } infile = argv[1]; outfile = argv[2]; if ( *infile == '\0' ) { go_BYE(-1); } if ( *outfile == '\0' ) { go_BYE(-1); } if ( strcmp(infile, outfile)== 0 ) { go_BYE(-1); } buffer = (char *)malloc(bufsize * sizeof(char)); return_if_malloc_failed(buffer); zero_string(buffer, bufsize); zero_string(filename, 256); ifp = fopen(infile, "r"); return_if_fopen_failed(ifp, infile, "r"); ofp = fopen(outfile, "w"); return_if_fopen_failed(ofp, outfile, "w"); for ( lno = 1; ; lno++ ) { zero_string_to_nullc(buffer); nr = getline(&buffer, &bufsize, ifp); if ( nr <= 0 ) { break; } if ( ( strcasecmp(buffer, "%% INCLUDE") == 0 ) || ( strcasecmp(buffer, "// INCLUDE") == 0 ) ) { status = open_temp_file(&tfp, &tempfile); cBYE(status); fclose_if_non_null(tfp); fprintf(ofp, "\\verbatiminput{%s}\n", tempfile); free_if_non_null(tempfile); } else { // copy from input to output fprintf(ofp, "%s", buffer); } } fclose_if_non_null(ifp); fclose_if_non_null(ofp); BYE: free_if_non_null(buffer); return(status); }
off_t oggedit_write_opus_metadata(DB_FILE *in, const char *fname, const off_t offset, const off_t stream_size, const int output_gain, const uint32_t num_tags, char **tags) { off_t res; char tempname[PATH_MAX] = ""; char *vendor = NULL; ogg_sync_state oy; ogg_sync_init(&oy); /* Original file must be writable whichever way we update it */ FILE *out = fopen(fname, "r+b"); if (!out) { res = OGGEDIT_CANNOT_UPDATE_FILE; goto cleanup; } /* Should we write the tags packet directly into the existing file ... */ const long tags_packet_size = check_opus_header(in, &oy, offset, &vendor); if (tags_packet_size <= OGGEDIT_EOF) { res = tags_packet_size; goto cleanup; } const int64_t metadata_size = strlen(TAGMAGIC) + vc_size(vendor, num_tags, tags); int64_t padding = tags_packet_size - metadata_size; const off_t file_size_k = in->vfs->getlength(in) / 1000; const off_t stream_size_k = stream_size ? stream_size / 1000 : file_size_k; if (file_size_k < 100 || padding < 0 || padding > file_size_k/10+stream_size_k+metadata_size) { res = open_temp_file(fname, tempname, &out); if (res) { goto cleanup; } } /* Re-pad if writing the whole file */ if (*tempname) padding = stream_size_k < 90 ? 0 : stream_size_k < 1000 ? 128 : stream_size_k < 10000 ? 1024 : 8192; /* Write pages until we reach the correct OpusHead, then write OpusTags */ ogg_page og; int64_t opus_serial = copy_up_to_codec(in, out, &oy, &og, *tempname ? 0 : offset, offset, OPUSNAME); if (opus_serial <= OGGEDIT_EOF) { res = opus_serial; goto cleanup; } if (output_gain > INT_MIN) { og.body[16] = output_gain & 0xFF; og.body[17] = output_gain >> 8 & 0xFF; ogg_page_checksum_set(&og); }
FileHolder Database_ResourceStorage::get_file_path(const std::string& key) { int ind = find_in_cache(key); if(ind >= 0) return FileCache[ind].file; guint32 entry_offset, entry_size; if(!ridx_file->lookup(key.c_str(), entry_offset, entry_size)) return FileHolder(); // key not found gchar *data = dict->GetData(entry_offset, entry_size); if(!data) return FileHolder(); std::string name_pattern; // in file name encoding if(!utf8_to_file_name(key, name_pattern)) return FileHolder(); std::string::size_type pos = name_pattern.find_last_of("."DB_DIR_SEPARATOR_S); if(pos != std::string::npos) { if(name_pattern[pos] == '.') name_pattern = name_pattern.substr(pos); else name_pattern.clear(); } else name_pattern.clear(); gint fd; FileHolder file(open_temp_file(name_pattern, fd)); if(file.empty()) return file; ssize_t write_size; #ifdef _WIN32 write_size = _write(fd, data+sizeof(guint32), entry_size); if (write_size == -1) { g_print("write error!\n"); } _close(fd); #else write_size = write(fd, data+sizeof(guint32), entry_size); if (write_size == -1) { g_print("write error!\n"); } close(fd); #endif ind = put_in_cache(key, file); return FileCache[ind].file; }
void csync_rs_sig(const char *filename) { FILE *basis_file = 0, *sig_file = 0; rs_stats_t stats; rs_result result; char tmpfname[MAXPATHLEN]; csync_debug(3, "Csync2 / Librsync: csync_rs_sig('%s')\n", filename); csync_debug(3, "Opening basis_file and sig_file..\n"); sig_file = open_temp_file(tmpfname, prefixsubst(filename)); if ( !sig_file ) goto io_error; if (unlink(tmpfname) < 0) goto io_error; basis_file = fopen(prefixsubst(filename), "rb"); if ( !basis_file ) basis_file = fopen("/dev/null", "rb"); csync_debug(3, "Running rs_sig_file() from librsync..\n"); result = rs_sig_file(basis_file, sig_file, RS_DEFAULT_BLOCK_LEN, RS_DEFAULT_STRONG_LEN, &stats); if (result != RS_DONE) csync_fatal("Got an error from librsync, too bad!\n"); csync_debug(3, "Sending sig_file to peer..\n"); csync_send_file(sig_file); csync_debug(3, "Signature has been created successfully.\n"); fclose(basis_file); fclose(sig_file); return; io_error: csync_debug(0, "I/O Error '%s' in rsync-sig: %s\n", strerror(errno), prefixsubst(filename)); if (basis_file) fclose(basis_file); if (sig_file) fclose(sig_file); }
// START FUNC DECL int mk_temp_file( char *opfile, size_t filesz ) // STOP FUNC DECL { int status = 0; char cwd[MAX_LEN_DIR_NAME+1]; if ( g_data_dir == NULL ) { go_BYE(-1); } zero_string(cwd, MAX_LEN_DIR_NAME+1); getcwd(cwd, MAX_LEN_DIR_NAME); if ( strlen(cwd) == 0 ) { go_BYE(-1); } status = open_temp_file(opfile, filesz); cBYE(status); status = chdir(g_data_dir); cBYE(status); status = mk_file(opfile, filesz); cBYE(status); status = chdir(cwd); cBYE(status); BYE: return(status); }
//--------------------------------------------------------------- // START FUNC DECL int parsort1( char *tbl, char *f1, char *f2, char *up_or_down /* not used right now */ ) // STOP FUNC DECL { int status = 0; char *f1_X = NULL; size_t f1_nX = 0; char *op_X = NULL; size_t op_nX = 0; char *cnt_X = NULL; size_t cnt_nX = 0; char *t2f2_X = NULL; size_t t2f2_nX = 0; FLD_TYPE *f1_meta = NULL; FLD_TYPE *f2_meta = NULL; FLD_TYPE *t2f2_meta = NULL; FLD_TYPE *cnt_meta = NULL; long long nR, nR2; int tbl_id = INT_MIN, f1_id = INT_MIN, f2_id = INT_MIN, cnt_id = INT_MIN; int t2f2_id = INT_MIN; char str_meta_data[1024]; FILE *ofp = NULL; char *opfile = NULL; FILE *tfp = NULL; char *tempfile = NULL; char str_rslt[32]; zero_string(str_rslt, 32); char t2[MAX_LEN_TBL_NAME]; int itemp; int *xxx = NULL, *f1lb = NULL, *f1ub = NULL; long long *count = NULL, *chk_count = NULL; int **offsets = NULL, **bak_offsets = NULL; int *inptr = NULL; // For multi-threading int rc; // result code for thread create pthread_t threads[MAX_NUM_THREADS]; pthread_attr_t attr; void *thread_status; // START: For timing struct timeval Tps; struct timezone Tpf; void *Tzp = NULL; long long t_before_sec = 0, t_before_usec = 0, t_before = 0; long long t_after_sec, t_after_usec, t_after; long long t_delta_usec; // STOP : For timing //---------------------------------------------------------------- if ( ( tbl == NULL ) || ( *tbl == '\0' ) ) { go_BYE(-1); } if ( ( f1 == NULL ) || ( *f1 == '\0' ) ) { go_BYE(-1); } if ( ( f2 == NULL ) || ( *f2 == '\0' ) ) { go_BYE(-1); } zero_string(str_meta_data, 1024); /* t2 isa temporary table */ zero_string(t2, MAX_LEN_TBL_NAME); status = qd_uq_str(t2, MAX_LEN_TBL_NAME); strcpy(t2, "t2"); // TODO DELETE THIS g_offsets = NULL; g_count = NULL; //-------------------------------------------------------- status = is_tbl(tbl, &tbl_id); cBYE(status); chk_range(tbl_id, 0, g_n_tbl); nR = g_tbl[tbl_id].nR; status = is_fld(NULL, tbl_id, f1, &f1_id); cBYE(status); chk_range(f1_id, 0, g_n_fld); f1_meta = &(g_fld[f1_id]); status = rs_mmap(f1_meta->filename, &f1_X, &f1_nX, 0); cBYE(status); // Not implemented for following cases if ( g_fld[f1_id].nn_fld_id >= 0 ) { go_BYE(-1); } if ( strcmp(f1_meta->fldtype, "int") != 0 ) { go_BYE(-1); } if ( nR <= 1048576 ) { go_BYE(-1); } //--------------------------------------------- status = gettimeofday(&Tps, &Tpf); cBYE(status); t_before_sec = (long long)Tps.tv_sec; t_before_usec = (long long)Tps.tv_usec; t_before = t_before_sec * 1000000 + t_before_usec; int reduction_factor = (int)(sqrt((double)nR)); sprintf(str_rslt, "%d", reduction_factor); status = subsample(tbl, f1, str_rslt, t2, "f2"); cBYE(status); status = gettimeofday(&Tps, &Tpf); cBYE(status); t_after_sec = (long long)Tps.tv_sec; t_after_usec = (long long)Tps.tv_usec; t_after = t_after_sec * 1000000 + t_after_usec; fprintf(stderr, "TIME0 = %lld \n", t_after - t_before); t_before = t_after; // Must have sufficient diversity of values status = f1opf2(t2, "f2", "op=shift:val=-1", "nextf2"); cBYE(status); status = drop_nn_fld(t2, "nextf2"); cBYE(status); status = f1f2opf3(t2, "f2", "nextf2", "==", "x"); cBYE(status); status = f_to_s(t2, "x", "sum", str_rslt); char *endptr; long long lltemp = strtoll(str_rslt, &endptr, 10); if ( lltemp != 0 ) { go_BYE(-1); } //------------------------------------------------- // Get range of values of f1 status = f_to_s(tbl, f1, "max", str_rslt); int f1max = strtoll(str_rslt, &endptr, 10); status = f_to_s(tbl, f1, "min", str_rslt); int f1min = strtoll(str_rslt, &endptr, 10); //------------------------------------------------- // Now we sort the values that we sampled status = fop(t2, "f2", "sortA"); cBYE(status); // status = pr_fld(t2, "f2", "", stdout); status = get_nR(t2, &nR2); // Now each thread selects a range to work on int nT; for ( int i = 0; i < MAX_NUM_THREADS; i++ ) { g_thread_id[i] = i; } status = get_num_threads(&nT); cBYE(status); //-------------------------------------------- #define MIN_ROWS_FOR_PARSORT1 1048576 if ( nR <= MIN_ROWS_FOR_PARSORT1 ) { nT = 1; } /* Don't create more threads than you can use */ if ( nT > nR ) { nT = nR; } //-------------------------------------------- double block_size = (double)nR2 / (double)nT; status = is_fld(t2, -1, "f2", &t2f2_id); cBYE(status); chk_range(t2f2_id, 0, g_n_fld); t2f2_meta = &(g_fld[t2f2_id]); status = rs_mmap(t2f2_meta->filename, &t2f2_X, &t2f2_nX, 0); cBYE(status); int *iptr = (int *)t2f2_X; xxx = malloc(nT * sizeof(int)); return_if_malloc_failed(xxx); f1lb = malloc(nT * sizeof(int)); return_if_malloc_failed(f1lb); f1ub = malloc(nT * sizeof(int)); return_if_malloc_failed(f1ub); /* FOR OLD_WAY count = malloc(nT * sizeof(long long)); return_if_malloc_failed(count); */ chk_count = malloc(nT * sizeof(long long)); return_if_malloc_failed(chk_count); g_count = malloc(nT * sizeof(long long)); return_if_malloc_failed(g_count); for ( int i = 0; i < nT; i++ ) { // FOR OLD_WAY count[i]= 0; chk_count[i]= 0; int j = i+1; long long idx = j * block_size; if ( idx >= nR2 ) { idx = nR2 -1 ; } int y = iptr[idx]; xxx[i] = y; // fprintf(stdout,"idx = %lld: j = %d: y = %d \n", idx, j, y); } for ( int i = 0; i < nT; i++ ) { if ( ( i == 0 ) && ( i == (nT - 1 ) ) ) { f1lb[i] = f1min; f1ub[i] = f1max; } else if ( i == 0 ) { f1lb[i] = f1min; f1ub[i] = xxx[i]; } else if ( i == (nT -1 ) ) { f1lb[i] = xxx[i-1] + 1; f1ub[i] = f1max; } else { f1lb[i] = xxx[i-1] + 1; f1ub[i] = xxx[i]; } } // STOP: Each thread has now a range to work on // Create a temporary table t3 to store ranges char t3[MAX_LEN_TBL_NAME]; int t3_id; zero_string(t3, MAX_LEN_TBL_NAME); status = qd_uq_str(t3, MAX_LEN_TBL_NAME); strcpy(t3, "t3"); // TODO DELETE THIS sprintf(str_rslt, "%d", nT); status = add_tbl(t3, str_rslt, &t3_id); // Add lower bound to t3 status = open_temp_file(&tfp, &tempfile, -1); cBYE(status); fclose_if_non_null(tfp); tfp = fopen(tempfile, "wb"); return_if_fopen_failed(tfp, tempfile, "wb"); fwrite(f1lb, sizeof(int), nT, tfp); fclose_if_non_null(tfp); sprintf(str_meta_data, "fldtype=%s:n_sizeof=%d:filename=%s", f1_meta->fldtype, f1_meta->n_sizeof, tempfile); status = add_fld(t3, "lb", str_meta_data, &itemp); cBYE(status); free_if_non_null(tempfile); // Add upper bound to t3 status = open_temp_file(&tfp, &tempfile, -1); cBYE(status); fclose_if_non_null(tfp); tfp = fopen(tempfile, "wb"); return_if_fopen_failed(tfp, tempfile, "wb"); fwrite(f1ub, sizeof(int), nT, tfp); fclose_if_non_null(tfp); sprintf(str_meta_data, "fldtype=%s:n_sizeof=%d:filename=%s", f1_meta->fldtype, f1_meta->n_sizeof, tempfile); status = add_fld(t3, "ub", str_meta_data, &itemp); cBYE(status); free_if_non_null(tempfile); #undef OLD_WAY #ifdef OLD_WAY // Now we count how much there is in each range inptr = (int *)f1_X; for ( long long i = 0; i < nR; i++ ) { int ival = *inptr++; int range_idx = INT_MIN; // TODO: Improve sequential search for ( int j = 0; j < nT; j++ ) { if ( ival >= f1lb[j] && ( ival <= f1ub[j] ) ) { range_idx = j; break; } } count[range_idx]++; } /* for ( int i = 0; i < nT; i++ ) { fprintf(stdout,"%d: (%d, %d) = %lld \n", i, f1lb[i], f1ub[i], count[i]); } */ #else status = num_in_range(tbl, f1, t3, "lb", "ub", "cnt"); cBYE(status); // Get a pointer to the count field status = is_tbl(t3, &t3_id); chk_range(t3_id, 0, g_n_tbl); status = is_fld(NULL, t3_id, "cnt", &cnt_id); chk_range(cnt_id, 0, g_n_fld); cnt_meta = &(g_fld[cnt_id]); status = rs_mmap(cnt_meta->filename, &cnt_X, &cnt_nX, 0); cBYE(status); count = (long long *)cnt_X; #endif status = gettimeofday(&Tps, &Tpf); cBYE(status); t_after_sec = (long long)Tps.tv_sec; t_after_usec = (long long)Tps.tv_usec; t_after = t_after_sec * 1000000 + t_after_usec; fprintf(stderr, "TIME1 = %lld \n", t_after - t_before); t_before = t_after; bak_offsets = malloc(nT * sizeof(int *)); return_if_malloc_failed(bak_offsets); g_offsets = malloc(nT * sizeof(int *)); return_if_malloc_failed(g_offsets); #ifdef OLD_WAY // Make space for output long long filesz = nR * f1_meta->n_sizeof; status = open_temp_file(&ofp, &opfile, filesz); cBYE(status); status = mk_file(opfile, filesz); cBYE(status); status = rs_mmap(opfile, &op_X, &op_nX, 1); cBYE(status); offsets = malloc(nT * sizeof(int *)); return_if_malloc_failed(offsets); long long cum_count = 0; for ( int i = 0; i < nT; i++ ) { bak_offsets[i] = offsets[i] = (int *)op_X; if ( i > 0 ) { cum_count += count[i-1]; offsets[i] += cum_count; bak_offsets[i] = offsets[i]; } } inptr = (int *)f1_X; // Now we place each item into its thread bucket for ( long long i = 0; i < nR; i++ ) { int ival = *inptr++; int range_idx = INT_MIN; // TODO: Improve sequential search for ( int j = 0; j < nT; j++ ) { if ( ival >= f1lb[j] && ( ival <= f1ub[j] ) ) { range_idx = j; break; } } int *xptr = offsets[range_idx]; *xptr = ival; offsets[range_idx]++; chk_count[range_idx]++; if ( chk_count[range_idx] > count[range_idx] ) { go_BYE(-1); } } cum_count = 0; for ( int i = 0; i < nT-1; i++ ) { if ( offsets[i] != bak_offsets[i+1] ) { go_BYE(-1); } } #else status = mv_range(tbl, f1, f2, t3, "lb", "ub", "cnt"); cBYE(status); status = is_fld(NULL, tbl_id, f2, &f2_id); chk_range(f2_id, 0, g_n_fld); f2_meta = &(g_fld[f2_id]); status = rs_mmap(f2_meta->filename, &op_X, &op_nX, 1); cBYE(status); #endif long long cum_count = 0; for ( int i = 0; i < nT; i++ ) { bak_offsets[i] = (int *)op_X; if ( i > 0 ) { cum_count += count[i-1]; bak_offsets[i] += cum_count; } } status = gettimeofday(&Tps, &Tpf); cBYE(status); t_after_sec = (long long)Tps.tv_sec; t_after_usec = (long long)Tps.tv_usec; t_after = t_after_sec * 1000000 + t_after_usec; fprintf(stderr, "TIME2 = %lld \n", t_after - t_before); t_before = t_after; // Set up global variables g_nT = nT; for ( int i = 0; i < nT; i++ ) { g_offsets[i] = bak_offsets[i]; g_count[i] = count[i]; } if ( g_nT == 1 ) { core_parsort1(&(g_thread_id[0])); } else { pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); for ( int t = 0; t < g_nT; t++ ) { rc = pthread_create(&threads[t], NULL, core_parsort1, &(g_thread_id[t])); if ( rc ) { go_BYE(-1); } } /* Free attribute and wait for the other threads */ pthread_attr_destroy(&attr); for ( int t = 0; t < g_nT; t++ ) { rc = pthread_join(threads[t], &thread_status); if ( rc ) { go_BYE(-1); } } } /* SEQUENTIAL CODE for ( int i = 0; i < nT; i++ ) { qsort_asc_int(bak_offsets[i], count[i], sizeof(int), NULL); } */ status = gettimeofday(&Tps, &Tpf); cBYE(status); t_after_sec = (long long)Tps.tv_sec; t_after_usec = (long long)Tps.tv_usec; t_after = t_after_sec * 1000000 + t_after_usec; fprintf(stderr, "TIME3 = %lld \n", t_after - t_before); // Indicate the dst_fld is sorted ascending status = set_fld_info(tbl, f2, "sort=1"); rs_munmap(op_X, op_nX); status = del_tbl(t2, -1); cBYE(status); status = del_tbl(t3, -1); cBYE(status); BYE: rs_munmap(op_X, op_nX); rs_munmap(cnt_X, cnt_nX); free_if_non_null(xxx); free_if_non_null(f1lb); free_if_non_null(f1ub); // Do not delete unless using OLD_WAY free_if_non_null(count); free_if_non_null(g_count); free_if_non_null(g_offsets); free_if_non_null(offsets); free_if_non_null(bak_offsets); free_if_non_null(chk_count); fclose_if_non_null(ofp); g_write_to_temp_dir = false; rs_munmap(f1_X, f1_nX); rs_munmap(op_X, op_nX); free_if_non_null(opfile); return(status); }
// START FUNC DECL int read_csv( char *infile, char fld_delim, /* double quote character */ char fld_sep, /* comma */ char rec_delim, /* new line character */ bool ignore_hdr, /* whether to ignore header or not */ char flds[MAX_NUM_FLDS][MAX_LEN_FLD_NAME+1], int n_flds, FLD_TYPE *fldtype, bool *is_null_null, /* whether to convert "null" to null */ bool *is_load, /* to load or not */ bool *is_all_def, /* whether all values are defined */ char fnames[MAX_NUM_FLDS][MAX_LEN_FILE_NAME+1], char nn_fnames[MAX_NUM_FLDS][MAX_LEN_FILE_NAME+1], long long *ptr_num_rows ) // STOP FUNC DECL { int status = 0; char *X = NULL; size_t nX = 0; bool in_fld, is_fld_delim; long long num_rows = 0; long long num_cells = 0, fld_idx; long long lb, ub; FILE *fps[MAX_NUM_FLDS]; FILE *nn_fps[MAX_NUM_FLDS]; bool in_escape_mode = false, is_fld_delim_escaped = false; g_num_rows = 0; g_buflen = 8192; g_buffer = (char *)malloc(g_buflen * sizeof(char)); return_if_malloc_failed(g_buffer); zero_string(g_buffer, g_buflen); /*----------------------------------------------------------------------------*/ if ( n_flds <= 0 ) { go_BYE(-1); } for ( int i = 0; i < MAX_NUM_FLDS; i++ ) { fps[i] = NULL; nn_fps[i] = NULL; } /*----------------------------------------------------------------------------*/ status = rs_mmap(infile, &X, &nX, 0); cBYE(status); lb = 0; fld_idx = 0; in_fld = false; is_fld_delim = false; for ( int i = 0; i < n_flds; i++ ) { if ( is_load[i] ) { status = open_temp_file((fnames[i]), 0); cBYE(status); fps[i] = fopen(fnames[i], "wb"); return_if_fopen_failed(fps[i], fnames[i], "wb"); if ( is_all_def[i] == false ) { status = open_temp_file(nn_fnames[i], 0); cBYE(status); nn_fps[i] = fopen(nn_fnames[i], "wb"); return_if_fopen_failed(nn_fps[i], nn_fnames[i], "wb"); } } } for ( long long i = 0; i < nX; i++ ) { // START: Error handling for escape conventions /* TODO: If I get an incorrect backslash, I simply replace it with a * space. This is incorrect. But I need it to read files created * with jspool utility from Jay Thomas. The correct thing to do is * to fix the utility. */ if ( fld_sep != '\t' ) { if ( in_escape_mode ) { if ( ( X[i] != '\\' ) && ( X[i] != '"' ) ) { fprintf(stderr, "Error. Bad backslash. \n"); fprintf(stderr, "num_rows = %lld \n", num_rows); go_BYE(-1); } } } // STOP: Error handling for escape conventions // START: Handling bslash and dquote if ( fld_sep != '\t' ) { if ( X[i] == '\\' ) { if ( in_escape_mode ) { in_escape_mode = false; } else { in_escape_mode = true; } } } // STOP: Handling bslash and dquote if ( fld_sep != '\t' ) { if ( X[i] == '"' ) { if ( in_escape_mode ) { is_fld_delim_escaped = true; in_escape_mode = false; } } } if ( in_fld == false ) { in_fld = true; if ( X[i] == fld_delim ) { /* consume this character */ is_fld_delim = true; lb = i+1; } else { lb = i; if ( ( ( fld_idx < (n_flds-1) ) && ( X[i] == fld_sep ) ) || ( ( fld_idx == n_flds-1 ) && ( X[i] == rec_delim ) ) ) { /* field has terminated */ ub = i-1; if ( is_load[fld_idx] ) { if ( ( ignore_hdr ) & ( num_rows == 0 ) ) { /* Do nothing */ } else { status = print_cell(X, lb, ub+1, fldtype[fld_idx], is_null_null[fld_idx], is_all_def[fld_idx], fps[fld_idx], nn_fps[fld_idx], fld_sep); cBYE(status); } } in_fld = false; num_cells++; if ( ( fld_idx == n_flds-1 ) && ( X[i] == rec_delim ) ) { num_rows++; g_num_rows = num_rows; } fld_idx++; if ( fld_idx == n_flds ) { fld_idx = 0; } } } } else { /* Now within field. Decide whether to continue or stop. */ if ( is_fld_delim ) { /* if field started with a fld_delim, it must end with one */ /* Note that we need to take care of escaping the dquote */ if ( ( X[i] == fld_delim ) && ( !is_fld_delim_escaped ) ) { /* next char must be fld_sep or rec_delim or we have come to eof */ if ( i+1 > nX ) { go_BYE(-1); } if ( ( fld_idx < (n_flds-1) ) && ( X[i+1] != fld_sep ) ) { fprintf(stderr, "num_rows = %lld \n", num_rows); go_BYE(-1); } if ( fld_idx == n_flds-1 ) { if ( X[i+1] != rec_delim ) { go_BYE(-1); } else { num_rows++; g_num_rows = num_rows; } } ub = i-1; if ( is_load[fld_idx] ) { if ( ( ignore_hdr ) & ( num_rows == 0 ) ) { /* Do nothing */ } else { status = print_cell(X, lb, ub+1, fldtype[fld_idx], is_null_null[fld_idx], is_all_def[fld_idx], fps[fld_idx], nn_fps[fld_idx], fld_sep); cBYE(status); } } in_fld = false; is_fld_delim = false; fld_idx++; if ( fld_idx == n_flds ) { fld_idx = 0; } num_cells++; i++; /* Consume next char which must be fld_sep or rec_delim */ } } else { if ( ( i == nX ) || ( ( fld_idx < (n_flds-1) ) && ( X[i] == fld_sep ) ) || ( ( fld_idx == n_flds-1 ) && ( X[i] == rec_delim ) ) ) { /* field has terminated */ if ( i == nX ) { ub = i; } else { ub = i-1; } if ( is_load[fld_idx] ) { if ( ( ignore_hdr ) & ( num_rows == 0 ) ) { /* Do nothing */ } else { status = print_cell(X, lb, ub+1, fldtype[fld_idx], is_null_null[fld_idx], is_all_def[fld_idx], fps[fld_idx], nn_fps[fld_idx], fld_sep); cBYE(status); } } in_fld = false; num_cells++; if ( ( fld_idx == n_flds-1 ) && ( X[i] == rec_delim ) ) { num_rows++; g_num_rows = num_rows; } fld_idx++; if ( fld_idx == n_flds ) { fld_idx = 0; } } } } is_fld_delim_escaped = false; } if ( num_cells != ( n_flds * num_rows ) ) { fprintf(stderr, "num_cells = %lld \n", num_cells); fprintf(stderr, "n_flds = %d \n", n_flds ); fprintf(stderr, "num_rows = %lld \n", num_rows); go_BYE(-1); } if ( ignore_hdr ) { num_rows--; // Because you have eliminated first row } /* TODO: Copy information into meta data structure */ for ( int i = 0; i < n_flds; i++ ) { if ( fps != NULL ) { fclose_if_non_null(fps[i]); } if ( nn_fps != NULL ) { fclose_if_non_null(nn_fps[i]); } } *ptr_num_rows = num_rows; BYE: rs_munmap(X, nX); free_if_non_null(g_buffer); return(status); }
//--------------------------------------------------------------- // START FUNC DECL int crossprod( char *docroot, sqlite3 *in_db, char *t1, char *f1, char *t2, char *f2, char *t3 ) // STOP FUNC DECL { int status = 0; sqlite3 *db = NULL; char *Y1 = NULL; size_t nY1 = 0; char *Y2 = NULL; size_t nY2 = 0; char *f1_X = NULL; size_t f1_nX = 0; char *f2_X = NULL; size_t f2_nX = 0; char *nn_f1_X = NULL; size_t nn_f1_nX = 0; char *nn_f2_X = NULL; size_t nn_f2_nX = 0; int t3_id, itemp; bool b_is_tbl; FLD_META_TYPE f1_meta, f2_meta; FLD_META_TYPE nn_f1_meta, nn_f2_meta; long long nR1, nR2, nn_nR1, nn_nR2, nR3; char str_meta_data[1024]; char *t3f1_opfile = NULL, *t3f2_opfile = NULL; FILE *ofp = NULL; char buffer[32]; //---------------------------------------------------------------- zero_string(str_meta_data, 1024); zero_string(buffer, 32); zero_fld_meta(&f1_meta); zero_fld_meta(&f2_meta); zero_fld_meta(&nn_f1_meta); zero_fld_meta(&nn_f2_meta); status = open_db_if_needed(docroot, in_db, &db); cBYE(status); //---------------------------------------------------------------- status = get_data(docroot, db, t1, f1, &nR1, &f1_meta, &f1_X, &f1_nX, &nn_f1_meta, &nn_f1_X, &nn_f1_nX, &nn_nR1); cBYE(status); status = get_data(docroot, db, t2, f2, &nR2, &f2_meta, &f2_X, &f2_nX, &nn_f2_meta, &nn_f2_X, &nn_f2_nX, &nn_nR2); cBYE(status); nR3 = nn_nR1 * nn_nR2; if ( nR3 == 0 ) { fprintf(stderr, "No data to create t3 \n"); goto BYE; } status = open_temp_file(&ofp, &t3f1_opfile); cBYE(status); fclose_if_non_null(ofp); status = mk_file(t3f1_opfile, nR3 * f1_meta.n_sizeof); cBYE(status); status = rs_mmap(t3f1_opfile, &Y1, &nY1, 1); cBYE(status); status = open_temp_file(&ofp, &t3f2_opfile); cBYE(status); fclose_if_non_null(ofp); status = mk_file(t3f2_opfile, nR3 * f2_meta.n_sizeof); cBYE(status); status = rs_mmap(t3f2_opfile, &Y2, &nY2, 1); cBYE(status); //---------------------------------------------------------------- if ( ( strcmp(f1_meta.fldtype, "int") == 0 ) && ( strcmp(f2_meta.fldtype, "int") == 0 ) ) { status = core_crossprod_I_I(f1_X, nn_f1_X, f2_X, nn_f2_X, nR1, nR2, nR3, Y1, Y2); } else if ( ( strcmp(f1_meta.fldtype, "int") == 0 ) && ( strcmp(f2_meta.fldtype, "long long") == 0 ) ) { status = core_crossprod_I_L(f1_X, nn_f1_X, f2_X, nn_f2_X, nR1, nR2, nR3, Y1, Y2); } else if ( ( strcmp(f1_meta.fldtype, "long long") == 0 ) && ( strcmp(f2_meta.fldtype, "int") == 0 ) ) { status = core_crossprod_L_I(f1_X, nn_f1_X, f2_X, nn_f2_X, nR1, nR2, nR3, Y1, Y2); } else if ( ( strcmp(f1_meta.fldtype, "long long") == 0 ) && ( strcmp(f2_meta.fldtype, "long long") == 0 ) ) { status = core_crossprod_L_L(f1_X, nn_f1_X, f2_X, nn_f2_X, nR1, nR2, nR3, Y1, Y2); } else { go_BYE(-1); } cBYE(status); //---------------------------------------------------------------- // Add output fields to t3 meta data status = is_tbl(docroot, db, t3, &b_is_tbl, &t3_id); cBYE(status); if ( b_is_tbl ) { status = del_tbl(docroot, db, t3); cBYE(status); } sprintf(buffer, "%lld", nR3); status = add_tbl(docroot, db, t3, buffer, &itemp); cBYE(status); sprintf(str_meta_data, "fldtype=int:n_sizeof=%u:filename=%s", f1_meta.n_sizeof, t3f1_opfile); status = add_fld(docroot, db, t3, f1, str_meta_data); cBYE(status); zero_string(str_meta_data, 1024); sprintf(str_meta_data, "fldtype=int:n_sizeof=%u:filename=%s", f2_meta.n_sizeof, t3f2_opfile); status = add_fld(docroot, db, t3, f2, str_meta_data); cBYE(status); BYE: fclose_if_non_null(ofp); if ( in_db == NULL ) { sqlite3_close(db); } rs_munmap(f1_X, f1_nX); rs_munmap(nn_f1_X, nn_f1_nX); rs_munmap(f2_X, f2_nX); rs_munmap(nn_f2_X, nn_f2_nX); rs_munmap(Y1, nY1); rs_munmap(Y2, nY2); free_if_non_null(t3f1_opfile); free_if_non_null(t3f2_opfile); return(status); }
//--------------------------------------------------------------- // START FUNC DECL int count_vals( char *src_tbl, char *src_fld, char *in_cnt_fld, char *dst_tbl, char *dst_fld, char *out_cnt_fld ) // STOP FUNC DECL { int status = 0; char *src_fld_X = NULL; size_t src_fld_nX = 0; char *in_cnt_fld_X = NULL; size_t in_cnt_fld_nX = 0; FLD_TYPE *src_fld_meta = NULL, *in_cnt_fld_meta = NULL; FLD_TYPE *cnt_fld_meta = NULL, *nn_src_fld_meta = NULL; long long src_nR; int src_tbl_id = INT_MIN, dst_tbl_id = INT_MIN; int src_fld_id = INT_MIN, nn_src_fld_id = INT_MIN; int dst_fld_id = INT_MIN; int cnt_fld_id = INT_MIN, nn_cnt_fld_id = INT_MIN; long long dst_nR, n_out; char buffer[32]; char str_meta_data[256]; char *nn_src_fld_X = NULL; size_t nn_src_fld_nX = 0; char *out_X = NULL; char *cnt_X = NULL; char *opfile = NULL, *tmp_opfile = NULL; FILE *ofp = 0; char *cnt_opfile = NULL, *tmp_cnt_opfile = NULL; FILE *cnt_ofp = 0; size_t n_out_X = 0, n_cnt_X = 0; unsigned int n_sizeof = INT_MAX; //---------------------------------------------------------------- if ( ( src_tbl == NULL ) || ( *src_tbl == '\0' ) ) { go_BYE(-1); } if ( ( src_fld == NULL ) || ( *src_fld == '\0' ) ) { go_BYE(-1); } if ( ( dst_tbl == NULL ) || ( *dst_tbl == '\0' ) ) { go_BYE(-1); } if ( ( dst_fld == NULL ) || ( *dst_fld == '\0' ) ) { go_BYE(-1); } if ( strcmp(src_tbl, dst_tbl) == 0 ) { go_BYE(-1); } zero_string(str_meta_data, 256); zero_string(buffer, 32); //-------------------------------------------------------- status = is_tbl(src_tbl, &src_tbl_id); cBYE(status); chk_range(src_tbl_id, 0, g_n_tbl); src_nR = g_tbl[src_tbl_id].nR; //-------------------------------------------------------- status = is_fld(NULL, src_tbl_id, src_fld, &src_fld_id); cBYE(status); chk_range(src_fld_id, 0, g_n_fld); src_fld_meta = &(g_fld[src_fld_id]); status = rs_mmap(src_fld_meta->filename, &src_fld_X, &src_fld_nX, 0); cBYE(status); nn_src_fld_id = src_fld_meta->nn_fld_id; if ( nn_src_fld_id >= 0 ) { nn_src_fld_meta = &(g_fld[nn_src_fld_id]); status = rs_mmap(nn_src_fld_meta->filename, &nn_src_fld_X, &nn_src_fld_nX, 0); cBYE(status); } // Get the count field if specified. It cannot have null values if ( *in_cnt_fld != '\0' ) { status = is_fld(NULL, src_tbl_id, src_fld, &src_fld_id); cBYE(status); chk_range(src_fld_id, 0, g_n_fld); src_fld_meta = &(g_fld[src_fld_id]); status = rs_mmap(src_fld_meta->filename, &src_fld_X, &src_fld_nX, 0); cBYE(status); nn_cnt_fld_id = cnt_fld_meta->nn_fld_id; chk_range(nn_cnt_fld_id, 0, g_n_fld); } /* Make sure src_fld field is sorted ascending */ zero_string(buffer, 32); status = f_to_s(src_tbl, src_fld, "is_sorted", buffer); cBYE(status); if ( strcmp(buffer, "ascending") != 0 ) { fprintf(stderr, "Field [%s] in Table [%s] not sorted ascending\n", src_fld, src_tbl); go_BYE(-1); } //-------------------------------------------------------- // Create 2 temporary files to store the results. We allocate space // differently based on field types g_write_to_temp_dir = true; status = open_temp_file(&ofp, &tmp_opfile, 0); cBYE(status); fclose_if_non_null(ofp); status = open_temp_file(&cnt_ofp, &tmp_cnt_opfile, 0); cBYE(status); fclose_if_non_null(cnt_ofp); //------------------------------------------------------ n_out = src_nR * sizeof(int); // upper bound if ( strcmp(src_fld_meta->fldtype, "int") == 0 ) { status = mk_file(tmp_cnt_opfile, sizeof(int) * src_nR); cBYE(status); rs_mmap(tmp_cnt_opfile, &cnt_X, &n_cnt_X, 1); cBYE(status); status = mk_file(tmp_opfile, sizeof(int) * src_nR); cBYE(status); rs_mmap(tmp_opfile, &out_X, &n_out_X, 1); cBYE(status); if ( nn_src_fld_X == NULL ) { if ( *in_cnt_fld == '\0' ) { uniq_alldef_int((int *)src_fld_X, src_nR, (int *)out_X, &dst_nR, (int *)cnt_X); } else { uniq_cnt_alldef_i_i((int *)src_fld_X, (int *)in_cnt_fld_X, src_nR, (int *)out_X, &dst_nR, (int *)cnt_X); } } else { uniq_int((int *)src_fld_X, src_nR, nn_src_fld_X, (int *)out_X, &dst_nR, (int *)cnt_X); } } else if ( strcmp(src_fld_meta->fldtype, "long long") == 0 ) { status = mk_file(tmp_opfile, sizeof(long long) * src_nR); cBYE(status); rs_mmap(tmp_opfile, &out_X, &n_out_X, 1); cBYE(status); if ( nn_src_fld_X == NULL ) { if ( *in_cnt_fld == '\0' ) { status = mk_file(tmp_cnt_opfile, sizeof(long long) * src_nR); cBYE(status); rs_mmap(tmp_cnt_opfile, &cnt_X, &n_cnt_X, 1); cBYE(status); uniq_alldef_longlong((long long *)src_fld_X, src_nR, (long long *)out_X, &dst_nR, (long long *)cnt_X); } else { if ( strcmp(in_cnt_fld_meta->fldtype, "long long") == 0 ) { status = mk_file(tmp_cnt_opfile, sizeof(long long) * src_nR); cBYE(status); rs_mmap(tmp_cnt_opfile, &cnt_X, &n_cnt_X, 1); cBYE(status); uniq_cnt_alldef_ll_ll((long long *)src_fld_X, (long long *)in_cnt_fld_X, src_nR, (long long *)out_X, &dst_nR, (long long *)cnt_X); } else if ( strcmp(in_cnt_fld_meta->fldtype, "double") == 0 ) { status = mk_file(tmp_cnt_opfile, sizeof(double) * src_nR); cBYE(status); rs_mmap(tmp_cnt_opfile, &cnt_X, &n_cnt_X, 1); cBYE(status); uniq_cnt_alldef_ll_dbl((long long *)src_fld_X, (double *)in_cnt_fld_X, src_nR, (long long *)out_X, &dst_nR, (double *)cnt_X); } } } else { if ( *in_cnt_fld == '\0' ) { status = mk_file(tmp_cnt_opfile, sizeof(long long) * src_nR); cBYE(status); rs_mmap(tmp_cnt_opfile, &cnt_X, &n_cnt_X, 1); cBYE(status); uniq_longlong((long long *)src_fld_X, src_nR, nn_src_fld_X, (long long *)out_X, &dst_nR, (long long *)cnt_X); } else { fprintf(stderr, "NOT IMPLEMENTED\n"); go_BYE(-1); } } } else { go_BYE(-1); } if ( dst_nR == 0 ) { goto BYE; } //----------------------------------------------------------- status = is_tbl(dst_tbl, &dst_tbl_id); cBYE(status); if ( dst_tbl_id >= 0 ) { status = del_tbl(NULL, dst_tbl_id); } sprintf(buffer, "%lld", dst_nR); status = add_tbl(dst_tbl, buffer, &dst_tbl_id); cBYE(status); //----------------------------------------------------------- // Now copy the temporary files over to real files status = open_temp_file(&ofp, &opfile, 0); cBYE(status); if ( strcmp(src_fld_meta->fldtype, "int") == 0 ) { fwrite(out_X, sizeof(int), dst_nR, ofp); n_sizeof = sizeof(int); } else if ( strcmp(src_fld_meta->fldtype, "long long") == 0 ) { fwrite(out_X, sizeof(long long), dst_nR, ofp); n_sizeof = sizeof(long long); } fclose_if_non_null(ofp); // Add output field to meta data sprintf(str_meta_data, "filename=%s:fldtype=%s:n_sizeof=%u", opfile, src_fld_meta->fldtype, n_sizeof); status = add_fld(dst_tbl, dst_fld, str_meta_data, &dst_fld_id); cBYE(status); // Since src_fld is sorted ascending, so is dst_fld g_fld[dst_fld_id].sorttype = 1; // Now copy the temporary count file to the real one status = open_temp_file(&cnt_ofp, &cnt_opfile, 0); cBYE(status); if ( *in_cnt_fld == '\0' ) { if ( strcmp(src_fld_meta->fldtype, "int") == 0 ) { fwrite(cnt_X, sizeof(int), dst_nR, cnt_ofp); sprintf(str_meta_data, "filename=%s:fldtype=int:n_sizeof=%lu", cnt_opfile, sizeof(int)); } else if ( strcmp(src_fld_meta->fldtype, "long long") == 0 ) { fwrite(cnt_X, sizeof(long long), dst_nR, cnt_ofp); sprintf(str_meta_data, "filename=%s:fldtype=long long:n_sizeof=%lu", cnt_opfile, sizeof(long long)); } else { go_BYE(-1); } } else { if ( strcmp(in_cnt_fld_meta->fldtype, "int") == 0 ) { fwrite(cnt_X, sizeof(int), dst_nR, cnt_ofp); sprintf(str_meta_data, "filename=%s:fldtype=int:n_sizeof=%lu", cnt_opfile, sizeof(int)); } else if ( strcmp(in_cnt_fld_meta->fldtype, "long long") == 0 ) { fwrite(cnt_X, sizeof(long long), dst_nR, cnt_ofp); sprintf(str_meta_data, "filename=%s:fldtype=long long:n_sizeof=%lu", cnt_opfile, sizeof(long long)); } else if ( strcmp(in_cnt_fld_meta->fldtype, "double") == 0 ) { fwrite(cnt_X, sizeof(double), dst_nR, cnt_ofp); sprintf(str_meta_data, "filename=%s:fldtype=double:n_sizeof=%lu", cnt_opfile, sizeof(double)); } else { go_BYE(-1); } } fclose_if_non_null(cnt_ofp); // Add count field to meta data status = add_fld(dst_tbl, out_cnt_fld, str_meta_data, &cnt_fld_id); cBYE(status); //----------------------------------------------------------- BYE: g_write_to_temp_dir = false; rs_munmap(src_fld_X, src_fld_nX); free_if_non_null(opfile); rs_munmap(src_fld_X, src_fld_nX); rs_munmap(in_cnt_fld_X, in_cnt_fld_nX); rs_munmap(cnt_X, n_cnt_X); rs_munmap(out_X, n_out_X); rs_munmap(nn_src_fld_X, nn_src_fld_nX); unlink(tmp_opfile); unlink(tmp_cnt_opfile); free_if_non_null(tmp_opfile); free_if_non_null(tmp_cnt_opfile); free_if_non_null(cnt_opfile); return(status); }
// START FUNC DECL int orphan_files( char *in_data_dir, char *action ) // STOP FUNC DECL { int status = 0; DIR *dp = NULL; FILE *ofp = NULL; struct dirent *ep = NULL; char Xopfile[MAX_LEN_FILE_NAME+1]; char Yopfile[MAX_LEN_FILE_NAME+1]; char *X = NULL; size_t nX = 0; char *Y = NULL; size_t nY = 0; char nullc = '\0'; int Xnum_files = 0, Ynum_files = 0; int ddir_id; char data_dir[MAX_LEN_DIR_NAME+1]; if ( ( in_data_dir == NULL ) || ( *in_data_dir == '\0' ) ) { status = chdir(g_data_dir); cBYE(status); ddir_id = -1; if ( strlen(g_data_dir) >= MAX_LEN_DIR_NAME ) { go_BYE(-1); } strcpy(data_dir, g_data_dir); } else { status = strip_trailing_slash(in_data_dir, data_dir, MAX_LEN_DIR_NAME+1); cBYE(status); status = get_ddir_id(data_dir, g_ddirs, g_n_ddir, false, &ddir_id); cBYE(status); status = chdir(data_dir); cBYE(status); } zero_string(Xopfile, MAX_LEN_FILE_NAME+1); zero_string(Yopfile, MAX_LEN_FILE_NAME+1); /* START: Create X. X = filenames listed in the meta data (whose * ddir_id matches that requested) in sorted order */ status = open_temp_file(g_cwd, g_cwd, Xopfile, 0); cBYE(status); ofp = fopen(Xopfile, "w"); return_if_fopen_failed(ofp, Xopfile, "w"); for ( int i = 0; i < g_n_fld; i++ ) { if ( g_flds[i].name[0] == '\0' ) { continue; } if ( g_flds[i].ddir_id != ddir_id ) { continue; } Xnum_files++; fwrite(g_flds[i].filename, sizeof(char), MAX_LEN_FILE_NAME+1, ofp); } fclose_if_non_null(ofp); if ( Xnum_files > 0 ) { status = rs_mmap(Xopfile, &X, &nX, 1); qsort(X, Xnum_files, MAX_LEN_FILE_NAME+1, str_compare); } /* STOP ---- */ status = open_temp_file(g_cwd, g_cwd, Yopfile, 0); cBYE(status); ofp = fopen(Yopfile, "w"); return_if_fopen_failed(ofp, Xopfile, "w"); dp = opendir(data_dir); if ( dp == NULL) { go_BYE(-1); } for ( ; ; ) { ep = readdir (dp); if ( ep == NULL ) { break; } char *filename = ep->d_name; if ( ( filename == NULL ) || ( *filename == '\0' ) ) { break; } if ( strncmp(filename, "docroot", 7) == 0 ) { continue; } if ( strncmp(filename, ".LDB_META", 9) == 0 ) { continue; } if ( strcmp(filename, "..") == 0 ) { continue; } if ( strcmp(filename, ".") == 0 ) { continue; } int len = strlen(filename); if ( len >= MAX_LEN_FILE_NAME ) { // This is clearly no good fprintf(stdout, "%s\n", filename); } else { Ynum_files++; fwrite(filename, sizeof(char), len, ofp); for ( int i = len; i < MAX_LEN_FILE_NAME+1; i++ ) { fwrite(&nullc, sizeof(char), 1, ofp); } } } fclose_if_non_null(ofp); (void) closedir (dp); if ( Ynum_files > 0 ) { status = rs_mmap(Yopfile, &Y, &nY, 1); qsort(Y, Ynum_files, MAX_LEN_FILE_NAME+1, str_compare); } /* Now compare X against Y */ char *xptr = X; char *yptr = Y; int seenX = 0, seenY = 0; for ( ; ( ( seenX < Xnum_files ) && ( seenY < Ynum_files ) ) ; ) { int cmpval = str_compare(xptr, yptr); switch ( cmpval ) { case 0 : /* File in meta data exists on disk. This is good */ xptr += MAX_LEN_FILE_NAME+1; yptr += MAX_LEN_FILE_NAME+1; seenX++; seenY++; break; case -1 : /* File exists in meta data but not on disk. This is horrendous */ printf("HORRENDOUS %s\n", yptr); xptr += MAX_LEN_FILE_NAME+1; seenX++; break; case 1 : /* File on disk but not in meta data. This is an "orphan" * */ printf("%s\n", yptr); yptr += MAX_LEN_FILE_NAME+1; seenY++; break; default : go_BYE(-1); break; } } if ( seenX < Xnum_files ) { for ( ; seenX < Xnum_files ; seenX++ ) { fprintf(stderr, "MISSING FILE %s\n", xptr); xptr += MAX_LEN_FILE_NAME; seenX++; } } if ( seenY < Ynum_files ) { for ( ; seenY < Ynum_files ; seenY++ ) { fprintf(stderr, "%s\n", yptr); yptr += MAX_LEN_FILE_NAME; seenY++; } } BYE: fclose_if_non_null(ofp); unlink_if_non_null(Xopfile); unlink_if_non_null(Yopfile); return(status); }
//--------------------------------------------------------------- int crossprod1( char *docroot, sqlite3 *in_db, char *t1, char *f1, char *f2 ) { int status = 0; sqlite3 *db = NULL; char *f1_X = NULL; size_t f1_nX = 0; char *f2_X = NULL; size_t f2_nX = 0; char *nn_f1_X = NULL; size_t nn_f1_nX = 0; char *nn_f2_X = NULL; size_t nn_f2_nX = 0; int t1_id, t2_id, t3_id, itemp; bool b_is_tbl; FLD_META_TYPE f1_meta, f2_meta; FLD_META_TYPE nn_f1_meta, nn_f2_meta; int i1val, i2val; int *iptr1 = NULL, *iptr2 = NULL; long long nR1, nR2, nR3 = 0; char str_meta_data[1024]; char *t3f1_opfile = NULL, *t3f2_opfile = NULL; FILE *ofp1 = NULL, *ofp2 = NULL; char str_nR3[32]; //---------------------------------------------------------------- zero_string(str_meta_data, 1024); zero_string(str_nR3, 32); zero_fld_meta(&f1_meta); zero_fld_meta(&f2_meta); zero_fld_meta(&nn_f1_meta); zero_fld_meta(&nn_f2_meta); //---------------------------------------------------------------- status = open_temp_file(&ofp1, &t3f1_opfile); cBYE(status); status = open_temp_file(&ofp2, &t3f2_opfile); cBYE(status); status = open_db_if_needed(docroot, in_db, &db); cBYE(status); //---------------------------------------------------------------- status = external_get_nR(docroot, db, t1, &nR1, &t1_id); cBYE(status); if ( nR1 <= 0 ) { go_BYE(-1); } status = fld_meta(docroot, db, t1, f1, -1, &f1_meta); cBYE(status); status = rs_mmap(f1_meta.filename, &f1_X, &f1_nX, 0); cBYE(status); // Get nn field for t1/f1 if it exists status = get_aux_field_if_it_exists(docroot, db, t1, f1_meta.id, "nn", &nn_f1_meta, &nn_f1_X, &nn_f1_nX); //---------------------------------------------------------------- status = external_get_nR(docroot, db, t2, &nR2, &t2_id); cBYE(status); if ( nR2 <= 0 ) { go_BYE(-1); } status = fld_meta(docroot, db, t2, f2, -1, &f2_meta); cBYE(status); status = rs_mmap(f2_meta.filename, &f2_X, &f2_nX, 0); cBYE(status); // Get nn field for t2/f2 if it exists status = get_aux_field_if_it_exists(docroot, db, t2, f2_meta.id, "nn", &nn_f2_meta, &nn_f2_X, &nn_f2_nX); //---------------------------------------------------------------- iptr1 = (int *)f1_X; iptr2 = (int *)f2_X; for ( long long i1 = 0; i1 < nR1; i1++ ) { if ( ( nn_f1_X == NULL ) || ( nn_f1_X[i1] == 1 ) ) { i1val = iptr1[i1]; for ( long long i2 = 0; i2 < nR2; i2++ ) { if ( ( nn_f2_X == NULL ) || ( nn_f2_X[i2] == 1 ) ) { i2val = iptr2[i2]; fwrite(&i1val, sizeof(int), 1, ofp1); fwrite(&i2val, sizeof(int), 1, ofp2); nR3++; } } } } fclose_if_non_null(ofp1); fclose_if_non_null(ofp2); //---------------------------------------------------------------- // Add output fields to t3 meta data status = is_tbl(docroot, db, t3, &b_is_tbl, &t3_id, &itemp); cBYE(status); if ( b_is_tbl ) { status = del_tbl(docroot, db, t3); cBYE(status); } if ( nR3 == 0 ) { fprintf(stderr, "No data to create t3 \n"); goto BYE; } sprintf(str_nR3, "%lld", nR3); status = add_tbl(docroot, db, t3, str_nR3, &itemp); cBYE(status); sprintf(str_meta_data, "fldtype=int:n_sizeof=%d:filename=%s", sizeof(int), t3f1_opfile); status = add_fld(docroot, db, t3, f1, str_meta_data); cBYE(status); zero_string(str_meta_data, 1024); sprintf(str_meta_data, "fldtype=int:n_sizeof=%d:filename=%s", sizeof(int), t3f2_opfile); status = add_fld(docroot, db, t3, f2, str_meta_data); cBYE(status); BYE: fclose_if_non_null(ofp1); fclose_if_non_null(ofp2); if ( in_db == NULL ) { sqlite3_close(db); } rs_munmap(f1_X, f1_nX); rs_munmap(nn_f1_X, nn_f1_nX); rs_munmap(f2_X, f2_nX); rs_munmap(nn_f2_X, nn_f2_nX); free_if_non_null(t3f1_opfile); free_if_non_null(t3f2_opfile); return(status); }
//--------------------------------------------------------------- // START FUNC DECL int count_strings( char *src_tbl, char *src_fld, char *str_trunc_len, // truncate strings beyond this len char *dst_tbl ) // STOP FUNC DECL { int status = 0; char *X = NULL; size_t nX = 0; char *src_fld_X = NULL; size_t src_fld_nX = 0; char *nn_src_fld_X = NULL; size_t nn_src_fld_nX = 0; char *sz_src_fld_X = NULL; size_t sz_src_fld_nX = 0; FLD_TYPE *src_fld_meta = NULL; FLD_TYPE *nn_src_fld_meta = NULL; FLD_TYPE *sz_src_fld_meta = NULL; long long src_nR; int src_tbl_id = INT_MIN, src_fld_id = INT_MIN; int dst_tbl_id = INT_MIN; int nn_src_fld_id = INT_MIN, sz_src_fld_id = INT_MIN; int itemp; char str_dst_nR[32]; long long dst_nR = 0; char str_meta_data[4096]; char *out_X = NULL; char *cnt_X = NULL; char *dst_fk_X = NULL; size_t dst_fk_nX = 0; FILE *ofp = NULL; char *opfile = NULL; FILE *tmp_ofp = NULL; char *tmp_opfile = NULL; FILE *nn_ofp = NULL; char *nn_opfile = NULL; FILE *sz_ofp = NULL; char *sz_opfile = NULL; FILE *cnt_ofp = NULL; char *cnt_opfile = NULL; FILE *from_dst_ofp = NULL; char *from_dst_opfile = NULL; FILE *dst_fk_ofp = NULL; char *dst_fk_opfile = NULL; int *idst_fk = NULL; FILE *to_dst_ofp = NULL; char *to_dst_opfile = NULL; size_t out_nX = 0, cnt_nX = 0; int *szptr = NULL; char *nnptr = NULL; char *cptr = NULL, *prev_cptr = NULL; char *sz_src_fld = NULL; char *str_maxlen = NULL; char *from_dst_fld = NULL, *to_dst_fld = NULL, *dst_fk_fld = NULL; char *null_str = NULL, *endptr = NULL; int maxlen, rec_size, trunc_len = -1; // no truncation by default long long cnt = 1; int icnt = 1; bool is_any_null = false; char is_nn; int sz; //---------------------------------------------------------------- if ( str_trunc_len != NULL ) { trunc_len = strtol(str_trunc_len, &endptr, 10); if ( *endptr != '\0' ) { go_BYE(-1); } if ( trunc_len <= 1 ) { go_BYE(-1); } trunc_len++; // to include null character } if ( ( src_tbl == NULL ) || ( *src_tbl == '\0' ) ) { go_BYE(-1); } if ( ( src_fld == NULL ) || ( *src_fld == '\0' ) ) { go_BYE(-1); } if ( ( dst_tbl == NULL ) || ( *dst_tbl == '\0' ) ) { go_BYE(-1); } zero_string(str_meta_data, 4096); zero_string(str_dst_nR, 32); //-------------------------------------------------------- status = is_tbl(src_tbl, &src_tbl_id); cBYE(status); chk_range(src_tbl_id, 0, g_n_tbl); status = is_fld(NULL, src_tbl_id, src_fld, &src_fld_id); cBYE(status); chk_range(src_fld_id, 0, g_n_fld); src_nR = g_tbl[src_tbl_id].nR; if ( src_nR > INT_MAX ) { fprintf(stderr, "NOT IMPLEMENTED\n"); go_BYE(-1); } //-------------------------------------------------------- src_fld_meta = &(g_fld[src_fld_id]); status = rs_mmap(src_fld_meta->filename, &src_fld_X, &src_fld_nX, 0); cBYE(status); //-------------------------------------------------------- if ( strcmp(src_fld_meta->fldtype, "char string") != 0 ) { go_BYE(-1); } cptr = src_fld_X; // Get nn field for source if if it exists nn_src_fld_id = g_fld[src_fld_id].nn_fld_id; if ( nn_src_fld_id >= 0 ) { nn_src_fld_meta = &(g_fld[nn_src_fld_id]); status = rs_mmap(nn_src_fld_meta->filename, &nn_src_fld_X, &nn_src_fld_nX, 0); cBYE(status); } nnptr = (char *)nn_src_fld_X; // Get sz field for source sz_src_fld_id = g_fld[src_fld_id].sz_fld_id; chk_range(sz_src_fld_id, 0, g_n_fld); sz_src_fld_meta = &(g_fld[sz_src_fld_id]); status = rs_mmap(sz_src_fld_meta->filename, &sz_src_fld_X, &sz_src_fld_nX, 0); cBYE(status); szptr = (int *)sz_src_fld_X; //------------------------------------ mcr_alloc_null_str(from_dst_fld, (MAX_LEN_FILE_NAME+16)); mcr_alloc_null_str(to_dst_fld, (MAX_LEN_FILE_NAME+16)); mcr_alloc_null_str(dst_fk_fld, (MAX_LEN_FILE_NAME+16)); mcr_alloc_null_str(sz_src_fld, (MAX_LEN_FILE_NAME+16)); mcr_alloc_null_str(str_maxlen, 32); strcpy(sz_src_fld, "_sz_"); strcat(sz_src_fld, src_fld); strcpy(from_dst_fld, src_tbl); strcat(from_dst_fld, "_from"); strcpy(to_dst_fld, dst_tbl); strcat(to_dst_fld, "_to"); strcpy(dst_fk_fld, dst_tbl); strcat(dst_fk_fld, "_id"); // Get max length of src field status = f_to_s(src_tbl, sz_src_fld, "max", str_maxlen); cBYE(status); maxlen = strtol(str_maxlen, &endptr, 10); if ( *endptr != '\0' ) { go_BYE(-1); } if ( maxlen <= 0 ) { go_BYE(-1); } if ( trunc_len < 0 ) { trunc_len = maxlen; } mcr_alloc_null_str(null_str, trunc_len); //-------------------------------------------------------- // Write out src field as constant length field status = open_temp_file(&tmp_ofp, &tmp_opfile, 0); cBYE(status); for ( int i = 0; i < src_nR; i++ ) { int sz, fill_sz; sz = szptr[i]; // includes terminating null character if ( sz <= trunc_len ) { fwrite(cptr, sizeof(char), sz, tmp_ofp); fill_sz = trunc_len - sz; fwrite(null_str, sizeof(char), fill_sz, tmp_ofp); } else { fwrite(cptr, sizeof(char), trunc_len-1, tmp_ofp); fwrite(null_str, sizeof(char), 1, tmp_ofp); } // Append origin of this string fwrite(&i, sizeof(int), 1, tmp_ofp); cptr += sz; // TODO: Check about nullc and sz } fclose_if_non_null(tmp_ofp); //--------------------------------------------------------- // Sort output file so that similar titles occur together status = rs_mmap(tmp_opfile, &X, &nX, 1); // 1 for writing cBYE(status); // Note that we added the origin of the string. Hence the size of a // record is trunc_len + sizeof(int) rec_size = trunc_len + sizeof(int); qsort(X, src_nR, rec_size, str_compare); rs_munmap(X, nX); //-------------------------------------------------------- // Now we need to compact the sorted output so that it is not padded // Open 4 files for // output string, sz(output string), nn(output string), cnt status = open_temp_file(&ofp, &opfile, 0); cBYE(status); status = open_temp_file(&nn_ofp, &nn_opfile, 0); cBYE(status); status = open_temp_file(&sz_ofp, &sz_opfile, 0); cBYE(status); status = open_temp_file(&cnt_ofp, &cnt_opfile, 0); cBYE(status); status = open_temp_file(&from_dst_ofp, &from_dst_opfile, 0); cBYE(status); status = open_temp_file(&to_dst_ofp, &to_dst_opfile, 0); cBYE(status); status = open_temp_file(&dst_fk_ofp, &dst_fk_opfile, 0); cBYE(status); fclose_if_non_null(dst_fk_ofp); mk_file(dst_fk_opfile, src_nR * sizeof(int)); cBYE(status); rs_mmap(dst_fk_opfile, &dst_fk_X, &dst_fk_nX, 1); cBYE(status); idst_fk = (int *)dst_fk_X; status = rs_mmap(tmp_opfile, &X, &nX, 0); cBYE(status); cptr = X; prev_cptr = cptr; cptr += rec_size; cnt = icnt = 1; for ( int i = 0; i < src_nR; i++ ) { char *xptr; int *iptr; xptr = prev_cptr + trunc_len; iptr = (int *)xptr; if ( ( *iptr < 0 ) || ( *iptr >= src_nR ) ) { go_BYE(-1); } fwrite(iptr, sizeof(int), 1, from_dst_ofp); fwrite(&dst_nR, sizeof(int), 1, to_dst_ofp); idst_fk[*iptr] = dst_nR; if ( strcmp(cptr, prev_cptr) != 0 ) { sz = strlen(prev_cptr); if ( sz == 0 ) { is_nn = FALSE; } else { is_nn = TRUE; } sz++; // for null character termination fwrite(prev_cptr, sizeof(char), sz, ofp); fwrite(&sz, sizeof(int), 1, sz_ofp); fwrite(&is_nn, sizeof(char), 1, nn_ofp); if ( cnt > INT_MAX ) { go_BYE(-1); } icnt = cnt; fwrite(&icnt, sizeof(int), 1, cnt_ofp); cnt = 1; dst_nR++; } else { cnt++; } prev_cptr = cptr; cptr += rec_size; } // START: Now for the last guy cnt--; // this is needed sz = strlen(prev_cptr); if ( sz == 0 ) { is_nn = FALSE; } else { is_nn = TRUE; } if ( is_nn == FALSE ) { is_any_null = true; } sz++; // for null character termination fwrite(prev_cptr, sizeof(char), sz, ofp); fwrite(&sz, sizeof(int), 1, sz_ofp); fwrite(&is_nn, sizeof(char), 1, nn_ofp); if ( cnt > INT_MAX ) { go_BYE(-1); } icnt = cnt; fwrite(&icnt, sizeof(int), 1, cnt_ofp); dst_nR++; // STOP: Now for the last guy //--------------------------- fclose_if_non_null(ofp); fclose_if_non_null(nn_ofp); fclose_if_non_null(sz_ofp); fclose_if_non_null(cnt_ofp); fclose_if_non_null(from_dst_ofp); fclose_if_non_null(to_dst_ofp); //----------------------------------------------- // Delete temp file unlink(tmp_opfile); //----------------------------------------------------------- status = is_tbl(dst_tbl, &dst_tbl_id); cBYE(status); if ( dst_tbl_id >= 0 ) { /* Delete table */ status = del_tbl(NULL, dst_tbl_id); } sprintf(str_dst_nR, "%lld", dst_nR); status = add_tbl(dst_tbl, str_dst_nR, &dst_tbl_id); cBYE(status); //----------------------------------------------------------- /* TODO I think we can dispense with from_dst_fld and to_dst_fld */ sprintf(str_meta_data, "filename=%s:fldtype=int:n_sizeof=4", from_dst_opfile); status = add_fld(src_tbl, from_dst_fld, str_meta_data, &itemp); cBYE(status); sprintf(str_meta_data, "filename=%s:fldtype=int:n_sizeof=4", to_dst_opfile); status = add_fld(src_tbl, to_dst_fld, str_meta_data, &itemp); cBYE(status); sprintf(str_meta_data, "filename=%s:fldtype=int:n_sizeof=4", dst_fk_opfile); status = add_fld(src_tbl, dst_fk_fld, str_meta_data, &itemp); cBYE(status); // Add output field to meta data sprintf(str_meta_data, "filename=%s:fldtype=char string:n_sizeof=0", opfile); status = add_fld(dst_tbl, src_fld, str_meta_data, &itemp); cBYE(status); fclose_if_non_null(cnt_ofp); // Add nn field if needed if ( is_any_null ) { status = add_aux_fld(dst_tbl, src_fld, nn_opfile, "nn", &itemp); cBYE(status); } else { unlink(nn_opfile); } // Add sz field status = add_aux_fld(dst_tbl, src_fld, sz_opfile, "sz", &itemp); cBYE(status); // Add count field to meta data sprintf(str_meta_data, "filename=%s:fldtype=int:n_sizeof=4", cnt_opfile); status = add_fld(dst_tbl, "cnt", str_meta_data, &itemp); cBYE(status); //----------------------------------------------------------- BYE: free_if_non_null(null_str); rs_munmap(X, nX); rs_munmap(src_fld_X, src_fld_nX); rs_munmap(cnt_X, cnt_nX); rs_munmap(out_X, out_nX); rs_munmap(dst_fk_X, dst_fk_nX); rs_munmap(nn_src_fld_X, nn_src_fld_nX); fclose_if_non_null(ofp); free_if_non_null(opfile); fclose_if_non_null(nn_ofp); free_if_non_null(nn_opfile); fclose_if_non_null(sz_ofp); free_if_non_null(sz_opfile); fclose_if_non_null(cnt_ofp); free_if_non_null(cnt_opfile); fclose_if_non_null(tmp_ofp); free_if_non_null(tmp_opfile); fclose_if_non_null(from_dst_ofp); free_if_non_null(from_dst_opfile); fclose_if_non_null(to_dst_ofp); free_if_non_null(to_dst_opfile); free_if_non_null(sz_src_fld); free_if_non_null(dst_fk_fld); free_if_non_null(str_maxlen); return(status); }
// START FUNC DECL int read_csv( FLD_META_TYPE *flds, int n_flds, char *infile, char *str_load_fld, char fld_delim, /* double quote character */ char fld_sep, /* comma */ char rec_delim, /* new line character */ bool ignore_hdr, /* whether to ignore header or not */ bool is_null_null, /* whether to convert "null" to null */ char ***ptr_nn_fnames, char ***ptr_sz_fnames, long long *ptr_num_rows ) // STOP FUNC DECL { int status = 0; int *load_fld = NULL; int n_load_fld = 0; char *X = NULL; size_t nX = 0; bool in_fld, is_fld_delim; long long num_rows = 0; long long num_cells = 0, fld_idx; long long lb, ub; FILE **fps = NULL; char **fnames = NULL; FILE **nn_fps = NULL; char **nn_fnames = NULL; FILE **sz_fps = NULL; char **sz_fnames = NULL; bool in_escape_mode = false, is_fld_delim_escaped = false; g_num_rows = 0; g_buflen = 8192; g_buffer = (char *)malloc(g_buflen * sizeof(char)); return_if_malloc_failed(g_buffer); zero_string(g_buffer, g_buflen); /*----------------------------------------------------------------------------*/ if ( n_flds <= 0 ) { go_BYE(-1); } fps = (FILE **)malloc(n_flds * sizeof(FILE *)); return_if_malloc_failed(fps); fnames = (char **)malloc(n_flds * sizeof(char *)); return_if_malloc_failed(fnames); nn_fps = (FILE **)malloc(n_flds * sizeof(FILE *)); return_if_malloc_failed(nn_fps); nn_fnames = (char **)malloc(n_flds * sizeof(char *)); return_if_malloc_failed(nn_fnames); sz_fps = (FILE **)malloc(n_flds * sizeof(FILE *)); return_if_malloc_failed(sz_fps); sz_fnames = (char **)malloc(n_flds * sizeof(char *)); return_if_malloc_failed(sz_fnames); for ( int i = 0; i < n_flds; i++ ) { fps[i] = NULL; nn_fps[i] = NULL; sz_fps[i] = NULL; fnames[i] = nn_fnames[i] = sz_fnames[i] = NULL; } /*----------------------------------------------------------------------------*/ if ( ( str_load_fld == NULL ) || ( *str_load_fld == '\0' ) ) { load_fld = (int *)malloc(n_flds * sizeof(int)); return_if_malloc_failed(load_fld); for ( int i = 0; i < n_flds; i++ ) { load_fld[i]= 1; } } status = parse_scalar_vals(str_load_fld, &load_fld, &n_load_fld); cBYE(status); if ( n_load_fld != n_flds ) { go_BYE(-1); } status = rs_mmap(infile, &X, &nX, 0); cBYE(status); lb = 0; fld_idx = 0; in_fld = false; is_fld_delim = false; for ( int i = 0; i < n_flds; i++ ) { if ( load_fld[i] == TRUE ) { status = open_temp_file(&(fps[i]), &(fnames[i])); cBYE(status); status = open_temp_file(&(nn_fps[i]), &(nn_fnames[i])); cBYE(status); if ( strcmp(flds[i].fldtype, "char string" ) == 0 ) { status = open_temp_file(&(sz_fps[i]), &(sz_fnames[i])); cBYE(status); } } } for ( long long i = 0; i < nX; i++ ) { // START: Error handling for escape conventions /* TODO: If I get an incorrect backslash, I simply replace it with a * space. This is incorrect. But I need it to read files created * with jspool utility from Jay Thomas. The correct thing to do is * to fix the utility. */ if ( in_escape_mode ) { if ( ( X[i] != '\\' ) && ( X[i] != '"' ) ) { /* replace the backslash that put is into escape mode wih a space */ X[i-1] = ' '; } } // STOP: Error handling for escape conventions // START: Handling bslash and dquote if ( X[i] == '\\' ) { if ( in_escape_mode ) { in_escape_mode = false; } else { in_escape_mode = true; } } // STOP: Handling bslash and dquote if ( X[i] == '"' ) { if ( in_escape_mode ) { is_fld_delim_escaped = true; in_escape_mode = false; } } if ( in_fld == false ) { in_fld = true; if ( X[i] == fld_delim ) { /* consume this character */ is_fld_delim = true; lb = i+1; } else { lb = i; if ( ( ( fld_idx < (n_flds-1) ) && ( X[i] == fld_sep ) ) || ( ( fld_idx == n_flds-1 ) && ( X[i] == rec_delim ) ) ) { /* field has terminated */ ub = i-1; if ( load_fld[fld_idx] == 1 ) { if ( ( ignore_hdr ) & ( num_rows == 0 ) ) { /* Do nothing */ } else { status = print_cell(X, lb, ub+1, flds[fld_idx].fldtype, flds[fld_idx].n_sizeof, is_null_null, fps[fld_idx], nn_fps[fld_idx], sz_fps[fld_idx]); cBYE(status); } } in_fld = false; num_cells++; if ( ( fld_idx == n_flds-1 ) && ( X[i] == rec_delim ) ) { num_rows++; g_num_rows = num_rows; } fld_idx++; if ( fld_idx == n_flds ) { fld_idx = 0; } } } } else { /* Now within field. Decide whether to continue or stop. */ if ( is_fld_delim ) { /* if field started with a fld_delim, it must end with one */ /* Note that we need to take care of escaping the dquote */ if ( ( X[i] == fld_delim ) && ( !is_fld_delim_escaped ) ) { /* next char must be fld_sep or rec_delim or we have come to eof */ if ( i+1 > nX ) { go_BYE(-1); } if ( ( fld_idx < (n_flds-1) ) && ( X[i+1] != fld_sep ) ) { go_BYE(-1); } if ( fld_idx == n_flds-1 ) { if ( X[i+1] != rec_delim ) { go_BYE(-1); } else { num_rows++; g_num_rows = num_rows; } } ub = i-1; if ( load_fld[fld_idx] == 1 ) { if ( ( ignore_hdr ) & ( num_rows == 0 ) ) { /* Do nothing */ } else { status = print_cell(X, lb, ub+1, flds[fld_idx].fldtype, flds[fld_idx].n_sizeof, is_null_null, fps[fld_idx], nn_fps[fld_idx], sz_fps[fld_idx]); cBYE(status); } } in_fld = false; is_fld_delim = false; fld_idx++; if ( fld_idx == n_flds ) { fld_idx = 0; } num_cells++; i++; /* Consume next char which must be fld_sep or rec_delim */ } } else { if ( ( i == nX ) || ( ( fld_idx < (n_flds-1) ) && ( X[i] == fld_sep ) ) || ( ( fld_idx == n_flds-1 ) && ( X[i] == rec_delim ) ) ) { /* field has terminated */ if ( i == nX ) { ub = i; } else { ub = i-1; } if ( load_fld[fld_idx] == 1 ) { if ( ( ignore_hdr ) & ( num_rows == 0 ) ) { /* Do nothing */ } else { status = print_cell(X, lb, ub+1, flds[fld_idx].fldtype, flds[fld_idx].n_sizeof, is_null_null, fps[fld_idx], nn_fps[fld_idx], sz_fps[fld_idx]); cBYE(status); } } in_fld = false; num_cells++; if ( ( fld_idx == n_flds-1 ) && ( X[i] == rec_delim ) ) { num_rows++; g_num_rows = num_rows; } fld_idx++; if ( fld_idx == n_flds ) { fld_idx = 0; } } } } is_fld_delim_escaped = false; } if ( num_cells != ( n_flds * num_rows ) ) { fprintf(stderr, "num_cells = %lld \n", num_cells); fprintf(stderr, "n_flds = %d \n", n_flds ); fprintf(stderr, "num_rows = %lld \n", num_rows); go_BYE(-1); } if ( ignore_hdr ) { num_rows--; // Because you have eliminated first row } /* TODO: Copy information into meta data structure */ for ( int i = 0; i < n_flds; i++ ) { if ( fnames[i] != NULL ) { // since not all fields are loaded strcpy(flds[i].filename, fnames[i]); } } for ( int i = 0; i < n_flds; i++ ) { if ( fnames != NULL ) { free_if_non_null(fnames[i]); } if ( fps != NULL ) { fclose_if_non_null(fps[i]); } if ( nn_fps != NULL ) { fclose_if_non_null(nn_fps[i]); } if ( sz_fps != NULL ) { fclose_if_non_null(sz_fps[i]); } } *ptr_nn_fnames = nn_fnames; *ptr_sz_fnames = sz_fnames; *ptr_num_rows = num_rows; BYE: rs_munmap(X, nX); free_if_non_null(load_fld); free_if_non_null(nn_fps); free_if_non_null(sz_fps); free_if_non_null(fnames); free_if_non_null(fps); free_if_non_null(g_buffer); return(status); }
int csync_rs_patch(const char *filename) { FILE *basis_file = 0, *delta_file = 0, *new_file = 0; int backup_errno; rs_stats_t stats; rs_result result; char *errstr = "?"; char tmpfname[MAXPATHLEN], newfname[MAXPATHLEN]; csync_debug(3, "Csync2 / Librsync: csync_rs_patch('%s')\n", filename); csync_debug(3, "Receiving delta_file from peer..\n"); delta_file = open_temp_file(tmpfname, prefixsubst(filename)); if ( !delta_file ) { errstr="creating delta temp file"; goto io_error; } if (unlink(tmpfname) < 0) { errstr="removing delta temp file"; goto io_error; } if ( csync_recv_file(delta_file) ) goto error; csync_debug(3, "Opening to be patched file on local host..\n"); basis_file = fopen(prefixsubst(filename), "rb"); if ( !basis_file ) { basis_file = open_temp_file(tmpfname, prefixsubst(filename)); if ( !basis_file ) { errstr="opening data file for reading"; goto io_error; } if (unlink(tmpfname) < 0) { errstr="removing data temp file"; goto io_error; } } csync_debug(3, "Opening temp file for new data on local host..\n"); new_file = open_temp_file(newfname, prefixsubst(filename)); if ( !new_file ) { errstr="creating new data temp file"; goto io_error; } csync_debug(3, "Running rs_patch_file() from librsync..\n"); result = rs_patch_file(basis_file, delta_file, new_file, &stats); if (result != RS_DONE) { csync_debug(0, "Internal error from rsync library!\n"); goto error; } csync_debug(3, "Renaming tmp file to data file..\n"); fclose(basis_file); #ifdef __CYGWIN__ /* TODO: needed? */ // This creates the file using the native windows API, bypassing // the cygwin wrappers and so making sure that we do not mess up the // permissions.. { char winfilename[MAX_PATH]; HANDLE winfh; cygwin_conv_to_win32_path(prefixsubst(filename), winfilename); winfh = CreateFile(TEXT(winfilename), GENERIC_WRITE, // open for writing 0, // do not share NULL, // default security CREATE_ALWAYS, // overwrite existing FILE_ATTRIBUTE_NORMAL | // normal file FILE_FLAG_OVERLAPPED, // asynchronous I/O NULL); // no attr. template if (winfh == INVALID_HANDLE_VALUE) { csync_debug(0, "Win32 I/O Error %d in rsync-patch: %s\n", (int)GetLastError(), winfilename); errno = EACCES; goto error; } CloseHandle(winfh); } #endif if (rename(newfname, prefixsubst(filename)) < 0) { errstr="renaming tmp file to to be patched file"; goto io_error; } csync_debug(3, "File has been patched successfully.\n"); fclose(delta_file); fclose(new_file); return 0; io_error: csync_debug(0, "I/O Error '%s' while %s in rsync-patch: %s\n", strerror(errno), errstr, prefixsubst(filename)); error:; backup_errno = errno; if ( delta_file ) fclose(delta_file); if ( basis_file ) fclose(basis_file); if ( new_file ) fclose(new_file); errno = backup_errno; return -1; }
static int get_data( char *tbl, char *fld, long long *ptr_nR, char **ptr_X, size_t *ptr_nX, char **ptr_opfile, int *ptr_fldtype, int *ptr_fldsz ) { int status = 0; int tbl_id = INT_MIN, fld_id = INT_MIN, nn_fld_id = INT_MIN; char *X = NULL; size_t nX = 0; char *nn_X = NULL; size_t nn_nX = 0; char *op_X = NULL; size_t op_nX = 0; char *endptr = NULL; char buffer[32]; long long nR = 0; FLD_TYPE *fld_meta = NULL, *nn_fld_meta = NULL; FILE *ofp = NULL; char *opfile = NULL; zero_string(buffer, 32); status = is_tbl(tbl, &tbl_id); cBYE(status); chk_range(tbl_id, 0, g_n_tbl); status = is_fld(NULL, tbl_id, fld, &fld_id); cBYE(status); chk_range(fld_id, 0, g_n_fld); nR = g_tbl[tbl_id].nR; fld_meta = &(g_fld[fld_id]); nn_fld_id = g_fld[fld_id].nn_fld_id; if ( nn_fld_id >= 0 ) { nn_fld_meta = &(g_fld[nn_fld_id]); status = rs_mmap(nn_fld_meta->filename, &nn_X, &nn_nX, 0); cBYE(status); } *ptr_fldsz = fld_meta->n_sizeof; status = mk_ifldtype(fld_meta->fldtype, ptr_fldtype); cBYE(status); switch ( *ptr_fldtype ) { case FLDTYPE_INT : case FLDTYPE_LONGLONG : /* all is well */ break; default : go_BYE(-1); } if ( nn_X == NULL ) { /* no nn field */ status = rs_mmap(fld_meta->filename, &X, &nX, 0); cBYE(status); *ptr_nR = nR; *ptr_X = X; *ptr_nX = nX; *ptr_opfile = NULL; } else { long long nn_nR = 0; status = f_to_s(tbl, nn_fld_meta->name, "sum", buffer); cBYE(status); nn_nR = strtol(buffer, &endptr, 10); if ( *endptr != '\0' ) { go_BYE(-1); } if ( ( nn_nR <= 0 ) || ( nn_nR >= nR ) ) { go_BYE(-1); } /* Now make a copy of just the non-null values */ status = rs_mmap(fld_meta->filename, &X, &nX, 0); cBYE(status); status = rs_mmap(nn_fld_meta->filename, &nn_X, &nn_nX, 0); cBYE(status); long long filesz = nn_nR * fld_meta->n_sizeof; status = open_temp_file(&ofp, &opfile, filesz); cBYE(status); fclose_if_non_null(ofp); status = mk_file(opfile, filesz); cBYE(status); status = rs_mmap(opfile, &op_X, &op_nX, 1); cBYE(status); status = copy_nn_vals(X, nR, nn_X, op_X, nn_nR, fld_meta->n_sizeof); cBYE(status); /* Cleanup and return values */ rs_munmap(X, nX); rs_munmap(nn_X, nn_nX); *ptr_nR = nn_nR; *ptr_X = op_X; *ptr_nX = op_nX; *ptr_opfile = opfile; } BYE: return(status); }
int csync_rs_delta(const char *filename) { FILE *sig_file = 0, *new_file = 0, *delta_file = 0; rs_result result; rs_signature_t *sumset; rs_stats_t stats; char tmpfname[MAXPATHLEN]; csync_debug(3, "Csync2 / Librsync: csync_rs_delta('%s')\n", filename); csync_debug(3, "Receiving sig_file from peer..\n"); sig_file = open_temp_file(tmpfname, prefixsubst(filename)); if ( !sig_file ) goto io_error; if (unlink(tmpfname) < 0) goto io_error; if ( csync_recv_file(sig_file) ) { fclose(sig_file); return -1; } result = rs_loadsig_file(sig_file, &sumset, &stats); if (result != RS_DONE) csync_fatal("Got an error from librsync, too bad!\n"); fclose(sig_file); csync_debug(3, "Opening new_file and delta_file..\n"); new_file = fopen(prefixsubst(filename), "rb"); if ( !new_file ) { int backup_errno = errno; csync_debug(0, "I/O Error '%s' while %s in rsync-delta: %s\n", strerror(errno), "opening data file for reading", filename); csync_send_error(); fclose(new_file); errno = backup_errno; return -1; } delta_file = open_temp_file(tmpfname, prefixsubst(filename)); if ( !delta_file ) goto io_error; if (unlink(tmpfname) < 0) goto io_error; csync_debug(3, "Running rs_build_hash_table() from librsync..\n"); result = rs_build_hash_table(sumset); if (result != RS_DONE) csync_fatal("Got an error from librsync, too bad!\n"); csync_debug(3, "Running rs_delta_file() from librsync..\n"); result = rs_delta_file(sumset, new_file, delta_file, &stats); if (result != RS_DONE) csync_fatal("Got an error from librsync, too bad!\n"); csync_debug(3, "Sending delta_file to peer..\n"); csync_send_file(delta_file); csync_debug(3, "Delta has been created successfully.\n"); rs_free_sumset(sumset); fclose(delta_file); fclose(new_file); return 0; io_error: csync_debug(0, "I/O Error '%s' in rsync-delta: %s\n", strerror(errno), prefixsubst(filename)); if (new_file) fclose(new_file); if (delta_file) fclose(delta_file); if (sig_file) fclose(sig_file); return -1; }
//--------------------------------------------------------------- // START FUNC DECL int f1opf2( char *tbl, char *f1, char *str_op_spec, char *f2 ) // STOP FUNC DECL { int status = 0; char *f1_X = NULL; size_t f1_nX = 0; char *nn_f1_X = NULL; size_t nn_f1_nX = 0; char *sz_f1_X = NULL; size_t sz_f1_nX = 0; FLD_TYPE *f1_meta = NULL, *nn_f1_meta = NULL, *sz_f1_meta = NULL; long long nR; int tbl_id = INT_MIN, f1_id = INT_MIN, nn_f1_id = INT_MIN, sz_f1_id = INT_MIN; int f2_id = INT_MIN, nn_f2_id = INT_MIN, sz_f2_id = INT_MIN; char str_meta_data[1024]; int f1type; char *op = NULL, *str_val = NULL; FILE *ofp = NULL; char *opfile = NULL; FILE *nn_ofp = NULL; char *nn_opfile = NULL; FILE *sz_ofp = NULL; char *sz_opfile = NULL; char *xform_enum = NULL, *hash_algo = NULL; //---------------------------------------------------------------- if ( ( tbl == NULL ) || ( *tbl == '\0' ) ) { go_BYE(-1); } if ( ( f1 == NULL ) || ( *f1 == '\0' ) ) { go_BYE(-1); } if ( ( f2 == NULL ) || ( *f2 == '\0' ) ) { go_BYE(-1); } if ( ( str_op_spec == NULL ) || ( *str_op_spec == '\0' ) ) { go_BYE(-1); } zero_string(str_meta_data, 1024); //-------------------------------------------------------- status = extract_name_value(str_op_spec, "op=", ":", &op); cBYE(status); if ( op == NULL ) { go_BYE(-1); } //-------------------------------------------------------- status = chk_if_ephemeral(&f2); cBYE(status); status = is_tbl(tbl, &tbl_id); cBYE(status); chk_range(tbl_id, 0, g_n_tbl); nR = g_tbl[tbl_id].nR; status = is_fld(NULL, tbl_id, f1, &f1_id); cBYE(status); chk_range(f1_id, 0, g_n_fld); f1_meta = &(g_fld[f1_id]); status = rs_mmap(f1_meta->filename, &f1_X, &f1_nX, 0); cBYE(status); // Get nn field for f1 if if it exists nn_f1_id = g_fld[f1_id].nn_fld_id; if ( nn_f1_id >= 0 ) { nn_f1_meta = &(g_fld[nn_f1_id]); status = rs_mmap(nn_f1_meta->filename, &nn_f1_X, &nn_f1_nX, 0); cBYE(status); } //--------------------------------------------- status = mk_ifldtype(f1_meta->fldtype, &f1type); /* TODO: Why do we get an error for 'char string' ? */ // If f1 type is 'char string', then get sz field if ( strcmp(f1_meta->fldtype, "char string") == 0 ) { sz_f1_id = g_fld[f1_id].sz_fld_id; chk_range(sz_f1_id, 0, g_n_fld); sz_f1_meta = &(g_fld[sz_f1_id]); status = rs_mmap(sz_f1_meta->filename, &sz_f1_X, &sz_f1_nX, 0); cBYE(status); } //-------------------------------------------------------- if ( strcmp(op, "xform") == 0 ) { bool is_some_null; status = open_temp_file(&ofp, &opfile, 0); cBYE(status); status = open_temp_file(&nn_ofp, &nn_opfile, 0); cBYE(status); status = open_temp_file(&sz_ofp, &sz_opfile, 0); cBYE(status); status = str_xform(nR, f1_X, (int *)sz_f1_X, nn_f1_X, str_op_spec, ofp, nn_ofp, sz_ofp, &is_some_null); cBYE(status); fclose_if_non_null(ofp); fclose_if_non_null(nn_ofp); fclose_if_non_null(sz_ofp); sprintf(str_meta_data, "filename=%s:fldtype=char string:n_sizeof=0", opfile); status = add_fld(tbl, f2, str_meta_data, &f2_id); cBYE(status); status = add_aux_fld(tbl, f2, sz_opfile, "sz", &sz_f2_id); cBYE(status); if ( is_some_null ) { status = add_aux_fld(tbl, f2, nn_opfile, "nn", &nn_f2_id); cBYE(status); } else { unlink(nn_opfile); } } else if ( strcmp(op, "hash") == 0 ) { int hash_len; char *endptr = NULL; status = extract_name_value(str_op_spec, "len=", ":", &str_val); cBYE(status); if ( str_val == NULL ) { go_BYE(-1); } hash_len = strtol(str_val, &endptr, 10); if ( *endptr != '\0' ) { go_BYE(-1); } status = extract_name_value(str_op_spec, "hash_algo=", ":", &hash_algo); cBYE(status); status = open_temp_file(&ofp, &opfile, 0); cBYE(status); status = f1opf2_hash(hash_algo, nR, f1_X, f1_nX, (int *)sz_f1_X, f1_meta->n_sizeof, nn_f1_X, hash_len, ofp); cBYE(status); fclose_if_non_null(ofp); sprintf(str_meta_data, "filename=%s", opfile); switch ( hash_len ) { case 4 : strcat(str_meta_data, ":fldtype=int:n_sizeof=4"); break; case 8 : strcat(str_meta_data, ":fldtype=long long:n_sizeof=8"); break; default : go_BYE(-1); break; } fclose_if_non_null(ofp); status = add_fld(tbl, f2, str_meta_data, &f2_id); cBYE(status); } else if ( ( strcmp(op, "conv") == 0 ) || ( strcmp(op, "dateconv" ) == 0 ) || ( strcmp(op, "bitcount" ) == 0 ) || ( strcmp(op, "sqrt" ) == 0 ) || ( strcmp(op, "!" ) == 0 ) || ( strcmp(op, "++" ) == 0 ) || ( strcmp(op, "--" ) == 0 ) || ( strcmp(op, "~" ) == 0 ) ) { status = vec_f1opf2(nR, f1type, str_meta_data, f1_X, nn_f1_X, sz_f1_X, op, str_op_spec, &opfile, &nn_opfile); cBYE(status); status = add_fld(tbl, f2, str_meta_data, &f2_id); cBYE(status); if ( nn_opfile != NULL ) { status = add_aux_fld(tbl, f2, nn_opfile, "nn", &nn_f2_id); cBYE(status); } } else if ( strcmp(op, "shift") == 0 ) { int ival; char *endptr = NULL; status = extract_name_value(str_op_spec, "val=", ":", &str_val); cBYE(status); ival = strtol(str_val, &endptr, 10); if ( *endptr != '\0' ) { fprintf(stderr, "invalid shift specifier = [%s] \n", str_val); go_BYE(-1); } if ( ( ival == 0 ) || ( ival > MAX_SHIFT ) || ( ival < -MAX_SHIFT ) || (ival >= nR )) { fprintf(stderr, "shift = [%d] is out of bounds \n", ival); go_BYE(-1); } status = open_temp_file(&ofp, &opfile, 0); cBYE(status); status = open_temp_file(&nn_ofp, &nn_opfile, 0); cBYE(status); status = f1opf2_shift(f1_X, nn_f1_X, nR, f1_meta->fldtype, ival, ofp, nn_ofp); cBYE(status); fclose_if_non_null(ofp); fclose_if_non_null(nn_ofp); sprintf(str_meta_data, "filename=%s:fldtype=%s:n_sizeof=%d",opfile, f1_meta->fldtype, f1_meta->n_sizeof); status = add_fld(tbl, f2, str_meta_data, &f2_id); cBYE(status); status = add_aux_fld(tbl, f2, nn_opfile, "nn", &nn_f2_id); cBYE(status); } else if ( strcmp(op, "cum") == 0 ) { /* TODO: Document. If you do a cum, resultant field is all def */ char *new_fld_type = NULL; int optype, n_sizeof; char *str_reset_on = NULL, *str_reset_to = NULL, *endptr = NULL; long long reset_on = 0, reset_to = 0; bool is_reset = false; status = extract_name_value(str_op_spec, "newtype=", ":", &new_fld_type); cBYE(status); status = extract_name_value(str_op_spec, "reset_on=", ":", &str_reset_on); cBYE(status); if ( str_reset_on != NULL ) { reset_on = strtoll(str_reset_on, &endptr, 10); if ( *endptr != '\0' ) { go_BYE(-1); } is_reset = true; status = extract_name_value(str_op_spec, "reset_to=", ":", &str_reset_to); cBYE(status); if ( str_reset_to == NULL ) { go_BYE(-1); } reset_to = strtoll(str_reset_to, &endptr, 10); if ( *endptr != '\0' ) { go_BYE(-1); } } if ( new_fld_type == NULL ) { optype = f1type; new_fld_type = strdup(f1_meta->fldtype); n_sizeof = f1_meta->n_sizeof; } else { if ( strcmp(new_fld_type, "int") == 0 ) { optype = FLDTYPE_INT; n_sizeof = sizeof(int); } else if ( strcmp(new_fld_type, "long long") == 0 ) { optype = FLDTYPE_LONGLONG; n_sizeof = sizeof(long long); } else { go_BYE(-1); } } status = f1opf2_cum(f1_X, nR, f1type, optype, is_reset, reset_on, reset_to, &opfile); cBYE(status); sprintf(str_meta_data, "filename=%s:fldtype=%s:n_sizeof=%d",opfile, new_fld_type, n_sizeof); status = add_fld(tbl, f2, str_meta_data, &f2_id); cBYE(status); free_if_non_null(new_fld_type); free_if_non_null(str_reset_on); free_if_non_null(str_reset_to); } else { fprintf(stderr, "Invalid op = [%s] \n", op); go_BYE(-1); } BYE: fclose_if_non_null(ofp); fclose_if_non_null(nn_ofp); g_write_to_temp_dir = false; rs_munmap(f1_X, f1_nX); rs_munmap(nn_f1_X, nn_f1_nX); rs_munmap(sz_f1_X, sz_f1_nX); free_if_non_null(op); free_if_non_null(xform_enum); free_if_non_null(hash_algo); free_if_non_null(str_val); free_if_non_null(opfile); free_if_non_null(nn_opfile); free_if_non_null(sz_opfile); return(status); }
int csync_rs_check(const char *filename, int isreg) { FILE *basis_file = 0, *sig_file = 0; char buffer1[512], buffer2[512]; int rc, chunk, found_diff = 0; int backup_errno; rs_stats_t stats; rs_result result; long size; char tmpfname[MAXPATHLEN]; csync_debug(3, "Csync2 / Librsync: csync_rs_check('%s', %d [%s])\n", filename, isreg, isreg ? "regular file" : "non-regular file"); csync_debug(3, "Opening basis_file and sig_file..\n"); sig_file = open_temp_file(tmpfname, prefixsubst(filename)); if ( !sig_file ) goto io_error; if (unlink(tmpfname) < 0) goto io_error; basis_file = fopen(prefixsubst(filename), "rb"); if ( !basis_file ) { /* ?? why a tmp file? */ basis_file = open_temp_file(tmpfname, prefixsubst(filename)); if ( !basis_file ) goto io_error; if (unlink(tmpfname) < 0) goto io_error; } if ( isreg ) { csync_debug(3, "Running rs_sig_file() from librsync....\n"); result = rs_sig_file(basis_file, sig_file, RS_DEFAULT_BLOCK_LEN, RS_DEFAULT_STRONG_LEN, &stats); if (result != RS_DONE) { csync_debug(0, "Internal error from rsync library!\n"); goto error; } } fclose(basis_file); basis_file = 0; { char line[100]; csync_debug(3, "Reading signature size from peer....\n"); if ( !conn_gets(line, 100) || sscanf(line, "octet-stream %ld\n", &size) != 1 ) csync_fatal("Format-error while receiving data.\n"); } fflush(sig_file); if ( size != ftell(sig_file) ) { csync_debug(2, "Signature size differs: local=%d, peer=%d\n", ftell(sig_file), size); found_diff = 1; } rewind(sig_file); csync_debug(3, "Receiving %ld bytes ..\n", size); while ( size > 0 ) { chunk = size > 512 ? 512 : size; rc = conn_read(buffer1, chunk); if ( rc <= 0 ) csync_fatal("Read-error while receiving data.\n"); chunk = rc; if ( fread(buffer2, chunk, 1, sig_file) != 1 ) { csync_debug(2, "Found EOF in local sig file.\n"); found_diff = 1; } if ( memcmp(buffer1, buffer2, chunk) ) { csync_debug(2, "Found diff in sig at -%d:-%d\n", size, size-chunk); found_diff = 1; } size -= chunk; csync_debug(3, "Got %d bytes, %ld bytes left ..\n", chunk, size); } csync_debug(3, "File has been checked successfully (%s).\n", found_diff ? "difference found" : "files are equal"); fclose(sig_file); return found_diff; io_error: csync_debug(0, "I/O Error '%s' in rsync-check: %s\n", strerror(errno), prefixsubst(filename)); error:; backup_errno = errno; if ( basis_file ) fclose(basis_file); if ( sig_file ) fclose(sig_file); errno = backup_errno; return -1; }
//--------------------------------------------------------------- // START FUNC DECL int srt_join( char *docroot, sqlite3 *in_db, char *src_tbl, char *src_lnk, char *src_val, char *dst_tbl, char *dst_lnk, char *dst_fld, char *op ) // STOP FUNC DECL { int status = 0; sqlite3 *db = NULL; char *src_val_X = NULL; size_t src_val_nX = 0; char *src_lnk_X = NULL; size_t src_lnk_nX = 0; char *nn_dst_val_X = NULL; size_t nn_dst_val_nX = 0; char *dst_val_X = NULL; size_t dst_val_nX = 0; char *dst_lnk_X = NULL; size_t dst_lnk_nX = 0; FLD_META_TYPE src_val_meta, src_lnk_meta; FLD_META_TYPE dst_lnk_meta; long long src_nR, dst_nR; // counts all rows char str_meta_data[1024]; int ijoin_op; bool is_any_null = false; int src_tbl_id, dst_tbl_id; bool b_is_tbl; FILE *ofp = NULL, *nn_ofp = NULL; char *opfile = NULL, *nn_opfile = NULL; //---------------------------------------------------------------- zero_string(str_meta_data, 1024); status = mk_mjoin_op(op, &ijoin_op); cBYE(status); status = open_db_if_needed(docroot, in_db, &db); cBYE(status); zero_fld_meta(&src_val_meta); zero_fld_meta(&src_lnk_meta); zero_fld_meta(&dst_lnk_meta); //---------------------------------------------------------------- // Get meta-data for all necessary fields status = is_tbl(docroot, db, src_tbl, &b_is_tbl, &src_tbl_id); cBYE(status); if ( b_is_tbl == false ) { fprintf(stderr, "tbl [%s] not found\n", src_tbl); go_BYE(-1); } status = is_tbl(docroot, db, dst_tbl, &b_is_tbl, &dst_tbl_id); cBYE(status); if ( b_is_tbl == false ) { fprintf(stderr, "tbl [%s] not found\n", src_tbl); go_BYE(-1); } status = fld_meta(docroot, db, src_tbl, src_lnk, -1, &src_lnk_meta); cBYE(status); if ( ( src_val != NULL ) && ( *src_val != '\0' ) ) { status = fld_meta(docroot, db, src_tbl, src_val, -1, &src_val_meta); cBYE(status); } status = fld_meta(docroot, db, dst_tbl, dst_lnk, -1, &dst_lnk_meta); cBYE(status); status = internal_get_nR(db, src_tbl_id, &src_nR); cBYE(status); status = internal_get_nR(db, dst_tbl_id, &dst_nR); cBYE(status); //---------------------------------------------------------------- // Get pointer access to all necessary fields status = rs_mmap(src_lnk_meta.filename, &src_lnk_X, &src_lnk_nX, 0); cBYE(status); if ( ( src_val != NULL ) && ( *src_val != '\0' ) ) { status = rs_mmap(src_val_meta.filename, &src_val_X, &src_val_nX, 0); cBYE(status); } status = rs_mmap(dst_lnk_meta.filename, &dst_lnk_X, &dst_lnk_nX, 0); cBYE(status); //-------------------------------------------------------- // Create output data files status = open_temp_file(&ofp, &opfile); cBYE(status); fclose_if_non_null(ofp); status = open_temp_file(&nn_ofp, &nn_opfile); cBYE(status); fclose_if_non_null(nn_ofp); if ( ( src_val != NULL ) && ( *src_val != '\0' ) ) { status = mk_file(opfile, src_val_meta.n_sizeof * dst_nR); cBYE(status); status = rs_mmap(opfile, &dst_val_X, &dst_val_nX, 1); cBYE(status); } status = mk_file(nn_opfile, sizeof(char) * dst_nR); cBYE(status); status = rs_mmap(nn_opfile, &nn_dst_val_X, &nn_dst_val_nX, 1); cBYE(status); //-------------------------------------------------------- // Core join if ( ( strcmp(src_lnk_meta.fldtype, "int") == 0 ) && ( strcmp(src_lnk_meta.fldtype, "int") == 0 ) && ( strcmp(dst_lnk_meta.fldtype, "int") == 0 ) ) { status = core_srt_join_I_I_I( (int *)src_lnk_X, (int *)src_val_X, src_nR, (int *)dst_lnk_X, (int *)dst_val_X, nn_dst_val_X, 0, dst_nR, ijoin_op, &is_any_null); } else if ( ( strcmp(src_lnk_meta.fldtype, "int") == 0 ) && ( strcmp(src_lnk_meta.fldtype, "int") == 0 ) && ( strcmp(dst_lnk_meta.fldtype, "long long") == 0 ) ) { status = core_srt_join_I_I_L( (int *)src_lnk_X, (int *)src_val_X, src_nR, (long long *)dst_lnk_X, (int *)dst_val_X, nn_dst_val_X, 0, dst_nR, ijoin_op, &is_any_null); } else if ( ( strcmp(src_lnk_meta.fldtype, "int") == 0 ) && ( strcmp(src_lnk_meta.fldtype, "long long") == 0 ) && ( strcmp(dst_lnk_meta.fldtype, "int") == 0 ) ) { status = core_srt_join_I_L_I( (int *)src_lnk_X, (long long *)src_val_X, src_nR, (int *)dst_lnk_X, (long long *)dst_val_X, nn_dst_val_X, 0, dst_nR, ijoin_op, &is_any_null); } else if ( ( strcmp(src_lnk_meta.fldtype, "int") == 0 ) && ( strcmp(src_lnk_meta.fldtype, "long long") == 0 ) && ( strcmp(dst_lnk_meta.fldtype, "long long") == 0 ) ) { status = core_srt_join_I_L_L((int *)src_lnk_X, (long long *)src_val_X, src_nR, (long long *)dst_lnk_X, (long long *)dst_val_X, nn_dst_val_X, 0, dst_nR, ijoin_op, &is_any_null); } else if ( ( strcmp(src_lnk_meta.fldtype, "long long") == 0 ) && ( strcmp(src_lnk_meta.fldtype, "int") == 0 ) && ( strcmp(dst_lnk_meta.fldtype, "int") == 0 ) ) { status = core_srt_join_L_I_I( (long long *)src_lnk_X, (int *)src_val_X, src_nR, (int *)dst_lnk_X, (int *)dst_val_X, nn_dst_val_X, 0, dst_nR, ijoin_op, &is_any_null); } else if ( ( strcmp(src_lnk_meta.fldtype, "long long") == 0 ) && ( strcmp(src_lnk_meta.fldtype, "int") == 0 ) && ( strcmp(dst_lnk_meta.fldtype, "long long") == 0 ) ) { status = core_srt_join_L_I_L( (long long *)src_lnk_X, (int *)src_val_X, src_nR, (long long *)dst_lnk_X, (int *)dst_val_X, nn_dst_val_X, 0, dst_nR, ijoin_op, &is_any_null); } else if ( ( strcmp(src_lnk_meta.fldtype, "long long") == 0 ) && ( strcmp(src_lnk_meta.fldtype, "long long") == 0 ) && ( strcmp(dst_lnk_meta.fldtype, "int") == 0 ) ) { status = core_srt_join_L_L_I( (long long *)src_lnk_X, (long long *)src_val_X, src_nR, (int *)dst_lnk_X, (long long *)dst_val_X, nn_dst_val_X, 0, dst_nR, ijoin_op, &is_any_null); } else if ( ( strcmp(src_lnk_meta.fldtype, "long long") == 0 ) && ( strcmp(src_lnk_meta.fldtype, "long long") == 0 ) && ( strcmp(dst_lnk_meta.fldtype, "long long") == 0 ) ) { status = core_srt_join_L_L_L( (long long *)src_lnk_X, (long long *)src_val_X, src_nR, (long long *)dst_lnk_X, (long long *)dst_val_X, nn_dst_val_X, 0, dst_nR, ijoin_op, &is_any_null); } else { go_BYE(-1); } //-------------------------------------------------------- // Add output field to meta data if ( ( src_val != NULL ) && ( *src_val != '\0' ) ) { sprintf(str_meta_data,"filename=%s:n_sizeof=%d:fldtype=%s", opfile, src_val_meta.n_sizeof, src_val_meta.fldtype); status = add_fld(docroot, db, dst_tbl, dst_fld, str_meta_data); cBYE(status); if ( is_any_null == true ) { status = add_aux_fld(docroot, db, dst_tbl, dst_fld, nn_opfile, "nn"); cBYE(status); } else { unlink(nn_opfile); free_if_non_null(nn_opfile); } } else { sprintf(str_meta_data,"filename=%s:n_sizeof=%ld:fldtype=bool", nn_opfile, sizeof(char)); status = add_fld(docroot, db, dst_tbl, dst_fld, str_meta_data); cBYE(status); } BYE: if ( in_db == NULL ) { sqlite3_close(db); } rs_munmap(src_val_X, src_val_nX); rs_munmap(src_lnk_X, src_lnk_nX); rs_munmap(dst_val_X, dst_val_nX); rs_munmap(nn_dst_val_X, nn_dst_val_nX); rs_munmap(dst_lnk_X, dst_lnk_nX); free_if_non_null(nn_opfile); free_if_non_null(opfile); return(status); }
off_t oggedit_write_flac_metadata(DB_FILE *in, const char *fname, const off_t offset, const int num_tags, char **tags) { off_t res; char tempname[PATH_MAX] = ""; ogg_packet **headers = NULL; char *vendor = NULL; ogg_sync_state oy; ogg_sync_init(&oy); /* Original file must be writable whichever way we update it */ FILE *out = fopen(fname, "r+b"); if (!out) { res = OGGEDIT_CANNOT_UPDATE_FILE; goto cleanup; } /* See if we can write the tags packet directly into the existing file ... */ if (!(headers = metadata_block_packets(in, &oy, offset, &vendor, (int *)&res))) goto cleanup; const off_t stream_size_k = in->vfs->getlength(in) / 1000; // use file size for now const size_t metadata_size = 4 + vc_size(vendor, num_tags, tags); ptrdiff_t padding = headers[0]->bytes - metadata_size; if (stream_size_k < 1000 || padding < 0 || (headers[1] && padding > 0) || padding > stream_size_k+metadata_size) { res = open_temp_file(fname, tempname, &out); if (res) { goto cleanup; } } /* Re-pad if writing the whole file */ if (*tempname) { size_t i = 0, j = 0; while (headers[i]) { headers[j++] = headers[i]; while (headers[++i] && (headers[i]->packet[0] & 0x3F) == PADTYPE) { ogg_packet_clear(headers[i]); free(headers[i]); } } headers[j] = NULL; padding = headers[1] || stream_size_k < 900 ? 0 : stream_size_k < 10000 ? 1024 : stream_size_k < 100000 ? 8192 : 65536; } /* Write pages until we reach the correct comment header */ ogg_page og; const int flac_serial = copy_up_to_codec(in, out, &oy, &og, *tempname ? 0 : offset, offset, FLACNAME); if (flac_serial <= OGGEDIT_EOF) { res = flac_serial; goto cleanup; } if ((res = copy_up_to_header(in, out, &oy, &og, flac_serial)) <= OGGEDIT_EOF) goto cleanup; const long pageno = write_metadata_block_packets(out, flac_serial, vendor, num_tags, tags, padding, headers); if (pageno < OGGEDIT_EOF) { res = pageno; goto cleanup; } /* If we have tempfile, copy the remaining pages */ if (*tempname) { if ((res = copy_remaining_pages(in, out, &oy, flac_serial, pageno)) <= OGGEDIT_EOF) goto cleanup; if (rename(tempname, fname)) { res = OGGEDIT_RENAME_FAILED; goto cleanup; } } res = file_size(fname); cleanup: clear_header_list(headers); cleanup(in, out, &oy, vendor); if (res < OGGEDIT_OK) unlink(tempname); return res; }
/* TODO: This routine needs more work to deal with escaping dquotes and * backslashes and dealing with eoln within the quotes */ int main( int argc, char **argv ) { int status = 0; char *infile = NULL; char *outfile_str = NULL; char *outfile_sz = NULL; char *outfile_nn = NULL; FILE *ofp = NULL, *szfp = NULL, *nnfp = NULL; char *X = NULL; size_t nX = 0; long long nR = 0, start_loc = -1, stop_loc =-1, len,fld_len; char nullc = '\0'; bool is_in_str, pr_str; char c_one = 1; char c_zero = 0; if ( argc != 2 ) { go_BYE(-1); } infile = argv[1]; status = open_temp_file(&ofp, &outfile_str, 0); cBYE(status); status = open_temp_file(&szfp, &outfile_sz, 0); cBYE(status); status = open_temp_file(&nnfp, &outfile_nn, 0); cBYE(status); status = rs_mmap(infile, &X, &nX, 0); cBYE(status); nR = 0; is_in_str = false; pr_str = false; for ( size_t i = 0; i < nX; i++ ) { if ( X[i] == '"' ) { if ( is_in_str == false ) { is_in_str = true; start_loc = i+1; } else { is_in_str = false; stop_loc = i-1; pr_str = true; } } if ( pr_str == true ) { if ( ( stop_loc < 0 ) || ( start_loc < 0 ) || ( ( start_loc - stop_loc ) > 1 ) ) { fprintf(stderr, "Error at line %lld, position %lld of %lld \n", nR, (long long)i, (long long)nX); go_BYE(-1); } nR++; pr_str = false; len = stop_loc - start_loc + 1; // + 1 bcos both are inclusive if ( len > 0 ) { fwrite(X+start_loc, sizeof(char), len, ofp); } if ( len > 0 ) { fwrite(&c_one, sizeof(char), 1, nnfp); } else { fwrite(&c_zero, sizeof(char), 1, nnfp); } fwrite(&nullc, sizeof(char), 1, ofp); fld_len = len + 1; fwrite(&fld_len, sizeof(int), 1, szfp); // Consume the eoln if it exists if ( i+1 < nX ) { if ( X[i+1] != '\n' ) { fprintf(stderr, "Error at line %lld, position %lld of %lld \n", nR, (long long)i, (long long)nX); go_BYE(-1); } i++; } } } fprintf(stdout, "%lld:%s:%s:%s", nR, outfile_str, outfile_sz, outfile_nn); BYE: fclose_if_non_null(ofp); fclose_if_non_null(szfp); fclose_if_non_null(nnfp); rs_munmap(X, nX); return(status); }
//--------------------------------------------------------------- // START FUNC DECL int xfer_string( FLD_TYPE *dst_idx_meta, char *src_fld_X, char *sz_src_fld_X, long long src_nR, char *dst_tbl, char *dst_fld, char *dst_idx_X, char *nn_dst_idx_X, long long dst_nR ) // STOP FUNC DECL { int status = 0; char str_meta_data[256]; char *szX = NULL; size_t n_szX= 0; int *szptr = NULL; char *nnX = NULL; size_t n_nnX= 0; char *nnptr = NULL; int dst_fld_id, nn_dst_fld_id, sz_dst_fld_id; FILE *ofp = NULL; char *opfile = NULL; FILE *nn_ofp = NULL; char *nn_opfile = NULL; FILE *sz_ofp = NULL; char *sz_opfile = NULL; long long *offset_src_fld = NULL; long long *ll_dst_idx = NULL; bool is_any_null = false; //-------------------------------------------------------- status = open_temp_file(&ofp, &opfile, 0); cBYE(status); /* We cannot allocate storage for this since we do not know how much * to allocate a-priori */ /* Open output fields for size and nn in mmmap mode */ long long filesz = dst_nR * sizeof(int); status = open_temp_file(&sz_ofp, &sz_opfile, filesz); cBYE(status); fclose_if_non_null(sz_ofp); status = mk_file(sz_opfile, filesz); cBYE(status); status = rs_mmap(sz_opfile, &szX, &n_szX, 1); cBYE(status); szptr = (int *)szX; long long nnfilesz = dst_nR * sizeof(char); status = open_temp_file(&nn_ofp, &nn_opfile, nnfilesz); cBYE(status); fclose_if_non_null(nn_ofp); status = mk_file(nn_opfile, nnfilesz); cBYE(status); status = rs_mmap(nn_opfile, &nnX, &n_nnX, 1); cBYE(status); nnptr = (char *)nnX; /*-------------------------------------------------------------*/ status = mk_offset((int *)sz_src_fld_X, src_nR, &offset_src_fld); cBYE(status); /* Convert from int to longlong if needed */ if ( strcmp(dst_idx_meta->fldtype, "int") == 0 ) { ll_dst_idx = malloc(dst_nR * sizeof(long long)); return_if_malloc_failed(ll_dst_idx); if ( dst_nR >= INT_MAX ) { fprintf(stderr, "TO BE IMPLEMENTED\n"); go_BYE(-1); } conv_int_to_longlong((int *)dst_idx_X, (int)dst_nR, ll_dst_idx); } else { ll_dst_idx = (long long *)dst_idx_X; } /*-------------------------------------------------------------*/ for ( long long i = 0; i < dst_nR; i++ ) { int *i_sz_src_fld_ptr = (int *)sz_src_fld_X; long long idx; size_t nw; char *testptr; char nullc = 0; long long offset; int sz; if ( ( nn_dst_idx_X != NULL ) && ( nn_dst_idx_X[i] == FALSE ) ) { // Output null value is_any_null = true; nnptr[i] = FALSE; szptr[i] = 1; nw = fwrite(&nullc, sizeof(char), 1, ofp); // Null terminate if ( nw != 1 ) { go_BYE(-1); } } else { idx = ll_dst_idx[i]; if ( ( idx < 0 ) || ( idx >= src_nR ) ) { go_BYE(-1); } sz = i_sz_src_fld_ptr[idx]; offset = offset_src_fld[idx]; // This checks that strings are null terminated testptr = src_fld_X + offset + sz - 1; if ( *testptr != '\0' ) { go_BYE(-1); } szptr[i] = sz; nnptr[i] = TRUE; nw = fwrite(src_fld_X + offset, sizeof(char), sz, ofp); if ( nw != sz ) { go_BYE(-1); } } } fclose_if_non_null(ofp); rs_munmap(szX, n_szX); rs_munmap(nnX, n_nnX); // Add output field to meta data zero_string(str_meta_data, 256); strcpy(str_meta_data, "fldtype=char string:n_sizeof=0"); strcat(str_meta_data, ":filename="); strcat(str_meta_data, opfile); status = add_fld(dst_tbl, dst_fld, str_meta_data, &dst_fld_id); cBYE(status); // Add size field status = add_aux_fld(dst_tbl, dst_fld, sz_opfile, "sz", &sz_dst_fld_id); cBYE(status); if ( is_any_null ) { status = add_aux_fld(dst_tbl, dst_fld, nn_opfile, "nn", &nn_dst_fld_id); cBYE(status); } else { unlink(nn_opfile); } BYE: if ( strcmp(dst_idx_meta->fldtype, "int") == 0 ) { free_if_non_null(ll_dst_idx); } fclose_if_non_null(ofp); fclose_if_non_null(nn_ofp); fclose_if_non_null(sz_ofp); free_if_non_null(opfile); free_if_non_null(nn_opfile); free_if_non_null(sz_opfile); free_if_non_null(offset_src_fld); return(status); }
//--------------------------------------------------------------- // START FUNC DECL int crossprod( char *t1, char *f1, char *t2, char *f2, char *t3 ) // STOP FUNC DECL { int status = 0; char *Y1 = NULL; size_t nY1 = 0; char *Y2 = NULL; size_t nY2 = 0; char *f1_X = NULL; size_t f1_nX = 0; char *f1_opfile = NULL; int f1type, f2type; char *f2_X = NULL; size_t f2_nX = 0; char *f2_opfile = NULL; long long f1size, f2size; int t2f1_fld_id = INT_MIN, t2f2_fld_id = INT_MIN; int t3_id = INT_MIN, itemp; long long chk_nR1 = 0, nR1, nR2, nR3; char str_meta_data[1024]; char *t3f1_opfile = NULL, *t3f2_opfile = NULL; FILE *ofp = NULL; char buffer[32]; // For multi-threading int rc; // result code for thread create pthread_t threads[MAX_NUM_THREADS]; pthread_attr_t attr; void *thread_status; //---------------------------------------------------------------- zero_string(str_meta_data, 1024); zero_string(buffer, 32); if ( strcmp(f1, f2) == 0 ) { go_BYE(-1); } /* Remove f1 != f2 restriction later. To do so, we need to specify * fields of t3 explicitly */ //---------------------------------------------------------------- status = get_data(t1, f1, &nR1, &f1_X, &f1_nX, &f1_opfile, &f1type, &f1size); cBYE(status); status = get_data(t2, f2, &nR2, &f2_X, &f2_nX, &f2_opfile, &f2type, &f2size); cBYE(status); nR3 = nR1 * nR2; if ( nR3 == 0 ) { fprintf(stderr, "No data to create t3 \n"); goto BYE; } // Create storage for field 1 in Table t3 */ long long filesz = nR3 * f1size; status = open_temp_file(&ofp, &t3f1_opfile, filesz); cBYE(status); fclose_if_non_null(ofp); status = mk_file(t3f1_opfile, filesz); cBYE(status); status = rs_mmap(t3f1_opfile, &Y1, &nY1, 1); cBYE(status); // Create storage for field 2 in Table t3 */ filesz = nR3 * f2size; status = open_temp_file(&ofp, &t3f2_opfile, filesz); cBYE(status); fclose_if_non_null(ofp); status = mk_file(t3f2_opfile, filesz); cBYE(status); status = rs_mmap(t3f2_opfile, &Y2, &nY2, 1); cBYE(status); //---------------------------------------------------------------- /* Set up parallelism computations. Parallelization strategy is * simple. Partition field 1 (nR1 rows) among the threads */ g_nR1 = nR1; g_nR2 = nR2; g_nR3 = nR3; g_f1type = f1type; g_f2type = f2type; g_f1size = f1size; g_f2size = f2size; g_f1_X = f1_X; g_f2_X = f2_X; g_Y1 = Y1; g_Y2 = Y2; for ( int i = 0; i < MAX_NUM_THREADS; i++ ) { g_thread_id[i] = i; g_num_rows_processed[i] = 0; } status = get_num_threads(&g_nT); cBYE(status); //-------------------------------------------- #define MIN_ROWS_FOR_CROSSPROD 4 // 1024 if ( nR1 <= MIN_ROWS_FOR_CROSSPROD ) { g_nT = 1; } /* Don't create more threads than you can use */ if ( g_nT > nR1 ) { g_nT = nR1; } if ( g_nT == 1 ) { core_crossprod(&(g_thread_id[0])); chk_nR1 = g_num_rows_processed[0]; } else { /* Create threads */ pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); for ( int t = 0; t < g_nT; t++ ) { rc = pthread_create(&threads[t], NULL, core_crossprod, &(g_thread_id[t])); if ( rc ) { go_BYE(-1); } } /* Free attribute and wait for the other threads */ pthread_attr_destroy(&attr); for ( int t = 0; t < g_nT; t++ ) { rc = pthread_join(threads[t], &thread_status); if ( rc ) { go_BYE(-1); } chk_nR1 += g_num_rows_processed[t]; } } if ( chk_nR1 != nR1 ) { go_BYE(-1); } //---------------------------------------------------------------- // Add output fields to t3 meta data status = is_tbl(t3, &t3_id); cBYE(status); if ( t3_id >= 0 ) { status = del_tbl(NULL, t3_id); cBYE(status); } sprintf(buffer, "%lld", nR3); status = add_tbl(t3, buffer, &itemp); cBYE(status); sprintf(str_meta_data, "fldtype=int:n_sizeof=%u:filename=%s", f1size, t3f1_opfile); status = add_fld(t3, f1, str_meta_data, &t2f1_fld_id); cBYE(status); zero_string(str_meta_data, 1024); sprintf(str_meta_data, "fldtype=int:n_sizeof=%u:filename=%s", f2size, t3f2_opfile); status = add_fld(t3, f2, str_meta_data, &t2f2_fld_id); cBYE(status); BYE: fclose_if_non_null(ofp); rs_munmap(f1_X, f1_nX); rs_munmap(f2_X, f2_nX); if ( f1_opfile != NULL ) { unlink(f1_opfile); free_if_non_null(f1_opfile); } if ( f2_opfile != NULL ) { unlink(f2_opfile); free_if_non_null(f2_opfile); } free_if_non_null(t3f1_opfile); free_if_non_null(t3f2_opfile); return(status); }
//--------------------------------------------------------------- // START FUNC DECL int dup_fld( char *tbl, char *f1, char *f2 ) // STOP FUNC DECL { int status = 0; char *Y = NULL; size_t nY = 0; char *X = NULL; size_t nX = 0; char *nn_X = NULL; size_t nn_nX = 0; char *sz_X = NULL; size_t sz_nX = 0; FILE *ofp = NULL; char *opfile = NULL; FILE *nn_ofp = NULL; char *nn_opfile = NULL; FILE *sz_ofp = NULL; char *sz_opfile = NULL; int tbl_id, f1_fld_id, f2_fld_id, nn_fld_id, sz_fld_id; //---------------------------------------------------------------- if ( ( tbl == NULL ) || ( *tbl == '\0' ) ) { go_BYE(-1); } if ( ( f1 == NULL ) || ( *f1 == '\0' ) ) { go_BYE(-1); } if ( ( f2 == NULL ) || ( *f2 == '\0' ) ) { go_BYE(-1); } if ( strcmp(f1, f2) == 0 ) { go_BYE(-1); } //-------------------------------------------------------- status = is_tbl(tbl, &tbl_id); cBYE(status); chk_range(tbl_id, 0, g_n_tbl); status = is_fld(NULL, tbl_id, f1, &f1_fld_id); cBYE(status); chk_range(f1_fld_id, 0, g_n_fld); status = is_fld(NULL, tbl_id, f2, &f2_fld_id); cBYE(status); if ( f2_fld_id >= 0 ) { status = del_fld(NULL, tbl_id, NULL, f2_fld_id); cBYE(status); } // Open input file status = rs_mmap(g_fld[f1_fld_id].filename, &X, &nX, 0); cBYE(status); // Create a copy of the data status = open_temp_file(&ofp, &opfile, nX); cBYE(status); fclose_if_non_null(ofp); status = mk_file(opfile, nX); cBYE(status); status = rs_mmap(opfile, &Y, &nY, 1); memcpy(Y, X, nX); rs_munmap(Y, nY); // Get empty spot for f2 status = get_empty_fld(&f2_fld_id); cBYE(status); // f2 is same as f1 except for data and "is_external" g_fld[f2_fld_id] = g_fld[f1_fld_id]; g_fld[f2_fld_id].is_external = false; g_fld[f2_fld_id].nn_fld_id = -1; g_fld[f2_fld_id].sz_fld_id = -1; status = chk_fld_name(f2, 0); zero_string(g_fld[f2_fld_id].name, MAX_LEN_FLD_NAME+1); zero_string(g_fld[f2_fld_id].filename, MAX_LEN_FILE_NAME+1); strcpy(g_fld[f2_fld_id].name, f2); strcpy(g_fld[f2_fld_id].filename, opfile); // Add to hash table //-------------------------------------------------------- // Add auxiliary field nn if if it exists nn_fld_id = g_fld[f1_fld_id].nn_fld_id; if ( nn_fld_id >= 0 ) { // open input file status = rs_mmap(g_fld[nn_fld_id].filename, &nn_X, &nn_nX, 0); cBYE(status); // create copy of data status = open_temp_file(&nn_ofp, &nn_opfile, nn_nX); cBYE(status); fclose_if_non_null(nn_ofp); status = mk_file(nn_opfile, nn_nX); cBYE(status); status = rs_mmap(nn_opfile, &Y, &nY, 1); memcpy(Y, nn_X, nn_nX); rs_munmap(Y, nY); rs_munmap(nn_X, nn_nX); nn_fld_id = INT_MIN; status = add_aux_fld(tbl, f2, nn_opfile, "nn", &nn_fld_id); cBYE(status); } //-------------------------------------------------------- // Add auxiliary field sz if if it exists sz_fld_id = g_fld[f1_fld_id].sz_fld_id; if ( sz_fld_id >= 0 ) { status = open_temp_file(&sz_ofp, &sz_opfile, sz_nX); cBYE(status); fclose_if_non_null(sz_ofp); status = mk_file(sz_opfile, sz_nX); cBYE(status); status = rs_mmap(sz_opfile, &Y, &nY, 1); status = rs_mmap(g_fld[sz_fld_id].filename, &sz_X, &sz_nX, 0); memcpy(Y, sz_X, sz_nX); rs_munmap(Y, nY); rs_munmap(sz_X, sz_nX); status = add_aux_fld(tbl, f2, sz_opfile, "sz", &sz_fld_id); cBYE(status); } //-------------------------------------------------------- BYE: rs_munmap(X, nX); rs_munmap(nn_X, nn_nX); rs_munmap(sz_X, sz_nX); rs_munmap(Y, nY); fclose_if_non_null(ofp); fclose_if_non_null(nn_ofp); fclose_if_non_null(sz_ofp); free_if_non_null(opfile); free_if_non_null(nn_opfile); free_if_non_null(sz_opfile); return(status); }
off_t oggedit_write_vorbis_metadata(DB_FILE *in, const char *fname, const off_t offset, const size_t stream_size, const int num_tags, char **tags) { off_t res; char tempname[PATH_MAX] = ""; char *vendor = NULL; ogg_packet codebooks; memset(&codebooks, '\0', sizeof(codebooks)); ogg_sync_state oy; ogg_sync_init(&oy); /* Original file must be writable whichever way we update it */ FILE *out = fopen(fname, "r+b"); if (!out) { res = OGGEDIT_CANNOT_UPDATE_FILE; goto cleanup; } /* See if we can write the tags packet directly into the existing file ... */ const ptrdiff_t tags_packet_size = check_vorbis_headers(in, &oy, offset, &vendor, &codebooks); if (tags_packet_size <= OGGEDIT_EOF) { res = tags_packet_size; goto cleanup; } const size_t metadata_size = strlen(VCMAGIC) + vc_size(vendor, num_tags, tags) + 1; ptrdiff_t padding = tags_packet_size - metadata_size; const off_t file_size_k = in->vfs->getlength(in) / 1000; const size_t stream_size_k = stream_size ? stream_size / 1000 : file_size_k; if (file_size_k < 100 || padding < 0 || padding > file_size_k/10+stream_size_k+metadata_size) { res = open_temp_file(fname, tempname, &out); if (res) { goto cleanup; } } /* Re-pad if writing the whole file */ if (*tempname) padding = stream_size_k < 90 ? 0 : stream_size_k < 1000 ? 128 : stream_size_k < 10000 ? 1024 : 8192; /* Write pages until the correct comment header */ ogg_page og; int64_t vorbis_serial = copy_up_to_codec(in, out, &oy, &og, *tempname ? 0 : offset, offset, VORBISNAME); if (vorbis_serial <= OGGEDIT_EOF) { res = vorbis_serial; goto cleanup; } vorbis_serial = copy_up_to_header(in, out, &oy, &og, vorbis_serial); if (vorbis_serial <= OGGEDIT_EOF) { res = vorbis_serial; goto cleanup; } const long pageno = write_vorbis_tags(out, vorbis_serial, vendor, num_tags, tags, padding, &codebooks); if (pageno < OGGEDIT_EOF) { res = pageno; goto cleanup; } /* If we have tempfile, copy the remaining pages */ if (*tempname) { vorbis_serial = copy_remaining_pages(in, out, &oy, vorbis_serial, pageno); if (vorbis_serial <= OGGEDIT_EOF) { res = vorbis_serial; goto cleanup; } if (rename(tempname, fname)) { res = OGGEDIT_RENAME_FAILED; goto cleanup; } } res = file_size(fname); cleanup: ogg_packet_clear(&codebooks); cleanup(in, out, &oy, vendor); if (res < OGGEDIT_OK) unlink(tempname); return res; }
//--------------------------------------------------------------- // START FUNC DECL int join( char *docroot, sqlite3 *in_db, char *src_tbl, char *src_lnk, char *src_val, char *dst_tbl, char *dst_lnk, char *dst_fld, char *op ) // STOP FUNC DECL { int status = 0; sqlite3 *db = NULL; char *src_val_X = NULL; size_t src_val_nX = 0; char *src_lnk_X = NULL; size_t src_lnk_nX = 0; char *src_idx_X = NULL; size_t src_idx_nX = 0; char *nn_dst_val_X = NULL; size_t nn_dst_val_nX = 0; char *dst_val_X = NULL; size_t dst_val_nX = 0; char *dst_lnk_X = NULL; size_t dst_lnk_nX = 0; char *dst_idx_X = NULL; size_t dst_idx_nX = 0; FLD_META_TYPE src_val_meta, src_lnk_meta, src_idx_meta; FLD_META_TYPE dst_val_meta, dst_lnk_meta, dst_idx_meta; long long src_nR, dst_nR; // counts all rows char str_meta_data[1024]; int sizeof_src_idx, sizeof_dst_idx; int *iptr; int ijoin_op; char *x_src_tbl = NULL, *x_dst_tbl = NULL; FILE *ofp = NULL, *nn_ofp = NULL; char *opfile = NULL, *nn_opfile = NULL; //---------------------------------------------------------------- zero_string(str_meta_data, 1024); status = mk_mjoin_op(op, &ijoin_op); cBYE(status); status = open_db_if_needed(docroot, in_db, &db); cBYE(status); zero_fld_meta(&src_val_meta); zero_fld_meta(&src_lnk_meta); zero_fld_meta(&src_idx_meta); zero_fld_meta(&dst_val_meta); zero_fld_meta(&dst_lnk_meta); zero_fld_meta(&dst_idx_meta); //---------------------------------------------------------------- /* Given the src_tbl, we create x_src_tbl which has no null values in the link fields and which has an index field called "id" with values 0, 1, .... |x_src_tbl| -1 */ status = elim_null_vals(docroot, db, src_tbl, src_lnk, &x_src_tbl); cBYE(status); /* Similarly for dst_tbl */ status = elim_null_vals(docroot, db, dst_tbl, dst_lnk, &x_dst_tbl); cBYE(status); status = sort_if_necessary(docroot, db, x_src_tbl, src_lnk, "id"); cBYE(status); status = sort_if_necessary(docroot, db, x_dst_tbl, dst_lnk, "id"); cBYE(status); // Get meta-data for all necessary fields status = fld_meta(docroot, db, x_src_tbl, src_lnk, -1, &src_lnk_meta); cBYE(status); status = fld_meta(docroot, db, x_src_tbl, XXXXXXX, -1, &src_idx_meta); cBYE(status); status = fld_meta(docroot, db, x_src_tbl, src_val, -1, &src_val_meta); cBYE(status); status = fld_meta(docroot, db, x_dst_tbl, dst_lnk, -1, &dst_lnk_meta); cBYE(status); status = fld_meta(docroot, db, x_dst_tbl, XXXXXXX, -1, &dst_idx_meta); cBYE(status); // Get pointer access to all necessary fields status = rs_mmap(src_lnk_meta.filename, &src_lnk_X, &src_lnk_nX, 0); cBYE(status); status = rs_mmap(src_idx_meta.filename, &src_idx_X, &src_idx_nX, 0); cBYE(status); status = rs_mmap(src_val_meta.filename, &src_val_X, &src_val_nX, 0); cBYE(status); status = rs_mmap(dst_lnk_meta.filename, &dst_lnk_X, &dst_lnk_nX, 0); cBYE(status); status = rs_mmap(dst_idx_meta.filename, &dst_idx_X, &dst_idx_nX, 0); cBYE(status); //-------------------------------------------------------- // Create output data files status = open_temp_file(&ofp, &opfile); cBYE(status); fclose_if_non_null(ofp); status = open_temp_file(&nn_ofp, &nn_opfile); cBYE(status); fclose_if_non_null(nn_ofp); status = mk_file(opfile, src_val_meta.n_sizeof * dst_nR); cBYE(status); status = mk_file(nn_opfile, sizeof(char) * dst_nR); cBYE(status); status = rs_mmap(opfile, &dst_val_X, &dst_val_nX, 1); cBYE(status); status = rs_mmap(nn_opfile, &nn_dst_val_X, &nn_dst_val_nX, 1); cBYE(status); //-------------------------------------------------------- // Core join status = core_join_I_I_I_I_I_I( (int *)cpy_src_lnk_X, (int *)cpy_src_idx_X, (int *)src_val_X, src_nR, nn_src_nR, (int *)cpy_dst_lnk_X, (int *)cpy_dst_idx_X, (int *)dst_val_X, nn_dst_val_X, dst_nR, nn_dst_nR, ijoin_op, &is_any_null); //-------------------------------------------------------- // Clean up aux data structures rs_munmap(cpy_src_lnk_X, cpy_src_lnk_nX); rs_munmap(cpy_src_idx_X, cpy_src_idx_nX); rs_munmap(cpy_dst_lnk_X, cpy_dst_lnk_nX); rs_munmap(cpy_dst_idx_X, cpy_dst_idx_nX); if ( cpy_src_lnk_file == NULL ) { status = del_fld(docroot, db, src_tbl,"_cpy_lnk"); cBYE(status); } else { unlink(cpy_src_lnk_file); free_if_non_null(cpy_src_lnk_file); } if ( cpy_src_idx_file == NULL ) { status = del_fld(docroot, db, src_tbl,"_cpy_idx"); cBYE(status); } else { unlink(cpy_src_idx_file); free_if_non_null(cpy_src_idx_file); } if ( cpy_dst_lnk_file == NULL ) { status = del_fld(docroot, db, dst_tbl,"_cpy_lnk"); cBYE(status); } else { unlink(cpy_dst_lnk_file); free_if_non_null(cpy_dst_lnk_file); } if ( cpy_dst_idx_file == NULL ) { status = del_fld(docroot, db, dst_tbl,"_cpy_idx"); cBYE(status); } else { unlink(cpy_dst_idx_file); free_if_non_null(cpy_dst_idx_file); } // Add output field to meta data sprintf(str_meta_data,"filename=%s:n_sizeof=%d:fldtype=%s", opfile, src_val_meta.n_sizeof, src_val_meta.fldtype); status = add_fld(docroot, db, dst_tbl, dst_fld, str_meta_data); cBYE(status); if ( is_any_null == true ) { status = add_aux_fld(docroot, db, dst_tbl, dst_fld, nn_opfile, "nn"); cBYE(status); } else { unlink(nn_opfile); free_if_non_null(nn_opfile); } BYE: if ( in_db == NULL ) { sqlite3_close(db); } rs_munmap(dst_val_X, dst_nX); rs_munmap(nn_dst_val_X, nn_dst_nX); rs_munmap(src_val_X, src_val_nX); rs_munmap(src_lnk_X, src_lnk_nX); rs_munmap(src_lnk_X, src_lnk_nX); rs_munmap(dst_lnk_X, dst_lnk_nX); rs_munmap(dst_idx_X, dst_idx_nX); free_if_non_null(nn_opfile); free_if_non_null(opfile); return(status); }
//--------------------------------------------------------------- // START FUNC DECL int num_in_range( char *t1, char *f1, char *t2, char *lb, char *ub, char *cnt ) // STOP FUNC DECL { int status = 0; char *f1_X = NULL; size_t f1_nX = 0; char *lb_X = NULL; size_t lb_nX = 0; char *ub_X = NULL; size_t ub_nX = 0; char *cnt_X = NULL; size_t cnt_nX = 0; int t1_id = INT_MIN, t2_id = INT_MIN; int f1_id = INT_MIN, lb_id = INT_MIN, ub_id = INT_MIN, cnt_id = INT_MIN; FLD_TYPE *f1_meta = NULL, *lb_meta = NULL, *ub_meta = NULL; long long nR1 = INT_MIN, nR2 = INT_MIN, chk_nR1 = INT_MIN; long long **cntptrs = NULL; // For multi-threading int nT; int rc; // result code for thread create pthread_t threads[MAX_NUM_THREADS]; pthread_attr_t attr; void *thread_status; char str_meta_data[1024]; char *opfile = NULL; FILE *ofp = NULL; //---------------------------------------------------------------- if ( ( t1 == NULL ) || ( *t1 == '\0' ) ) { go_BYE(-1); } if ( ( f1 == NULL ) || ( *f1 == '\0' ) ) { go_BYE(-1); } if ( ( t2 == NULL ) || ( *t2 == '\0' ) ) { go_BYE(-1); } if ( ( lb == NULL ) || ( *lb == '\0' ) ) { go_BYE(-1); } if ( ( ub == NULL ) || ( *ub == '\0' ) ) { go_BYE(-1); } if ( ( cnt == NULL ) || ( *cnt == '\0' ) ) { go_BYE(-1); } if ( strcmp(t1, t2) == 0 ) { go_BYE(-1); } zero_string(str_meta_data, 1024); //-------------------------------------------------------- status = is_tbl(t1, &t1_id); cBYE(status); chk_range(t1_id, 0, g_n_tbl); nR1 = g_tbl[t1_id].nR; //-------------------------------------------------------- status = is_fld(NULL, t1_id, f1, &f1_id); cBYE(status); chk_range(f1_id, 0, g_n_fld); f1_meta = &(g_fld[f1_id]); status = rs_mmap(f1_meta->filename, &f1_X, &f1_nX, 0); cBYE(status); // Have not implemented case where f1 has null field if ( f1_meta->nn_fld_id >= 0 ) { go_BYE(-1); } // Have implemented only for int if ( strcmp(f1_meta->fldtype, "int") != 0 ) { cBYE(-1); } //-------------------------------------------------------- status = is_tbl(t2, &t2_id); cBYE(status); chk_range(t2_id, 0, g_n_tbl); nR2 = g_tbl[t2_id].nR; //-------------------------------------------------------- status = is_fld(NULL, t2_id, lb, &lb_id); cBYE(status); chk_range(lb_id, 0, g_n_fld); lb_meta = &(g_fld[lb_id]); status = rs_mmap(lb_meta->filename, &lb_X, &lb_nX, 0); cBYE(status); // Have not implemented case where lb has null field if ( lb_meta->nn_fld_id >= 0 ) { go_BYE(-1); } // Have implemented only for int if ( strcmp(lb_meta->fldtype, "int") != 0 ) { cBYE(-1); } //-------------------------------------------------------- status = is_fld(NULL, t2_id, ub, &ub_id); cBYE(status); chk_range(ub_id, 0, g_n_fld); ub_meta = &(g_fld[ub_id]); status = rs_mmap(ub_meta->filename, &ub_X, &ub_nX, 0); cBYE(status); // Have not implemented case where ub has null field if ( ub_meta->nn_fld_id >= 0 ) { go_BYE(-1); } // Have implemented only for int if ( strcmp(ub_meta->fldtype, "int") != 0 ) { cBYE(-1); } //-------------------------------------------------------- // Set up access to input int *inptr = (int *)f1_X; int *lbptr = (int *)lb_X; int *ubptr = (int *)ub_X; //-------------------------------------------------------- //--- Decide on how much parallelism to use for ( int i = 0; i < MAX_NUM_THREADS; i++ ) { g_thread_id[i] = i; g_num_rows[i] = 0; } status = get_num_threads(&nT); cBYE(status); //-------------------------------------------- #define MIN_ROWS_FOR_SUBSAMPLE 10000 // 1048576 if ( nR1 <= MIN_ROWS_FOR_SUBSAMPLE ) { nT = 1; } /* Don't create more threads than you can use */ if ( nT > nR1 ) { nT = nR1; } //-------------------------------------------- /* Make space for output */ long long filesz = nR2 * sizeof(long long); status = open_temp_file(&ofp, &opfile, filesz); cBYE(status); fclose_if_non_null(ofp); status = mk_file(opfile, filesz); cBYE(status); status = rs_mmap(opfile, &cnt_X, &cnt_nX, 1); long long *cntptr = (long long *)cnt_X; /* Make a holding tank for partial results */ cntptrs = malloc(nT * sizeof(long long *)); return_if_malloc_failed(cntptrs); for ( int i = 0; i < nT; i++ ) { cntptrs[i] = malloc(nR2 * sizeof(long long)); return_if_malloc_failed(cntptrs[i]); for ( long long j = 0; j <nR2; j++ ) { cntptrs[i][j] = 0; } } // Add count field to meta data sprintf(str_meta_data, "fldtype=long long:n_sizeof=8:filename=%s", opfile); status = add_fld(t2, cnt, str_meta_data, &cnt_id); cBYE(status); chk_range(cnt_id, 0, g_n_fld); //----------------------------------------------------------- // Now we count how much there is in each range // Set up global variables g_nT = nT; g_inptr = inptr; g_lbptr = lbptr; g_ubptr = ubptr; g_cntptrs = cntptrs; g_nR1 = nR1; g_nR2 = nR2; if ( g_nT == 1 ) { core_num_in_range(&(g_thread_id[0])); chk_nR1 = g_num_rows[0]; } else { chk_nR1 = 0; pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); for ( int t = 0; t < g_nT; t++ ) { rc = pthread_create(&threads[t], NULL, core_num_in_range, &(g_thread_id[t])); if ( rc ) { go_BYE(-1); } } /* Free attribute and wait for the other threads */ pthread_attr_destroy(&attr); for ( int t = 0; t < g_nT; t++ ) { rc = pthread_join(threads[t], &thread_status); if ( rc ) { go_BYE(-1); } chk_nR1 += g_num_rows[t]; } } if ( chk_nR1 != nR1 ) { go_BYE(-1); } // Accumulate partial results for ( long long i = 0; i < nR2; i++ ) { cntptr[i] = 0; for ( int j= 0; j < nT; j++ ) { cntptr[i] += cntptrs[j][i]; } } BYE: g_write_to_temp_dir = false; rs_munmap(f1_X, f1_nX); rs_munmap(lb_X, lb_nX); rs_munmap(ub_X, ub_nX); rs_munmap(cnt_X, cnt_nX); free_if_non_null(opfile); return(status); }
/* Example multipart form-data request content format: --AaB03x Content-Disposition: form-data; name="submit-name" Larry --AaB03x Content-Disposition: form-data; name="file"; filename="file1.txt" Content-Type: text/plain ... contents of file1.txt ... --AaB03x-- */ static evhtp_res upload_read_cb (evhtp_request_t *req, evbuf_t *buf, void *arg) { RecvFSM *fsm = arg; char *line; size_t len; gboolean no_line = FALSE; int res = EVHTP_RES_OK; if (fsm->state == RECV_ERROR) return EVHTP_RES_OK; /* Update upload progress. */ fsm->progress->uploaded += (gint64)evbuffer_get_length(buf); seaf_debug ("progress: %lld/%lld\n", fsm->progress->uploaded, fsm->progress->size); evbuffer_add_buffer (fsm->line, buf); /* Drain the buffer so that evhtp don't copy it to another buffer * after this callback returns. */ evbuffer_drain (buf, evbuffer_get_length (buf)); while (!no_line) { switch (fsm->state) { case RECV_INIT: line = evbuffer_readln (fsm->line, &len, EVBUFFER_EOL_CRLF_STRICT); if (line != NULL) { seaf_debug ("[upload] boundary line: %s.\n", line); if (!strstr (line, fsm->boundary)) { seaf_warning ("[upload] no boundary found in the first line.\n"); free (line); res = EVHTP_RES_BADREQ; goto out; } else { fsm->state = RECV_HEADERS; free (line); } } else { no_line = TRUE; } break; case RECV_HEADERS: line = evbuffer_readln (fsm->line, &len, EVBUFFER_EOL_CRLF_STRICT); if (line != NULL) { seaf_debug ("[upload] mime header line: %s.\n", line); if (len == 0) { /* Read an blank line, headers end. */ free (line); if (g_strcmp0 (fsm->input_name, "file") == 0) { if (open_temp_file (fsm) < 0) { seaf_warning ("[upload] Failed open temp file.\n"); res = EVHTP_RES_SERVERR; goto out; } } seaf_debug ("[upload] Start to recv %s.\n", fsm->input_name); fsm->state = RECV_CONTENT; } else if (parse_mime_header (line, fsm) < 0) { free (line); res = EVHTP_RES_BADREQ; goto out; } else { free (line); } } else { no_line = TRUE; } break; case RECV_CONTENT: if (g_strcmp0 (fsm->input_name, "file") == 0) res = recv_file_data (fsm, &no_line); else res = recv_form_field (fsm, &no_line); if (res != EVHTP_RES_OK) goto out; break; } } out: if (res != EVHTP_RES_OK) { /* Don't receive any data before the connection is closed. */ evhtp_request_pause (req); /* Set keepalive to 0. This will cause evhtp to close the * connection after sending the reply. */ req->keepalive = 0; fsm->state = RECV_ERROR; } if (res == EVHTP_RES_BADREQ) { evhtp_send_reply (req, EVHTP_RES_BADREQ); } else if (res == EVHTP_RES_SERVERR) { evbuffer_add_printf (req->buffer_out, "Internal server error\n"); evhtp_send_reply (req, EVHTP_RES_SERVERR); } return EVHTP_RES_OK; }
//--------------------------------------------------------------- // START FUNC DECL int f1f2f3f4opf5( char *tbl, char *f1, char *f2, char *f3, char *f4, char *str_op_spec, char *f5 ) // STOP FUNC DECL { int status = 0; char *f1_X = NULL; size_t f1_nX = 0; char *nn_f1_X = NULL; size_t nn_f1_nX = 0; char *f2_X = NULL; size_t f2_nX = 0; char *nn_f2_X = NULL; size_t nn_f2_nX = 0; char *f3_X = NULL; size_t f3_nX = 0; char *nn_f3_X = NULL; size_t nn_f3_nX = 0; char *f4_X = NULL; size_t f4_nX = 0; char *nn_f4_X = NULL; size_t nn_f4_nX = 0; char *out_X = NULL; size_t out_nX = 0; char *nn_out_X = NULL; size_t nn_out_nX = 0; char *opfile = NULL, *nn_opfile = NULL; FILE *ofp = NULL; FLD_TYPE *f1_meta = NULL, *nn_f1_meta = NULL; FLD_TYPE *f2_meta = NULL, *nn_f2_meta = NULL; FLD_TYPE *f3_meta = NULL, *nn_f3_meta = NULL; FLD_TYPE *f4_meta = NULL, *nn_f4_meta = NULL; long long nR; int tbl_id = INT_MIN; int f1_id = INT_MIN, f2_id = INT_MIN, f3_id = INT_MIN, f4_id = INT_MIN, f5_id = INT_MIN; int nn_f1_id = INT_MIN, nn_f2_id = INT_MIN, nn_f3_id = INT_MIN, nn_f4_id = INT_MIN, nn_f5_id = INT_MIN; char str_meta_data[1024]; bool is_any_null = false; float *f1ptr = NULL, *f2ptr = NULL, *f3ptr = NULL, *f4ptr = NULL; float *foutptr = NULL; char *nn_out = NULL; char *op = NULL; //---------------------------------------------------------------- if ( ( tbl == NULL ) || ( *tbl == '\0' ) ) { go_BYE(-1); } if ( ( f1 == NULL ) || ( *f1 == '\0' ) ) { go_BYE(-1); } if ( ( f2 == NULL ) || ( *f2 == '\0' ) ) { go_BYE(-1); } if ( ( f3 == NULL ) || ( *f3 == '\0' ) ) { go_BYE(-1); } if ( ( f4 == NULL ) || ( *f4 == '\0' ) ) { go_BYE(-1); } if ( ( f5 == NULL ) || ( *f5 == '\0' ) ) { go_BYE(-1); } if ( ( str_op_spec == NULL ) || ( *str_op_spec == '\0' ) ) { go_BYE(-1); } zero_string(str_meta_data, 1024); //-------------------------------------------------------- status = extract_name_value(str_op_spec, "op=", ":", &op); cBYE(status); if ( op == NULL ) { go_BYE(-1); } //-------------------------------------------------------- status = is_tbl(tbl, &tbl_id); cBYE(status); chk_range(tbl_id, 0, g_n_tbl); nR = g_tbl[tbl_id].nR; //-------------------------------------------------------- /* Get information for field f1 */ status = is_fld(NULL, tbl_id, f1, &f1_id); f1_meta = &(g_fld[f1_id]); status = rs_mmap(f1_meta->filename, &f1_X, &f1_nX, 0); cBYE(status); nn_f1_id = f1_meta->nn_fld_id; if ( nn_f1_id >= 0 ) { nn_f1_meta = &(g_fld[nn_f1_id]); status = rs_mmap(nn_f1_meta->filename, &nn_f1_X, &nn_f1_nX, 0); cBYE(status); } //-------------------------------------------------------- /* Get information for field f2 */ status = is_fld(NULL, tbl_id, f2, &f2_id); f2_meta = &(g_fld[f2_id]); status = rs_mmap(f2_meta->filename, &f2_X, &f2_nX, 0); cBYE(status); nn_f2_id = f2_meta->nn_fld_id; if ( nn_f2_id >= 0 ) { nn_f2_meta = &(g_fld[nn_f2_id]); status = rs_mmap(nn_f2_meta->filename, &nn_f2_X, &nn_f2_nX, 0); cBYE(status); } //-------------------------------------------------------- /* Get information for field f3 */ status = is_fld(NULL, tbl_id, f3, &f3_id); f3_meta = &(g_fld[f3_id]); status = rs_mmap(f3_meta->filename, &f3_X, &f3_nX, 0); cBYE(status); nn_f3_id = f3_meta->nn_fld_id; if ( nn_f3_id >= 0 ) { nn_f3_meta = &(g_fld[nn_f3_id]); status = rs_mmap(nn_f3_meta->filename, &nn_f3_X, &nn_f3_nX, 0); cBYE(status); } //-------------------------------------------------------- /* Get information for field f4 */ status = is_fld(NULL, tbl_id, f4, &f4_id); f4_meta = &(g_fld[f4_id]); status = rs_mmap(f4_meta->filename, &f4_X, &f4_nX, 0); cBYE(status); nn_f4_id = f4_meta->nn_fld_id; if ( nn_f4_id >= 0 ) { nn_f4_meta = &(g_fld[nn_f4_id]); status = rs_mmap(nn_f4_meta->filename, &nn_f4_X, &nn_f4_nX, 0); cBYE(status); } //-------------------------------------------------------- if ( strcmp(f1_meta->fldtype, "float") != 0 ) { go_BYE(-1); } if ( strcmp(f2_meta->fldtype, "float") != 0 ) { go_BYE(-1); } if ( strcmp(f3_meta->fldtype, "float") != 0 ) { go_BYE(-1); } if ( strcmp(f4_meta->fldtype, "float") != 0 ) { go_BYE(-1); } //-------------------------------------------------------- f1ptr = (float *)f1_X; f2ptr = (float *)f2_X; f3ptr = (float *)f3_X; f4ptr = (float *)f4_X; //-------------------------------------------------------- // Create output file long long filesz = sizeof(float) * nR; status = open_temp_file(&ofp, &opfile, filesz); cBYE(status); fclose_if_non_null(ofp); status = mk_file(opfile, filesz); cBYE(status); status = rs_mmap(opfile, &out_X, &out_nX, 1); cBYE(status); foutptr = (float*)out_X; //-------------------------------------------------------- // Create nn field for output file long long nnfilesz = sizeof(char) * nR; status = open_temp_file(&ofp, &nn_opfile, nnfilesz); cBYE(status); fclose_if_non_null(ofp); status = mk_file(nn_opfile, filesz); cBYE(status); status = rs_mmap(nn_opfile, &nn_out_X, &nn_out_nX, 1); cBYE(status); nn_out = (char*)nn_out_X; //-------------------------------------------------------- if ( strcmp(op, "lat_long_dist") == 0 ) { for ( long long i = 0; i < nR; i++ ) { if ( ( nn_f1_X != NULL ) && ( nn_f1_X[i] == 0 ) ) { is_any_null = true; foutptr[i] = 0; nn_out[i] = 0; continue; } if ( ( nn_f2_X != NULL ) && ( nn_f2_X[i] == 0 ) ) { is_any_null = true; foutptr[i] = 0; nn_out[i] = 0; continue; } if ( ( nn_f3_X != NULL ) && ( nn_f3_X[i] == 0 ) ) { is_any_null = true; foutptr[i] = 0; nn_out[i] = 0; continue; } if ( ( nn_f4_X != NULL ) && ( nn_f4_X[i] == 0 ) ) { is_any_null = true; foutptr[i] = 0; nn_out[i] = 0; continue; } if ( ( f1ptr[i] < -90 ) || ( f1ptr[i] > 90 ) ) { go_BYE(-1); } if ( ( f2ptr[i] < -180 ) || ( f2ptr[i] > 180 ) ) { go_BYE(-1); } if ( ( f3ptr[i] < -90 ) || ( f3ptr[i] > 90 ) ) { go_BYE(-1); } if ( ( f4ptr[i] < -180 ) || ( f4ptr[i] > 180 ) ) { go_BYE(-1); } status = latlong_distance(f1ptr[i], f2ptr[i], f3ptr[i], f4ptr[i], &(foutptr[i])); cBYE(status); nn_out[i] = TRUE; } } sprintf(str_meta_data, "filename=%s:fldtype=float:n_sizeof=4", opfile); status = add_fld(tbl, f5, str_meta_data, &f5_id); cBYE(status); if ( is_any_null ) { status = add_aux_fld(tbl, f5, nn_opfile, "nn", &nn_f5_id); cBYE(status); } else { unlink(nn_opfile); free_if_non_null(nn_opfile); } BYE: rs_munmap(f1_X, f1_nX); rs_munmap(nn_f1_X, nn_f1_nX); rs_munmap(f2_X, f2_nX); rs_munmap(nn_f2_X, nn_f2_nX); rs_munmap(f3_X, f3_nX); rs_munmap(nn_f3_X, nn_f3_nX); rs_munmap(f4_X, f4_nX); rs_munmap(nn_f4_X, nn_f4_nX); rs_munmap(out_X, out_nX); rs_munmap(nn_out_X, nn_out_nX); free_if_non_null(opfile); free_if_non_null(nn_opfile); free_if_non_null(op); return(status); }
int collect_lnk_idx_vals_int( char *src_lnk_X, char *nn_src_lnk_X, char *nn_src_fld_X, int src_nR, char *dst_lnk_X, char *nn_dst_lnk_X, int dst_nR, INT_INT_TYPE **ptr_src_pair, int *ptr_nn_src_nR, INT_INT_TYPE **ptr_dst_pair, int *ptr_nn_dst_nR, char **ptr_temp_src_X, size_t *ptr_temp_src_nX, char **ptr_temp_dst_X, size_t *ptr_temp_dst_nX ) { int status = 0; int nn_src_nR, nn_dst_nR; INT_INT_TYPE *src_pair = NULL, *dst_pair = NULL; int rec_size = sizeof(int) + sizeof(int); char *temp_src_file = NULL, *temp_dst_file = NULL; FILE *tfp = NULL; int *ll_dst_lnk_ptr = NULL, *ll_src_lnk_ptr = NULL; char *temp_src_X = NULL; size_t temp_src_nX = 0; char *temp_dst_X = NULL; size_t temp_dst_nX = 0; /* START: Collect nn source values and sort them on link field */ status = open_temp_file(&tfp, &temp_src_file, 0); cBYE(status); nn_src_nR = 0; ll_src_lnk_ptr = (int*)src_lnk_X; for ( int i = 0; i < src_nR; i++ ) { if ( ( nn_src_fld_X != NULL ) && ( nn_src_fld_X[i] == FALSE ) ) { continue; } if ( ( nn_src_lnk_X != NULL ) && ( nn_src_lnk_X[i] == FALSE ) ) { continue; } nn_src_nR++; fwrite(ll_src_lnk_ptr+i, 1, sizeof(int), tfp); fwrite(&i, 1, sizeof(int), tfp); } fclose_if_non_null(tfp); if ( nn_src_nR == 0 ) { /* No values to process */ go_BYE(-1); } status = rs_mmap(temp_src_file, &temp_src_X, &temp_src_nX, 1); cBYE(status); if ( ( ( temp_src_nX/ sizeof(INT_INT_TYPE) ) * sizeof(INT_INT_TYPE) ) != temp_src_nX) { go_BYE(-1); } qsort(temp_src_X, nn_src_nR, rec_size, sort2_asc_int); src_pair = (INT_INT_TYPE *)temp_src_X; /* STOP: Collect nn source values and sort them on link field */ /*------------------------------------------------*/ /* START: Sort destination link fields */ status = open_temp_file(&tfp, &temp_dst_file, 0); cBYE(status); nn_dst_nR = 0; ll_dst_lnk_ptr = (int *)dst_lnk_X; for ( int i = 0; i < dst_nR; i++ ) { if ( ( nn_dst_lnk_X != NULL ) && ( nn_dst_lnk_X[i] == FALSE ) ) { continue; } nn_dst_nR++; fwrite(ll_dst_lnk_ptr+i, 1, sizeof(int), tfp); fwrite(&i, 1, sizeof(int), tfp); } fclose_if_non_null(tfp); if ( nn_dst_nR == 0 ) { /* No values to process */ go_BYE(-1); } status = rs_mmap(temp_dst_file, &temp_dst_X, &temp_dst_nX, 1); cBYE(status); if ( ( ( temp_dst_nX/ sizeof(INT_INT_TYPE) ) * sizeof(INT_INT_TYPE) ) != temp_dst_nX) { go_BYE(-1); } qsort(temp_dst_X, nn_dst_nR, rec_size, sort2_asc_int); dst_pair = (INT_INT_TYPE *)temp_dst_X; /*------------------------------------------------*/ /* STOP : Sort destination link fields */ //-------------------------------------------------------- *ptr_nn_src_nR = nn_src_nR; *ptr_nn_dst_nR = nn_dst_nR; *ptr_src_pair = src_pair; *ptr_dst_pair = dst_pair; *ptr_temp_src_X = temp_src_X; *ptr_temp_src_nX = temp_src_nX; *ptr_temp_dst_X = temp_dst_X; *ptr_temp_dst_nX = temp_dst_nX; BYE: unlink(temp_src_file); unlink(temp_dst_file); free_if_non_null(temp_src_file); free_if_non_null(temp_dst_file); return(status); }