int
main(
int argc,
char **argv
)
{
int status = 0;
char *buffer = NULL;
char filename[256];
size_t bufsize = BUFSIZE;
FILE *ifp = NULL, *ofp = NULL, *tfp = NULL;
char *infile = NULL, *outfile = NULL, *tempfile = NULL;
int nr, lno;
if ( argc != 3 ) {
fprintf(stderr, "Usage is txt_sub <infile> <outfile> \n");
go_BYE(-1);
}
infile = argv[1];
outfile = argv[2];
if ( *infile == '\0' ) { go_BYE(-1); }
if ( *outfile == '\0' ) { go_BYE(-1); }
if ( strcmp(infile, outfile)== 0 ) { go_BYE(-1); }
buffer = (char *)malloc(bufsize * sizeof(char));
return_if_malloc_failed(buffer);
zero_string(buffer, bufsize);
zero_string(filename, 256);
ifp = fopen(infile, "r");
return_if_fopen_failed(ifp, infile, "r");
ofp = fopen(outfile, "w");
return_if_fopen_failed(ofp, outfile, "w");
for ( lno = 1; ; lno++ ) {
zero_string_to_nullc(buffer);
nr = getline(&buffer, &bufsize, ifp);
if ( nr <= 0 ) { break; }
if ( ( strcasecmp(buffer, "%% INCLUDE") == 0 ) ||
( strcasecmp(buffer, "// INCLUDE") == 0 ) ) {
status = open_temp_file(&tfp, &tempfile); cBYE(status);
fclose_if_non_null(tfp);
fprintf(ofp, "\\verbatiminput{%s}\n", tempfile);
free_if_non_null(tempfile);
}
else { // copy from input to output
fprintf(ofp, "%s", buffer);
}
}
fclose_if_non_null(ifp);
fclose_if_non_null(ofp);
BYE:
free_if_non_null(buffer);
return(status);
}
off_t oggedit_write_opus_metadata(DB_FILE *in, const char *fname, const off_t offset, const off_t stream_size, const int output_gain, const uint32_t num_tags, char **tags)
{
off_t res;
char tempname[PATH_MAX] = "";
char *vendor = NULL;
ogg_sync_state oy;
ogg_sync_init(&oy);
/* Original file must be writable whichever way we update it */
FILE *out = fopen(fname, "r+b");
if (!out) {
res = OGGEDIT_CANNOT_UPDATE_FILE;
goto cleanup;
}
/* Should we write the tags packet directly into the existing file ... */
const long tags_packet_size = check_opus_header(in, &oy, offset, &vendor);
if (tags_packet_size <= OGGEDIT_EOF) {
res = tags_packet_size;
goto cleanup;
}
const int64_t metadata_size = strlen(TAGMAGIC) + vc_size(vendor, num_tags, tags);
int64_t padding = tags_packet_size - metadata_size;
const off_t file_size_k = in->vfs->getlength(in) / 1000;
const off_t stream_size_k = stream_size ? stream_size / 1000 : file_size_k;
if (file_size_k < 100 || padding < 0 || padding > file_size_k/10+stream_size_k+metadata_size) {
res = open_temp_file(fname, tempname, &out);
if (res) {
goto cleanup;
}
}
/* Re-pad if writing the whole file */
if (*tempname)
padding = stream_size_k < 90 ? 0 : stream_size_k < 1000 ? 128 : stream_size_k < 10000 ? 1024 : 8192;
/* Write pages until we reach the correct OpusHead, then write OpusTags */
ogg_page og;
int64_t opus_serial = copy_up_to_codec(in, out, &oy, &og, *tempname ? 0 : offset, offset, OPUSNAME);
if (opus_serial <= OGGEDIT_EOF) {
res = opus_serial;
goto cleanup;
}
if (output_gain > INT_MIN) {
og.body[16] = output_gain & 0xFF;
og.body[17] = output_gain >> 8 & 0xFF;
ogg_page_checksum_set(&og);
}
FileHolder Database_ResourceStorage::get_file_path(const std::string& key)
{
int ind = find_in_cache(key);
if(ind >= 0)
return FileCache[ind].file;
guint32 entry_offset, entry_size;
if(!ridx_file->lookup(key.c_str(), entry_offset, entry_size))
return FileHolder(); // key not found
gchar *data = dict->GetData(entry_offset, entry_size);
if(!data)
return FileHolder();
std::string name_pattern; // in file name encoding
if(!utf8_to_file_name(key, name_pattern))
return FileHolder();
std::string::size_type pos = name_pattern.find_last_of("."DB_DIR_SEPARATOR_S);
if(pos != std::string::npos) {
if(name_pattern[pos] == '.')
name_pattern = name_pattern.substr(pos);
else
name_pattern.clear();
} else
name_pattern.clear();
gint fd;
FileHolder file(open_temp_file(name_pattern, fd));
if(file.empty())
return file;
ssize_t write_size;
#ifdef _WIN32
write_size = _write(fd, data+sizeof(guint32), entry_size);
if (write_size == -1) {
g_print("write error!\n");
}
_close(fd);
#else
write_size = write(fd, data+sizeof(guint32), entry_size);
if (write_size == -1) {
g_print("write error!\n");
}
close(fd);
#endif
ind = put_in_cache(key, file);
return FileCache[ind].file;
}
void csync_rs_sig(const char *filename)
{
FILE *basis_file = 0, *sig_file = 0;
rs_stats_t stats;
rs_result result;
char tmpfname[MAXPATHLEN];
csync_debug(3, "Csync2 / Librsync: csync_rs_sig('%s')\n", filename);
csync_debug(3, "Opening basis_file and sig_file..\n");
sig_file = open_temp_file(tmpfname, prefixsubst(filename));
if ( !sig_file ) goto io_error;
if (unlink(tmpfname) < 0) goto io_error;
basis_file = fopen(prefixsubst(filename), "rb");
if ( !basis_file ) basis_file = fopen("/dev/null", "rb");
csync_debug(3, "Running rs_sig_file() from librsync..\n");
result = rs_sig_file(basis_file, sig_file,
RS_DEFAULT_BLOCK_LEN, RS_DEFAULT_STRONG_LEN, &stats);
if (result != RS_DONE)
csync_fatal("Got an error from librsync, too bad!\n");
csync_debug(3, "Sending sig_file to peer..\n");
csync_send_file(sig_file);
csync_debug(3, "Signature has been created successfully.\n");
fclose(basis_file);
fclose(sig_file);
return;
io_error:
csync_debug(0, "I/O Error '%s' in rsync-sig: %s\n",
strerror(errno), prefixsubst(filename));
if (basis_file) fclose(basis_file);
if (sig_file) fclose(sig_file);
}
// START FUNC DECL
int
mk_temp_file(
char *opfile,
size_t filesz
)
// STOP FUNC DECL
{
int status = 0;
char cwd[MAX_LEN_DIR_NAME+1];
if ( g_data_dir == NULL ) { go_BYE(-1); }
zero_string(cwd, MAX_LEN_DIR_NAME+1);
getcwd(cwd, MAX_LEN_DIR_NAME);
if ( strlen(cwd) == 0 ) { go_BYE(-1); }
status = open_temp_file(opfile, filesz); cBYE(status);
status = chdir(g_data_dir); cBYE(status);
status = mk_file(opfile, filesz); cBYE(status);
status = chdir(cwd); cBYE(status);
BYE:
return(status);
}
//---------------------------------------------------------------
// START FUNC DECL
int
parsort1(
char *tbl,
char *f1,
char *f2,
char *up_or_down /* not used right now */
)
// STOP FUNC DECL
{
int status = 0;
char *f1_X = NULL; size_t f1_nX = 0;
char *op_X = NULL; size_t op_nX = 0;
char *cnt_X = NULL; size_t cnt_nX = 0;
char *t2f2_X = NULL; size_t t2f2_nX = 0;
FLD_TYPE *f1_meta = NULL;
FLD_TYPE *f2_meta = NULL;
FLD_TYPE *t2f2_meta = NULL;
FLD_TYPE *cnt_meta = NULL;
long long nR, nR2;
int tbl_id = INT_MIN, f1_id = INT_MIN, f2_id = INT_MIN, cnt_id = INT_MIN;
int t2f2_id = INT_MIN;
char str_meta_data[1024];
FILE *ofp = NULL; char *opfile = NULL;
FILE *tfp = NULL; char *tempfile = NULL;
char str_rslt[32]; zero_string(str_rslt, 32);
char t2[MAX_LEN_TBL_NAME];
int itemp;
int *xxx = NULL, *f1lb = NULL, *f1ub = NULL;
long long *count = NULL, *chk_count = NULL;
int **offsets = NULL, **bak_offsets = NULL;
int *inptr = NULL;
// For multi-threading
int rc; // result code for thread create
pthread_t threads[MAX_NUM_THREADS];
pthread_attr_t attr;
void *thread_status;
// START: For timing
struct timeval Tps;
struct timezone Tpf;
void *Tzp = NULL;
long long t_before_sec = 0, t_before_usec = 0, t_before = 0;
long long t_after_sec, t_after_usec, t_after;
long long t_delta_usec;
// STOP : For timing
//----------------------------------------------------------------
if ( ( tbl == NULL ) || ( *tbl == '\0' ) ) { go_BYE(-1); }
if ( ( f1 == NULL ) || ( *f1 == '\0' ) ) { go_BYE(-1); }
if ( ( f2 == NULL ) || ( *f2 == '\0' ) ) { go_BYE(-1); }
zero_string(str_meta_data, 1024);
/* t2 isa temporary table */
zero_string(t2, MAX_LEN_TBL_NAME);
status = qd_uq_str(t2, MAX_LEN_TBL_NAME);
strcpy(t2, "t2"); // TODO DELETE THIS
g_offsets = NULL;
g_count = NULL;
//--------------------------------------------------------
status = is_tbl(tbl, &tbl_id); cBYE(status);
chk_range(tbl_id, 0, g_n_tbl);
nR = g_tbl[tbl_id].nR;
status = is_fld(NULL, tbl_id, f1, &f1_id); cBYE(status);
chk_range(f1_id, 0, g_n_fld);
f1_meta = &(g_fld[f1_id]);
status = rs_mmap(f1_meta->filename, &f1_X, &f1_nX, 0);
cBYE(status);
// Not implemented for following cases
if ( g_fld[f1_id].nn_fld_id >= 0 ) { go_BYE(-1); }
if ( strcmp(f1_meta->fldtype, "int") != 0 ) { go_BYE(-1); }
if ( nR <= 1048576 ) { go_BYE(-1); }
//---------------------------------------------
status = gettimeofday(&Tps, &Tpf); cBYE(status);
t_before_sec = (long long)Tps.tv_sec;
t_before_usec = (long long)Tps.tv_usec;
t_before = t_before_sec * 1000000 + t_before_usec;
int reduction_factor = (int)(sqrt((double)nR));
sprintf(str_rslt, "%d", reduction_factor);
status = subsample(tbl, f1, str_rslt, t2, "f2"); cBYE(status);
status = gettimeofday(&Tps, &Tpf); cBYE(status);
t_after_sec = (long long)Tps.tv_sec;
t_after_usec = (long long)Tps.tv_usec;
t_after = t_after_sec * 1000000 + t_after_usec;
fprintf(stderr, "TIME0 = %lld \n", t_after - t_before);
t_before = t_after;
// Must have sufficient diversity of values
status = f1opf2(t2, "f2", "op=shift:val=-1", "nextf2"); cBYE(status);
status = drop_nn_fld(t2, "nextf2"); cBYE(status);
status = f1f2opf3(t2, "f2", "nextf2", "==", "x"); cBYE(status);
status = f_to_s(t2, "x", "sum", str_rslt);
char *endptr;
long long lltemp = strtoll(str_rslt, &endptr, 10);
if ( lltemp != 0 ) { go_BYE(-1); }
//-------------------------------------------------
// Get range of values of f1
status = f_to_s(tbl, f1, "max", str_rslt);
int f1max = strtoll(str_rslt, &endptr, 10);
status = f_to_s(tbl, f1, "min", str_rslt);
int f1min = strtoll(str_rslt, &endptr, 10);
//-------------------------------------------------
// Now we sort the values that we sampled
status = fop(t2, "f2", "sortA"); cBYE(status);
// status = pr_fld(t2, "f2", "", stdout);
status = get_nR(t2, &nR2);
// Now each thread selects a range to work on
int nT;
for ( int i = 0; i < MAX_NUM_THREADS; i++ ) {
g_thread_id[i] = i;
}
status = get_num_threads(&nT);
cBYE(status);
//--------------------------------------------
#define MIN_ROWS_FOR_PARSORT1 1048576
if ( nR <= MIN_ROWS_FOR_PARSORT1 ) {
nT = 1;
}
/* Don't create more threads than you can use */
if ( nT > nR ) { nT = nR; }
//--------------------------------------------
double block_size = (double)nR2 / (double)nT;
status = is_fld(t2, -1, "f2", &t2f2_id); cBYE(status);
chk_range(t2f2_id, 0, g_n_fld);
t2f2_meta = &(g_fld[t2f2_id]);
status = rs_mmap(t2f2_meta->filename, &t2f2_X, &t2f2_nX, 0);
cBYE(status);
int *iptr = (int *)t2f2_X;
xxx = malloc(nT * sizeof(int)); return_if_malloc_failed(xxx);
f1lb = malloc(nT * sizeof(int)); return_if_malloc_failed(f1lb);
f1ub = malloc(nT * sizeof(int)); return_if_malloc_failed(f1ub);
/* FOR OLD_WAY
count = malloc(nT * sizeof(long long)); return_if_malloc_failed(count);
*/
chk_count = malloc(nT * sizeof(long long));
return_if_malloc_failed(chk_count);
g_count = malloc(nT * sizeof(long long)); return_if_malloc_failed(g_count);
for ( int i = 0; i < nT; i++ ) {
// FOR OLD_WAY count[i]= 0;
chk_count[i]= 0;
int j = i+1;
long long idx = j * block_size;
if ( idx >= nR2 ) { idx = nR2 -1 ; }
int y = iptr[idx];
xxx[i] = y;
// fprintf(stdout,"idx = %lld: j = %d: y = %d \n", idx, j, y);
}
for ( int i = 0; i < nT; i++ ) {
if ( ( i == 0 ) && ( i == (nT - 1 ) ) ) {
f1lb[i] = f1min;
f1ub[i] = f1max;
}
else if ( i == 0 ) {
f1lb[i] = f1min;
f1ub[i] = xxx[i];
}
else if ( i == (nT -1 ) ) {
f1lb[i] = xxx[i-1] + 1;
f1ub[i] = f1max;
}
else {
f1lb[i] = xxx[i-1] + 1;
f1ub[i] = xxx[i];
}
}
// STOP: Each thread has now a range to work on
// Create a temporary table t3 to store ranges
char t3[MAX_LEN_TBL_NAME]; int t3_id;
zero_string(t3, MAX_LEN_TBL_NAME);
status = qd_uq_str(t3, MAX_LEN_TBL_NAME);
strcpy(t3, "t3"); // TODO DELETE THIS
sprintf(str_rslt, "%d", nT);
status = add_tbl(t3, str_rslt, &t3_id);
// Add lower bound to t3
status = open_temp_file(&tfp, &tempfile, -1); cBYE(status);
fclose_if_non_null(tfp);
tfp = fopen(tempfile, "wb"); return_if_fopen_failed(tfp, tempfile, "wb");
fwrite(f1lb, sizeof(int), nT, tfp);
fclose_if_non_null(tfp);
sprintf(str_meta_data, "fldtype=%s:n_sizeof=%d:filename=%s",
f1_meta->fldtype, f1_meta->n_sizeof, tempfile);
status = add_fld(t3, "lb", str_meta_data, &itemp); cBYE(status);
free_if_non_null(tempfile);
// Add upper bound to t3
status = open_temp_file(&tfp, &tempfile, -1); cBYE(status);
fclose_if_non_null(tfp);
tfp = fopen(tempfile, "wb"); return_if_fopen_failed(tfp, tempfile, "wb");
fwrite(f1ub, sizeof(int), nT, tfp);
fclose_if_non_null(tfp);
sprintf(str_meta_data, "fldtype=%s:n_sizeof=%d:filename=%s",
f1_meta->fldtype, f1_meta->n_sizeof, tempfile);
status = add_fld(t3, "ub", str_meta_data, &itemp); cBYE(status);
free_if_non_null(tempfile);
#undef OLD_WAY
#ifdef OLD_WAY
// Now we count how much there is in each range
inptr = (int *)f1_X;
for ( long long i = 0; i < nR; i++ ) {
int ival = *inptr++;
int range_idx = INT_MIN;
// TODO: Improve sequential search
for ( int j = 0; j < nT; j++ ) {
if ( ival >= f1lb[j] && ( ival <= f1ub[j] ) ) {
range_idx = j;
break;
}
}
count[range_idx]++;
}
/*
for ( int i = 0; i < nT; i++ ) {
fprintf(stdout,"%d: (%d, %d) = %lld \n", i, f1lb[i], f1ub[i], count[i]);
}
*/
#else
status = num_in_range(tbl, f1, t3, "lb", "ub", "cnt"); cBYE(status);
// Get a pointer to the count field
status = is_tbl(t3, &t3_id);
chk_range(t3_id, 0, g_n_tbl);
status = is_fld(NULL, t3_id, "cnt", &cnt_id);
chk_range(cnt_id, 0, g_n_fld);
cnt_meta = &(g_fld[cnt_id]);
status = rs_mmap(cnt_meta->filename, &cnt_X, &cnt_nX, 0); cBYE(status);
count = (long long *)cnt_X;
#endif
status = gettimeofday(&Tps, &Tpf); cBYE(status);
t_after_sec = (long long)Tps.tv_sec;
t_after_usec = (long long)Tps.tv_usec;
t_after = t_after_sec * 1000000 + t_after_usec;
fprintf(stderr, "TIME1 = %lld \n", t_after - t_before);
t_before = t_after;
bak_offsets = malloc(nT * sizeof(int *)); return_if_malloc_failed(bak_offsets);
g_offsets = malloc(nT * sizeof(int *)); return_if_malloc_failed(g_offsets);
#ifdef OLD_WAY
// Make space for output
long long filesz = nR * f1_meta->n_sizeof;
status = open_temp_file(&ofp, &opfile, filesz); cBYE(status);
status = mk_file(opfile, filesz); cBYE(status);
status = rs_mmap(opfile, &op_X, &op_nX, 1); cBYE(status);
offsets = malloc(nT * sizeof(int *)); return_if_malloc_failed(offsets);
long long cum_count = 0;
for ( int i = 0; i < nT; i++ ) {
bak_offsets[i] = offsets[i] = (int *)op_X;
if ( i > 0 ) {
cum_count += count[i-1];
offsets[i] += cum_count;
bak_offsets[i] = offsets[i];
}
}
inptr = (int *)f1_X;
// Now we place each item into its thread bucket
for ( long long i = 0; i < nR; i++ ) {
int ival = *inptr++;
int range_idx = INT_MIN;
// TODO: Improve sequential search
for ( int j = 0; j < nT; j++ ) {
if ( ival >= f1lb[j] && ( ival <= f1ub[j] ) ) {
range_idx = j;
break;
}
}
int *xptr = offsets[range_idx];
*xptr = ival;
offsets[range_idx]++;
chk_count[range_idx]++;
if ( chk_count[range_idx] > count[range_idx] ) {
go_BYE(-1);
}
}
cum_count = 0;
for ( int i = 0; i < nT-1; i++ ) {
if ( offsets[i] != bak_offsets[i+1] ) {
go_BYE(-1);
}
}
#else
status = mv_range(tbl, f1, f2, t3, "lb", "ub", "cnt");
cBYE(status);
status = is_fld(NULL, tbl_id, f2, &f2_id);
chk_range(f2_id, 0, g_n_fld);
f2_meta = &(g_fld[f2_id]);
status = rs_mmap(f2_meta->filename, &op_X, &op_nX, 1); cBYE(status);
#endif
long long cum_count = 0;
for ( int i = 0; i < nT; i++ ) {
bak_offsets[i] = (int *)op_X;
if ( i > 0 ) {
cum_count += count[i-1];
bak_offsets[i] += cum_count;
}
}
status = gettimeofday(&Tps, &Tpf); cBYE(status);
t_after_sec = (long long)Tps.tv_sec;
t_after_usec = (long long)Tps.tv_usec;
t_after = t_after_sec * 1000000 + t_after_usec;
fprintf(stderr, "TIME2 = %lld \n", t_after - t_before);
t_before = t_after;
// Set up global variables
g_nT = nT;
for ( int i = 0; i < nT; i++ ) {
g_offsets[i] = bak_offsets[i];
g_count[i] = count[i];
}
if ( g_nT == 1 ) {
core_parsort1(&(g_thread_id[0]));
}
else {
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
for ( int t = 0; t < g_nT; t++ ) {
rc = pthread_create(&threads[t], NULL, core_parsort1,
&(g_thread_id[t]));
if ( rc ) { go_BYE(-1); }
}
/* Free attribute and wait for the other threads */
pthread_attr_destroy(&attr);
for ( int t = 0; t < g_nT; t++ ) {
rc = pthread_join(threads[t], &thread_status);
if ( rc ) { go_BYE(-1); }
}
}
/* SEQUENTIAL CODE
for ( int i = 0; i < nT; i++ ) {
qsort_asc_int(bak_offsets[i], count[i], sizeof(int), NULL);
}
*/
status = gettimeofday(&Tps, &Tpf); cBYE(status);
t_after_sec = (long long)Tps.tv_sec;
t_after_usec = (long long)Tps.tv_usec;
t_after = t_after_sec * 1000000 + t_after_usec;
fprintf(stderr, "TIME3 = %lld \n", t_after - t_before);
// Indicate the dst_fld is sorted ascending
status = set_fld_info(tbl, f2, "sort=1");
rs_munmap(op_X, op_nX);
status = del_tbl(t2, -1); cBYE(status);
status = del_tbl(t3, -1); cBYE(status);
BYE:
rs_munmap(op_X, op_nX);
rs_munmap(cnt_X, cnt_nX);
free_if_non_null(xxx);
free_if_non_null(f1lb);
free_if_non_null(f1ub);
// Do not delete unless using OLD_WAY free_if_non_null(count);
free_if_non_null(g_count);
free_if_non_null(g_offsets);
free_if_non_null(offsets);
free_if_non_null(bak_offsets);
free_if_non_null(chk_count);
fclose_if_non_null(ofp);
g_write_to_temp_dir = false;
rs_munmap(f1_X, f1_nX);
rs_munmap(op_X, op_nX);
free_if_non_null(opfile);
return(status);
}
// START FUNC DECL
int
read_csv(
char *infile,
char fld_delim, /* double quote character */
char fld_sep, /* comma */
char rec_delim, /* new line character */
bool ignore_hdr, /* whether to ignore header or not */
char flds[MAX_NUM_FLDS][MAX_LEN_FLD_NAME+1],
int n_flds,
FLD_TYPE *fldtype,
bool *is_null_null, /* whether to convert "null" to null */
bool *is_load, /* to load or not */
bool *is_all_def, /* whether all values are defined */
char fnames[MAX_NUM_FLDS][MAX_LEN_FILE_NAME+1],
char nn_fnames[MAX_NUM_FLDS][MAX_LEN_FILE_NAME+1],
long long *ptr_num_rows
)
// STOP FUNC DECL
{
int status = 0;
char *X = NULL; size_t nX = 0;
bool in_fld, is_fld_delim;
long long num_rows = 0; long long num_cells = 0, fld_idx;
long long lb, ub;
FILE *fps[MAX_NUM_FLDS];
FILE *nn_fps[MAX_NUM_FLDS];
bool in_escape_mode = false, is_fld_delim_escaped = false;
g_num_rows = 0;
g_buflen = 8192;
g_buffer = (char *)malloc(g_buflen * sizeof(char));
return_if_malloc_failed(g_buffer);
zero_string(g_buffer, g_buflen);
/*----------------------------------------------------------------------------*/
if ( n_flds <= 0 ) { go_BYE(-1); }
for ( int i = 0; i < MAX_NUM_FLDS; i++ ) {
fps[i] = NULL;
nn_fps[i] = NULL;
}
/*----------------------------------------------------------------------------*/
status = rs_mmap(infile, &X, &nX, 0);
cBYE(status);
lb = 0; fld_idx = 0; in_fld = false; is_fld_delim = false;
for ( int i = 0; i < n_flds; i++ ) {
if ( is_load[i] ) {
status = open_temp_file((fnames[i]), 0);
cBYE(status);
fps[i] = fopen(fnames[i], "wb");
return_if_fopen_failed(fps[i], fnames[i], "wb");
if ( is_all_def[i] == false ) {
status = open_temp_file(nn_fnames[i], 0);
cBYE(status);
nn_fps[i] = fopen(nn_fnames[i], "wb");
return_if_fopen_failed(nn_fps[i], nn_fnames[i], "wb");
}
}
}
for ( long long i = 0; i < nX; i++ ) {
// START: Error handling for escape conventions
/* TODO: If I get an incorrect backslash, I simply replace it with a
* space. This is incorrect. But I need it to read files created
* with jspool utility from Jay Thomas. The correct thing to do is
* to fix the utility.
*/
if ( fld_sep != '\t' ) {
if ( in_escape_mode ) {
if ( ( X[i] != '\\' ) && ( X[i] != '"' ) ) {
fprintf(stderr, "Error. Bad backslash. \n");
fprintf(stderr, "num_rows = %lld \n", num_rows);
go_BYE(-1);
}
}
}
// STOP: Error handling for escape conventions
// START: Handling bslash and dquote
if ( fld_sep != '\t' ) {
if ( X[i] == '\\' ) {
if ( in_escape_mode ) {
in_escape_mode = false;
}
else {
in_escape_mode = true;
}
}
}
// STOP: Handling bslash and dquote
if ( fld_sep != '\t' ) {
if ( X[i] == '"' ) {
if ( in_escape_mode ) {
is_fld_delim_escaped = true;
in_escape_mode = false;
}
}
}
if ( in_fld == false ) {
in_fld = true;
if ( X[i] == fld_delim ) { /* consume this character */
is_fld_delim = true;
lb = i+1;
}
else {
lb = i;
if ( ( ( fld_idx < (n_flds-1) ) && ( X[i] == fld_sep ) ) ||
( ( fld_idx == n_flds-1 ) && ( X[i] == rec_delim ) ) ) {
/* field has terminated */
ub = i-1;
if ( is_load[fld_idx] ) {
if ( ( ignore_hdr ) & ( num_rows == 0 ) ) {
/* Do nothing */
}
else {
status = print_cell(X, lb, ub+1, fldtype[fld_idx],
is_null_null[fld_idx], is_all_def[fld_idx],
fps[fld_idx], nn_fps[fld_idx], fld_sep);
cBYE(status);
}
}
in_fld = false;
num_cells++;
if ( ( fld_idx == n_flds-1 ) && ( X[i] == rec_delim ) ) {
num_rows++;
g_num_rows = num_rows;
}
fld_idx++;
if ( fld_idx == n_flds ) { fld_idx = 0; }
}
}
}
else { /* Now within field. Decide whether to continue or stop. */
if ( is_fld_delim ) {
/* if field started with a fld_delim, it must end with one */
/* Note that we need to take care of escaping the dquote */
if ( ( X[i] == fld_delim ) && ( !is_fld_delim_escaped ) ) {
/* next char must be fld_sep or rec_delim or we have come to eof */
if ( i+1 > nX ) { go_BYE(-1); }
if ( ( fld_idx < (n_flds-1) ) && ( X[i+1] != fld_sep ) ) {
fprintf(stderr, "num_rows = %lld \n", num_rows);
go_BYE(-1);
}
if ( fld_idx == n_flds-1 ) {
if ( X[i+1] != rec_delim ) {
go_BYE(-1);
}
else {
num_rows++;
g_num_rows = num_rows;
}
}
ub = i-1;
if ( is_load[fld_idx] ) {
if ( ( ignore_hdr ) & ( num_rows == 0 ) ) {
/* Do nothing */
}
else {
status = print_cell(X, lb, ub+1, fldtype[fld_idx],
is_null_null[fld_idx], is_all_def[fld_idx],
fps[fld_idx], nn_fps[fld_idx], fld_sep);
cBYE(status);
}
}
in_fld = false;
is_fld_delim = false;
fld_idx++;
if ( fld_idx == n_flds ) { fld_idx = 0; }
num_cells++;
i++; /* Consume next char which must be fld_sep or rec_delim */
}
}
else {
if ( ( i == nX ) ||
( ( fld_idx < (n_flds-1) ) && ( X[i] == fld_sep ) ) ||
( ( fld_idx == n_flds-1 ) && ( X[i] == rec_delim ) ) ) {
/* field has terminated */
if ( i == nX ) {
ub = i;
}
else {
ub = i-1;
}
if ( is_load[fld_idx] ) {
if ( ( ignore_hdr ) & ( num_rows == 0 ) ) {
/* Do nothing */
}
else {
status = print_cell(X, lb, ub+1, fldtype[fld_idx],
is_null_null[fld_idx], is_all_def[fld_idx],
fps[fld_idx], nn_fps[fld_idx], fld_sep);
cBYE(status);
}
}
in_fld = false;
num_cells++;
if ( ( fld_idx == n_flds-1 ) && ( X[i] == rec_delim ) ) {
num_rows++;
g_num_rows = num_rows;
}
fld_idx++;
if ( fld_idx == n_flds ) { fld_idx = 0; }
}
}
}
is_fld_delim_escaped = false;
}
if ( num_cells != ( n_flds * num_rows ) ) {
fprintf(stderr, "num_cells = %lld \n", num_cells);
fprintf(stderr, "n_flds = %d \n", n_flds );
fprintf(stderr, "num_rows = %lld \n", num_rows);
go_BYE(-1);
}
if ( ignore_hdr ) {
num_rows--; // Because you have eliminated first row
}
/* TODO: Copy information into meta data structure */
for ( int i = 0; i < n_flds; i++ ) {
if ( fps != NULL ) { fclose_if_non_null(fps[i]); }
if ( nn_fps != NULL ) { fclose_if_non_null(nn_fps[i]); }
}
*ptr_num_rows = num_rows;
BYE:
rs_munmap(X, nX);
free_if_non_null(g_buffer);
return(status);
}
//---------------------------------------------------------------
// START FUNC DECL
int
crossprod(
char *docroot,
sqlite3 *in_db,
char *t1,
char *f1,
char *t2,
char *f2,
char *t3
)
// STOP FUNC DECL
{
int status = 0;
sqlite3 *db = NULL;
char *Y1 = NULL; size_t nY1 = 0;
char *Y2 = NULL; size_t nY2 = 0;
char *f1_X = NULL; size_t f1_nX = 0;
char *f2_X = NULL; size_t f2_nX = 0;
char *nn_f1_X = NULL; size_t nn_f1_nX = 0;
char *nn_f2_X = NULL; size_t nn_f2_nX = 0;
int t3_id, itemp; bool b_is_tbl;
FLD_META_TYPE f1_meta, f2_meta;
FLD_META_TYPE nn_f1_meta, nn_f2_meta;
long long nR1, nR2, nn_nR1, nn_nR2, nR3;
char str_meta_data[1024];
char *t3f1_opfile = NULL, *t3f2_opfile = NULL;
FILE *ofp = NULL;
char buffer[32];
//----------------------------------------------------------------
zero_string(str_meta_data, 1024);
zero_string(buffer, 32);
zero_fld_meta(&f1_meta);
zero_fld_meta(&f2_meta);
zero_fld_meta(&nn_f1_meta);
zero_fld_meta(&nn_f2_meta);
status = open_db_if_needed(docroot, in_db, &db);
cBYE(status);
//----------------------------------------------------------------
status = get_data(docroot, db, t1, f1, &nR1, &f1_meta, &f1_X,
&f1_nX, &nn_f1_meta, &nn_f1_X, &nn_f1_nX, &nn_nR1);
cBYE(status);
status = get_data(docroot, db, t2, f2, &nR2, &f2_meta, &f2_X,
&f2_nX, &nn_f2_meta, &nn_f2_X, &nn_f2_nX, &nn_nR2);
cBYE(status);
nR3 = nn_nR1 * nn_nR2;
if ( nR3 == 0 ) {
fprintf(stderr, "No data to create t3 \n");
goto BYE;
}
status = open_temp_file(&ofp, &t3f1_opfile); cBYE(status);
fclose_if_non_null(ofp);
status = mk_file(t3f1_opfile, nR3 * f1_meta.n_sizeof); cBYE(status);
status = rs_mmap(t3f1_opfile, &Y1, &nY1, 1); cBYE(status);
status = open_temp_file(&ofp, &t3f2_opfile); cBYE(status);
fclose_if_non_null(ofp);
status = mk_file(t3f2_opfile, nR3 * f2_meta.n_sizeof); cBYE(status);
status = rs_mmap(t3f2_opfile, &Y2, &nY2, 1); cBYE(status);
//----------------------------------------------------------------
if ( ( strcmp(f1_meta.fldtype, "int") == 0 ) &&
( strcmp(f2_meta.fldtype, "int") == 0 ) ) {
status = core_crossprod_I_I(f1_X, nn_f1_X, f2_X, nn_f2_X, nR1, nR2, nR3, Y1, Y2);
}
else if ( ( strcmp(f1_meta.fldtype, "int") == 0 ) &&
( strcmp(f2_meta.fldtype, "long long") == 0 ) ) {
status = core_crossprod_I_L(f1_X, nn_f1_X, f2_X, nn_f2_X, nR1, nR2, nR3, Y1, Y2);
}
else if ( ( strcmp(f1_meta.fldtype, "long long") == 0 ) &&
( strcmp(f2_meta.fldtype, "int") == 0 ) ) {
status = core_crossprod_L_I(f1_X, nn_f1_X, f2_X, nn_f2_X, nR1, nR2, nR3, Y1, Y2);
}
else if ( ( strcmp(f1_meta.fldtype, "long long") == 0 ) &&
( strcmp(f2_meta.fldtype, "long long") == 0 ) ) {
status = core_crossprod_L_L(f1_X, nn_f1_X, f2_X, nn_f2_X, nR1, nR2, nR3, Y1, Y2);
}
else { go_BYE(-1); }
cBYE(status);
//----------------------------------------------------------------
// Add output fields to t3 meta data
status = is_tbl(docroot, db, t3, &b_is_tbl, &t3_id);
cBYE(status);
if ( b_is_tbl ) {
status = del_tbl(docroot, db, t3);
cBYE(status);
}
sprintf(buffer, "%lld", nR3);
status = add_tbl(docroot, db, t3, buffer, &itemp);
cBYE(status);
sprintf(str_meta_data, "fldtype=int:n_sizeof=%u:filename=%s",
f1_meta.n_sizeof, t3f1_opfile);
status = add_fld(docroot, db, t3, f1, str_meta_data);
cBYE(status);
zero_string(str_meta_data, 1024);
sprintf(str_meta_data, "fldtype=int:n_sizeof=%u:filename=%s",
f2_meta.n_sizeof, t3f2_opfile);
status = add_fld(docroot, db, t3, f2, str_meta_data);
cBYE(status);
BYE:
fclose_if_non_null(ofp);
if ( in_db == NULL ) { sqlite3_close(db); }
rs_munmap(f1_X, f1_nX);
rs_munmap(nn_f1_X, nn_f1_nX);
rs_munmap(f2_X, f2_nX);
rs_munmap(nn_f2_X, nn_f2_nX);
rs_munmap(Y1, nY1);
rs_munmap(Y2, nY2);
free_if_non_null(t3f1_opfile);
free_if_non_null(t3f2_opfile);
return(status);
}
//---------------------------------------------------------------
// START FUNC DECL
int
count_vals(
char *src_tbl,
char *src_fld,
char *in_cnt_fld,
char *dst_tbl,
char *dst_fld,
char *out_cnt_fld
)
// STOP FUNC DECL
{
int status = 0;
char *src_fld_X = NULL; size_t src_fld_nX = 0;
char *in_cnt_fld_X = NULL; size_t in_cnt_fld_nX = 0;
FLD_TYPE *src_fld_meta = NULL, *in_cnt_fld_meta = NULL;
FLD_TYPE *cnt_fld_meta = NULL, *nn_src_fld_meta = NULL;
long long src_nR;
int src_tbl_id = INT_MIN, dst_tbl_id = INT_MIN;
int src_fld_id = INT_MIN, nn_src_fld_id = INT_MIN;
int dst_fld_id = INT_MIN;
int cnt_fld_id = INT_MIN, nn_cnt_fld_id = INT_MIN;
long long dst_nR, n_out;
char buffer[32];
char str_meta_data[256];
char *nn_src_fld_X = NULL; size_t nn_src_fld_nX = 0;
char *out_X = NULL; char *cnt_X = NULL;
char *opfile = NULL, *tmp_opfile = NULL; FILE *ofp = 0;
char *cnt_opfile = NULL, *tmp_cnt_opfile = NULL; FILE *cnt_ofp = 0;
size_t n_out_X = 0, n_cnt_X = 0;
unsigned int n_sizeof = INT_MAX;
//----------------------------------------------------------------
if ( ( src_tbl == NULL ) || ( *src_tbl == '\0' ) ) { go_BYE(-1); }
if ( ( src_fld == NULL ) || ( *src_fld == '\0' ) ) { go_BYE(-1); }
if ( ( dst_tbl == NULL ) || ( *dst_tbl == '\0' ) ) { go_BYE(-1); }
if ( ( dst_fld == NULL ) || ( *dst_fld == '\0' ) ) { go_BYE(-1); }
if ( strcmp(src_tbl, dst_tbl) == 0 ) { go_BYE(-1); }
zero_string(str_meta_data, 256);
zero_string(buffer, 32);
//--------------------------------------------------------
status = is_tbl(src_tbl, &src_tbl_id); cBYE(status);
chk_range(src_tbl_id, 0, g_n_tbl);
src_nR = g_tbl[src_tbl_id].nR;
//--------------------------------------------------------
status = is_fld(NULL, src_tbl_id, src_fld, &src_fld_id); cBYE(status);
chk_range(src_fld_id, 0, g_n_fld);
src_fld_meta = &(g_fld[src_fld_id]);
status = rs_mmap(src_fld_meta->filename, &src_fld_X, &src_fld_nX, 0); cBYE(status);
nn_src_fld_id = src_fld_meta->nn_fld_id;
if ( nn_src_fld_id >= 0 ) {
nn_src_fld_meta = &(g_fld[nn_src_fld_id]);
status = rs_mmap(nn_src_fld_meta->filename, &nn_src_fld_X, &nn_src_fld_nX, 0);
cBYE(status);
}
// Get the count field if specified. It cannot have null values
if ( *in_cnt_fld != '\0' ) {
status = is_fld(NULL, src_tbl_id, src_fld, &src_fld_id); cBYE(status);
chk_range(src_fld_id, 0, g_n_fld);
src_fld_meta = &(g_fld[src_fld_id]);
status = rs_mmap(src_fld_meta->filename, &src_fld_X, &src_fld_nX, 0); cBYE(status);
nn_cnt_fld_id = cnt_fld_meta->nn_fld_id;
chk_range(nn_cnt_fld_id, 0, g_n_fld);
}
/* Make sure src_fld field is sorted ascending */
zero_string(buffer, 32);
status = f_to_s(src_tbl, src_fld, "is_sorted", buffer);
cBYE(status);
if ( strcmp(buffer, "ascending") != 0 ) {
fprintf(stderr, "Field [%s] in Table [%s] not sorted ascending\n",
src_fld, src_tbl);
go_BYE(-1);
}
//--------------------------------------------------------
// Create 2 temporary files to store the results. We allocate space
// differently based on field types
g_write_to_temp_dir = true;
status = open_temp_file(&ofp, &tmp_opfile, 0); cBYE(status);
fclose_if_non_null(ofp);
status = open_temp_file(&cnt_ofp, &tmp_cnt_opfile, 0); cBYE(status);
fclose_if_non_null(cnt_ofp);
//------------------------------------------------------
n_out = src_nR * sizeof(int); // upper bound
if ( strcmp(src_fld_meta->fldtype, "int") == 0 ) {
status = mk_file(tmp_cnt_opfile, sizeof(int) * src_nR);
cBYE(status);
rs_mmap(tmp_cnt_opfile, &cnt_X, &n_cnt_X, 1);
cBYE(status);
status = mk_file(tmp_opfile, sizeof(int) * src_nR);
cBYE(status);
rs_mmap(tmp_opfile, &out_X, &n_out_X, 1);
cBYE(status);
if ( nn_src_fld_X == NULL ) {
if ( *in_cnt_fld == '\0' ) {
uniq_alldef_int((int *)src_fld_X, src_nR, (int *)out_X,
&dst_nR, (int *)cnt_X);
}
else {
uniq_cnt_alldef_i_i((int *)src_fld_X, (int *)in_cnt_fld_X,
src_nR, (int *)out_X, &dst_nR, (int *)cnt_X);
}
}
else {
uniq_int((int *)src_fld_X, src_nR, nn_src_fld_X,
(int *)out_X, &dst_nR, (int *)cnt_X);
}
}
else if ( strcmp(src_fld_meta->fldtype, "long long") == 0 ) {
status = mk_file(tmp_opfile, sizeof(long long) * src_nR);
cBYE(status);
rs_mmap(tmp_opfile, &out_X, &n_out_X, 1);
cBYE(status);
if ( nn_src_fld_X == NULL ) {
if ( *in_cnt_fld == '\0' ) {
status = mk_file(tmp_cnt_opfile, sizeof(long long) * src_nR);
cBYE(status);
rs_mmap(tmp_cnt_opfile, &cnt_X, &n_cnt_X, 1);
cBYE(status);
uniq_alldef_longlong((long long *)src_fld_X, src_nR,
(long long *)out_X, &dst_nR, (long long *)cnt_X);
}
else {
if ( strcmp(in_cnt_fld_meta->fldtype, "long long") == 0 ) {
status = mk_file(tmp_cnt_opfile, sizeof(long long) * src_nR);
cBYE(status);
rs_mmap(tmp_cnt_opfile, &cnt_X, &n_cnt_X, 1);
cBYE(status);
uniq_cnt_alldef_ll_ll((long long *)src_fld_X,
(long long *)in_cnt_fld_X, src_nR, (long long *)out_X, &dst_nR,
(long long *)cnt_X);
}
else if ( strcmp(in_cnt_fld_meta->fldtype, "double") == 0 ) {
status = mk_file(tmp_cnt_opfile, sizeof(double) * src_nR);
cBYE(status);
rs_mmap(tmp_cnt_opfile, &cnt_X, &n_cnt_X, 1);
cBYE(status);
uniq_cnt_alldef_ll_dbl((long long *)src_fld_X,
(double *)in_cnt_fld_X, src_nR, (long long *)out_X, &dst_nR,
(double *)cnt_X);
}
}
}
else {
if ( *in_cnt_fld == '\0' ) {
status = mk_file(tmp_cnt_opfile, sizeof(long long) * src_nR);
cBYE(status);
rs_mmap(tmp_cnt_opfile, &cnt_X, &n_cnt_X, 1);
cBYE(status);
uniq_longlong((long long *)src_fld_X, src_nR, nn_src_fld_X,
(long long *)out_X, &dst_nR, (long long *)cnt_X);
}
else {
fprintf(stderr, "NOT IMPLEMENTED\n"); go_BYE(-1);
}
}
}
else { go_BYE(-1); }
if ( dst_nR == 0 ) { goto BYE; }
//-----------------------------------------------------------
status = is_tbl(dst_tbl, &dst_tbl_id); cBYE(status);
if ( dst_tbl_id >= 0 ) {
status = del_tbl(NULL, dst_tbl_id);
}
sprintf(buffer, "%lld", dst_nR);
status = add_tbl(dst_tbl, buffer, &dst_tbl_id);
cBYE(status);
//-----------------------------------------------------------
// Now copy the temporary files over to real files
status = open_temp_file(&ofp, &opfile, 0);
cBYE(status);
if ( strcmp(src_fld_meta->fldtype, "int") == 0 ) {
fwrite(out_X, sizeof(int), dst_nR, ofp);
n_sizeof = sizeof(int);
}
else if ( strcmp(src_fld_meta->fldtype, "long long") == 0 ) {
fwrite(out_X, sizeof(long long), dst_nR, ofp);
n_sizeof = sizeof(long long);
}
fclose_if_non_null(ofp);
// Add output field to meta data
sprintf(str_meta_data, "filename=%s:fldtype=%s:n_sizeof=%u", opfile,
src_fld_meta->fldtype, n_sizeof);
status = add_fld(dst_tbl, dst_fld, str_meta_data, &dst_fld_id);
cBYE(status);
// Since src_fld is sorted ascending, so is dst_fld
g_fld[dst_fld_id].sorttype = 1;
// Now copy the temporary count file to the real one
status = open_temp_file(&cnt_ofp, &cnt_opfile, 0);
cBYE(status);
if ( *in_cnt_fld == '\0' ) {
if ( strcmp(src_fld_meta->fldtype, "int") == 0 ) {
fwrite(cnt_X, sizeof(int), dst_nR, cnt_ofp);
sprintf(str_meta_data, "filename=%s:fldtype=int:n_sizeof=%lu",
cnt_opfile, sizeof(int));
}
else if ( strcmp(src_fld_meta->fldtype, "long long") == 0 ) {
fwrite(cnt_X, sizeof(long long), dst_nR, cnt_ofp);
sprintf(str_meta_data, "filename=%s:fldtype=long long:n_sizeof=%lu",
cnt_opfile, sizeof(long long));
}
else { go_BYE(-1); }
}
else {
if ( strcmp(in_cnt_fld_meta->fldtype, "int") == 0 ) {
fwrite(cnt_X, sizeof(int), dst_nR, cnt_ofp);
sprintf(str_meta_data, "filename=%s:fldtype=int:n_sizeof=%lu",
cnt_opfile, sizeof(int));
}
else if ( strcmp(in_cnt_fld_meta->fldtype, "long long") == 0 ) {
fwrite(cnt_X, sizeof(long long), dst_nR, cnt_ofp);
sprintf(str_meta_data, "filename=%s:fldtype=long long:n_sizeof=%lu",
cnt_opfile, sizeof(long long));
}
else if ( strcmp(in_cnt_fld_meta->fldtype, "double") == 0 ) {
fwrite(cnt_X, sizeof(double), dst_nR, cnt_ofp);
sprintf(str_meta_data, "filename=%s:fldtype=double:n_sizeof=%lu",
cnt_opfile, sizeof(double));
}
else { go_BYE(-1); }
}
fclose_if_non_null(cnt_ofp);
// Add count field to meta data
status = add_fld(dst_tbl, out_cnt_fld, str_meta_data, &cnt_fld_id);
cBYE(status);
//-----------------------------------------------------------
BYE:
g_write_to_temp_dir = false;
rs_munmap(src_fld_X, src_fld_nX);
free_if_non_null(opfile);
rs_munmap(src_fld_X, src_fld_nX);
rs_munmap(in_cnt_fld_X, in_cnt_fld_nX);
rs_munmap(cnt_X, n_cnt_X);
rs_munmap(out_X, n_out_X);
rs_munmap(nn_src_fld_X, nn_src_fld_nX);
unlink(tmp_opfile);
unlink(tmp_cnt_opfile);
free_if_non_null(tmp_opfile);
free_if_non_null(tmp_cnt_opfile);
free_if_non_null(cnt_opfile);
return(status);
}
// START FUNC DECL
int
orphan_files(
char *in_data_dir,
char *action
)
// STOP FUNC DECL
{
int status = 0;
DIR *dp = NULL; FILE *ofp = NULL;
struct dirent *ep = NULL;
char Xopfile[MAX_LEN_FILE_NAME+1];
char Yopfile[MAX_LEN_FILE_NAME+1];
char *X = NULL; size_t nX = 0;
char *Y = NULL; size_t nY = 0;
char nullc = '\0';
int Xnum_files = 0, Ynum_files = 0;
int ddir_id;
char data_dir[MAX_LEN_DIR_NAME+1];
if ( ( in_data_dir == NULL ) || ( *in_data_dir == '\0' ) ) {
status = chdir(g_data_dir); cBYE(status);
ddir_id = -1;
if ( strlen(g_data_dir) >= MAX_LEN_DIR_NAME ) { go_BYE(-1); }
strcpy(data_dir, g_data_dir);
}
else {
status = strip_trailing_slash(in_data_dir, data_dir, MAX_LEN_DIR_NAME+1);
cBYE(status);
status = get_ddir_id(data_dir, g_ddirs, g_n_ddir, false, &ddir_id);
cBYE(status);
status = chdir(data_dir); cBYE(status);
}
zero_string(Xopfile, MAX_LEN_FILE_NAME+1);
zero_string(Yopfile, MAX_LEN_FILE_NAME+1);
/* START: Create X. X = filenames listed in the meta data (whose
* ddir_id matches that requested) in sorted order */
status = open_temp_file(g_cwd, g_cwd, Xopfile, 0); cBYE(status);
ofp = fopen(Xopfile, "w");
return_if_fopen_failed(ofp, Xopfile, "w");
for ( int i = 0; i < g_n_fld; i++ ) {
if ( g_flds[i].name[0] == '\0' ) { continue; }
if ( g_flds[i].ddir_id != ddir_id ) { continue; }
Xnum_files++;
fwrite(g_flds[i].filename, sizeof(char), MAX_LEN_FILE_NAME+1, ofp);
}
fclose_if_non_null(ofp);
if ( Xnum_files > 0 ) {
status = rs_mmap(Xopfile, &X, &nX, 1);
qsort(X, Xnum_files, MAX_LEN_FILE_NAME+1, str_compare);
}
/* STOP ---- */
status = open_temp_file(g_cwd, g_cwd, Yopfile, 0); cBYE(status);
ofp = fopen(Yopfile, "w");
return_if_fopen_failed(ofp, Xopfile, "w");
dp = opendir(data_dir);
if ( dp == NULL) { go_BYE(-1); }
for ( ; ; ) {
ep = readdir (dp);
if ( ep == NULL ) { break; }
char *filename = ep->d_name;
if ( ( filename == NULL ) || ( *filename == '\0' ) ) { break; }
if ( strncmp(filename, "docroot", 7) == 0 ) { continue; }
if ( strncmp(filename, ".LDB_META", 9) == 0 ) { continue; }
if ( strcmp(filename, "..") == 0 ) { continue; }
if ( strcmp(filename, ".") == 0 ) { continue; }
int len = strlen(filename);
if ( len >= MAX_LEN_FILE_NAME ) {
// This is clearly no good
fprintf(stdout, "%s\n", filename);
}
else {
Ynum_files++;
fwrite(filename, sizeof(char), len, ofp);
for ( int i = len; i < MAX_LEN_FILE_NAME+1; i++ ) {
fwrite(&nullc, sizeof(char), 1, ofp);
}
}
}
fclose_if_non_null(ofp);
(void) closedir (dp);
if ( Ynum_files > 0 ) {
status = rs_mmap(Yopfile, &Y, &nY, 1);
qsort(Y, Ynum_files, MAX_LEN_FILE_NAME+1, str_compare);
}
/* Now compare X against Y */
char *xptr = X;
char *yptr = Y;
int seenX = 0, seenY = 0;
for ( ; ( ( seenX < Xnum_files ) && ( seenY < Ynum_files ) ) ; ) {
int cmpval = str_compare(xptr, yptr);
switch ( cmpval ) {
case 0 :
/* File in meta data exists on disk. This is good */
xptr += MAX_LEN_FILE_NAME+1;
yptr += MAX_LEN_FILE_NAME+1;
seenX++;
seenY++;
break;
case -1 :
/* File exists in meta data but not on disk. This is horrendous */
printf("HORRENDOUS %s\n", yptr);
xptr += MAX_LEN_FILE_NAME+1;
seenX++;
break;
case 1 :
/* File on disk but not in meta data. This is an "orphan" * */
printf("%s\n", yptr);
yptr += MAX_LEN_FILE_NAME+1;
seenY++;
break;
default :
go_BYE(-1);
break;
}
}
if ( seenX < Xnum_files ) {
for ( ; seenX < Xnum_files ; seenX++ ) {
fprintf(stderr, "MISSING FILE %s\n", xptr);
xptr += MAX_LEN_FILE_NAME;
seenX++;
}
}
if ( seenY < Ynum_files ) {
for ( ; seenY < Ynum_files ; seenY++ ) {
fprintf(stderr, "%s\n", yptr);
yptr += MAX_LEN_FILE_NAME;
seenY++;
}
}
BYE:
fclose_if_non_null(ofp);
unlink_if_non_null(Xopfile);
unlink_if_non_null(Yopfile);
return(status);
}
//---------------------------------------------------------------
int
crossprod1(
char *docroot,
sqlite3 *in_db,
char *t1,
char *f1,
char *f2
)
{
int status = 0;
sqlite3 *db = NULL;
char *f1_X = NULL; size_t f1_nX = 0;
char *f2_X = NULL; size_t f2_nX = 0;
char *nn_f1_X = NULL; size_t nn_f1_nX = 0;
char *nn_f2_X = NULL; size_t nn_f2_nX = 0;
int t1_id, t2_id, t3_id, itemp; bool b_is_tbl;
FLD_META_TYPE f1_meta, f2_meta;
FLD_META_TYPE nn_f1_meta, nn_f2_meta;
int i1val, i2val; int *iptr1 = NULL, *iptr2 = NULL;
long long nR1, nR2, nR3 = 0;
char str_meta_data[1024];
char *t3f1_opfile = NULL, *t3f2_opfile = NULL;
FILE *ofp1 = NULL, *ofp2 = NULL;
char str_nR3[32];
//----------------------------------------------------------------
zero_string(str_meta_data, 1024);
zero_string(str_nR3, 32);
zero_fld_meta(&f1_meta);
zero_fld_meta(&f2_meta);
zero_fld_meta(&nn_f1_meta);
zero_fld_meta(&nn_f2_meta);
//----------------------------------------------------------------
status = open_temp_file(&ofp1, &t3f1_opfile);
cBYE(status);
status = open_temp_file(&ofp2, &t3f2_opfile);
cBYE(status);
status = open_db_if_needed(docroot, in_db, &db);
cBYE(status);
//----------------------------------------------------------------
status = external_get_nR(docroot, db, t1, &nR1, &t1_id);
cBYE(status);
if ( nR1 <= 0 ) { go_BYE(-1); }
status = fld_meta(docroot, db, t1, f1, -1, &f1_meta);
cBYE(status);
status = rs_mmap(f1_meta.filename, &f1_X, &f1_nX, 0);
cBYE(status);
// Get nn field for t1/f1 if it exists
status = get_aux_field_if_it_exists(docroot, db, t1, f1_meta.id,
"nn", &nn_f1_meta, &nn_f1_X, &nn_f1_nX);
//----------------------------------------------------------------
status = external_get_nR(docroot, db, t2, &nR2, &t2_id);
cBYE(status);
if ( nR2 <= 0 ) { go_BYE(-1); }
status = fld_meta(docroot, db, t2, f2, -1, &f2_meta);
cBYE(status);
status = rs_mmap(f2_meta.filename, &f2_X, &f2_nX, 0);
cBYE(status);
// Get nn field for t2/f2 if it exists
status = get_aux_field_if_it_exists(docroot, db, t2, f2_meta.id,
"nn", &nn_f2_meta, &nn_f2_X, &nn_f2_nX);
//----------------------------------------------------------------
iptr1 = (int *)f1_X;
iptr2 = (int *)f2_X;
for ( long long i1 = 0; i1 < nR1; i1++ ) {
if ( ( nn_f1_X == NULL ) || ( nn_f1_X[i1] == 1 ) ) {
i1val = iptr1[i1];
for ( long long i2 = 0; i2 < nR2; i2++ ) {
if ( ( nn_f2_X == NULL ) || ( nn_f2_X[i2] == 1 ) ) {
i2val = iptr2[i2];
fwrite(&i1val, sizeof(int), 1, ofp1);
fwrite(&i2val, sizeof(int), 1, ofp2);
nR3++;
}
}
}
}
fclose_if_non_null(ofp1);
fclose_if_non_null(ofp2);
//----------------------------------------------------------------
// Add output fields to t3 meta data
status = is_tbl(docroot, db, t3, &b_is_tbl, &t3_id, &itemp);
cBYE(status);
if ( b_is_tbl ) {
status = del_tbl(docroot, db, t3);
cBYE(status);
}
if ( nR3 == 0 ) {
fprintf(stderr, "No data to create t3 \n");
goto BYE;
}
sprintf(str_nR3, "%lld", nR3);
status = add_tbl(docroot, db, t3, str_nR3, &itemp);
cBYE(status);
sprintf(str_meta_data, "fldtype=int:n_sizeof=%d:filename=%s",
sizeof(int), t3f1_opfile);
status = add_fld(docroot, db, t3, f1, str_meta_data);
cBYE(status);
zero_string(str_meta_data, 1024);
sprintf(str_meta_data, "fldtype=int:n_sizeof=%d:filename=%s",
sizeof(int), t3f2_opfile);
status = add_fld(docroot, db, t3, f2, str_meta_data);
cBYE(status);
BYE:
fclose_if_non_null(ofp1);
fclose_if_non_null(ofp2);
if ( in_db == NULL ) { sqlite3_close(db); }
rs_munmap(f1_X, f1_nX);
rs_munmap(nn_f1_X, nn_f1_nX);
rs_munmap(f2_X, f2_nX);
rs_munmap(nn_f2_X, nn_f2_nX);
free_if_non_null(t3f1_opfile);
free_if_non_null(t3f2_opfile);
return(status);
}
//---------------------------------------------------------------
// START FUNC DECL
int
count_strings(
char *src_tbl,
char *src_fld,
char *str_trunc_len,
// truncate strings beyond this len
char *dst_tbl
)
// STOP FUNC DECL
{
int status = 0;
char *X = NULL; size_t nX = 0;
char *src_fld_X = NULL; size_t src_fld_nX = 0;
char *nn_src_fld_X = NULL; size_t nn_src_fld_nX = 0;
char *sz_src_fld_X = NULL; size_t sz_src_fld_nX = 0;
FLD_TYPE *src_fld_meta = NULL;
FLD_TYPE *nn_src_fld_meta = NULL;
FLD_TYPE *sz_src_fld_meta = NULL;
long long src_nR;
int src_tbl_id = INT_MIN, src_fld_id = INT_MIN;
int dst_tbl_id = INT_MIN;
int nn_src_fld_id = INT_MIN, sz_src_fld_id = INT_MIN;
int itemp;
char str_dst_nR[32]; long long dst_nR = 0;
char str_meta_data[4096];
char *out_X = NULL; char *cnt_X = NULL;
char *dst_fk_X = NULL; size_t dst_fk_nX = 0;
FILE *ofp = NULL; char *opfile = NULL;
FILE *tmp_ofp = NULL; char *tmp_opfile = NULL;
FILE *nn_ofp = NULL; char *nn_opfile = NULL;
FILE *sz_ofp = NULL; char *sz_opfile = NULL;
FILE *cnt_ofp = NULL; char *cnt_opfile = NULL;
FILE *from_dst_ofp = NULL; char *from_dst_opfile = NULL;
FILE *dst_fk_ofp = NULL; char *dst_fk_opfile = NULL; int *idst_fk = NULL;
FILE *to_dst_ofp = NULL; char *to_dst_opfile = NULL;
size_t out_nX = 0, cnt_nX = 0;
int *szptr = NULL; char *nnptr = NULL;
char *cptr = NULL, *prev_cptr = NULL;
char *sz_src_fld = NULL; char *str_maxlen = NULL;
char *from_dst_fld = NULL, *to_dst_fld = NULL, *dst_fk_fld = NULL;
char *null_str = NULL, *endptr = NULL;
int maxlen, rec_size, trunc_len = -1; // no truncation by default
long long cnt = 1; int icnt = 1;
bool is_any_null = false;
char is_nn; int sz;
//----------------------------------------------------------------
if ( str_trunc_len != NULL ) {
trunc_len = strtol(str_trunc_len, &endptr, 10);
if ( *endptr != '\0' ) { go_BYE(-1); }
if ( trunc_len <= 1 ) { go_BYE(-1); }
trunc_len++; // to include null character
}
if ( ( src_tbl == NULL ) || ( *src_tbl == '\0' ) ) { go_BYE(-1); }
if ( ( src_fld == NULL ) || ( *src_fld == '\0' ) ) { go_BYE(-1); }
if ( ( dst_tbl == NULL ) || ( *dst_tbl == '\0' ) ) { go_BYE(-1); }
zero_string(str_meta_data, 4096);
zero_string(str_dst_nR, 32);
//--------------------------------------------------------
status = is_tbl(src_tbl, &src_tbl_id); cBYE(status);
chk_range(src_tbl_id, 0, g_n_tbl);
status = is_fld(NULL, src_tbl_id, src_fld, &src_fld_id); cBYE(status);
chk_range(src_fld_id, 0, g_n_fld);
src_nR = g_tbl[src_tbl_id].nR;
if ( src_nR > INT_MAX ) {
fprintf(stderr, "NOT IMPLEMENTED\n"); go_BYE(-1);
}
//--------------------------------------------------------
src_fld_meta = &(g_fld[src_fld_id]);
status = rs_mmap(src_fld_meta->filename, &src_fld_X, &src_fld_nX, 0);
cBYE(status);
//--------------------------------------------------------
if ( strcmp(src_fld_meta->fldtype, "char string") != 0 ) { go_BYE(-1); }
cptr = src_fld_X;
// Get nn field for source if if it exists
nn_src_fld_id = g_fld[src_fld_id].nn_fld_id;
if ( nn_src_fld_id >= 0 ) {
nn_src_fld_meta = &(g_fld[nn_src_fld_id]);
status = rs_mmap(nn_src_fld_meta->filename, &nn_src_fld_X, &nn_src_fld_nX, 0);
cBYE(status);
}
nnptr = (char *)nn_src_fld_X;
// Get sz field for source
sz_src_fld_id = g_fld[src_fld_id].sz_fld_id;
chk_range(sz_src_fld_id, 0, g_n_fld);
sz_src_fld_meta = &(g_fld[sz_src_fld_id]);
status = rs_mmap(sz_src_fld_meta->filename, &sz_src_fld_X, &sz_src_fld_nX, 0);
cBYE(status);
szptr = (int *)sz_src_fld_X;
//------------------------------------
mcr_alloc_null_str(from_dst_fld, (MAX_LEN_FILE_NAME+16));
mcr_alloc_null_str(to_dst_fld, (MAX_LEN_FILE_NAME+16));
mcr_alloc_null_str(dst_fk_fld, (MAX_LEN_FILE_NAME+16));
mcr_alloc_null_str(sz_src_fld, (MAX_LEN_FILE_NAME+16));
mcr_alloc_null_str(str_maxlen, 32);
strcpy(sz_src_fld, "_sz_"); strcat(sz_src_fld, src_fld);
strcpy(from_dst_fld, src_tbl); strcat(from_dst_fld, "_from");
strcpy(to_dst_fld, dst_tbl); strcat(to_dst_fld, "_to");
strcpy(dst_fk_fld, dst_tbl); strcat(dst_fk_fld, "_id");
// Get max length of src field
status = f_to_s(src_tbl, sz_src_fld, "max", str_maxlen);
cBYE(status);
maxlen = strtol(str_maxlen, &endptr, 10);
if ( *endptr != '\0' ) { go_BYE(-1); }
if ( maxlen <= 0 ) { go_BYE(-1); }
if ( trunc_len < 0 ) {
trunc_len = maxlen;
}
mcr_alloc_null_str(null_str, trunc_len);
//--------------------------------------------------------
// Write out src field as constant length field
status = open_temp_file(&tmp_ofp, &tmp_opfile, 0);
cBYE(status);
for ( int i = 0; i < src_nR; i++ ) {
int sz, fill_sz;
sz = szptr[i]; // includes terminating null character
if ( sz <= trunc_len ) {
fwrite(cptr, sizeof(char), sz, tmp_ofp);
fill_sz = trunc_len - sz;
fwrite(null_str, sizeof(char), fill_sz, tmp_ofp);
}
else {
fwrite(cptr, sizeof(char), trunc_len-1, tmp_ofp);
fwrite(null_str, sizeof(char), 1, tmp_ofp);
}
// Append origin of this string
fwrite(&i, sizeof(int), 1, tmp_ofp);
cptr += sz;
// TODO: Check about nullc and sz
}
fclose_if_non_null(tmp_ofp);
//---------------------------------------------------------
// Sort output file so that similar titles occur together
status = rs_mmap(tmp_opfile, &X, &nX, 1); // 1 for writing
cBYE(status);
// Note that we added the origin of the string. Hence the size of a
// record is trunc_len + sizeof(int)
rec_size = trunc_len + sizeof(int);
qsort(X, src_nR, rec_size, str_compare);
rs_munmap(X, nX);
//--------------------------------------------------------
// Now we need to compact the sorted output so that it is not padded
// Open 4 files for
// output string, sz(output string), nn(output string), cnt
status = open_temp_file(&ofp, &opfile, 0); cBYE(status);
status = open_temp_file(&nn_ofp, &nn_opfile, 0); cBYE(status);
status = open_temp_file(&sz_ofp, &sz_opfile, 0); cBYE(status);
status = open_temp_file(&cnt_ofp, &cnt_opfile, 0); cBYE(status);
status = open_temp_file(&from_dst_ofp, &from_dst_opfile, 0); cBYE(status);
status = open_temp_file(&to_dst_ofp, &to_dst_opfile, 0); cBYE(status);
status = open_temp_file(&dst_fk_ofp, &dst_fk_opfile, 0); cBYE(status);
fclose_if_non_null(dst_fk_ofp);
mk_file(dst_fk_opfile, src_nR * sizeof(int)); cBYE(status);
rs_mmap(dst_fk_opfile, &dst_fk_X, &dst_fk_nX, 1); cBYE(status);
idst_fk = (int *)dst_fk_X;
status = rs_mmap(tmp_opfile, &X, &nX, 0);
cBYE(status);
cptr = X;
prev_cptr = cptr;
cptr += rec_size;
cnt = icnt = 1;
for ( int i = 0; i < src_nR; i++ ) {
char *xptr; int *iptr;
xptr = prev_cptr + trunc_len;
iptr = (int *)xptr;
if ( ( *iptr < 0 ) || ( *iptr >= src_nR ) ) { go_BYE(-1); }
fwrite(iptr, sizeof(int), 1, from_dst_ofp);
fwrite(&dst_nR, sizeof(int), 1, to_dst_ofp);
idst_fk[*iptr] = dst_nR;
if ( strcmp(cptr, prev_cptr) != 0 ) {
sz = strlen(prev_cptr);
if ( sz == 0 ) { is_nn = FALSE; } else { is_nn = TRUE; }
sz++; // for null character termination
fwrite(prev_cptr, sizeof(char), sz, ofp);
fwrite(&sz, sizeof(int), 1, sz_ofp);
fwrite(&is_nn, sizeof(char), 1, nn_ofp);
if ( cnt > INT_MAX ) { go_BYE(-1); }
icnt = cnt;
fwrite(&icnt, sizeof(int), 1, cnt_ofp);
cnt = 1;
dst_nR++;
}
else {
cnt++;
}
prev_cptr = cptr;
cptr += rec_size;
}
// START: Now for the last guy
cnt--; // this is needed
sz = strlen(prev_cptr);
if ( sz == 0 ) { is_nn = FALSE; } else { is_nn = TRUE; }
if ( is_nn == FALSE ) {
is_any_null = true;
}
sz++; // for null character termination
fwrite(prev_cptr, sizeof(char), sz, ofp);
fwrite(&sz, sizeof(int), 1, sz_ofp);
fwrite(&is_nn, sizeof(char), 1, nn_ofp);
if ( cnt > INT_MAX ) { go_BYE(-1); }
icnt = cnt;
fwrite(&icnt, sizeof(int), 1, cnt_ofp);
dst_nR++;
// STOP: Now for the last guy
//---------------------------
fclose_if_non_null(ofp);
fclose_if_non_null(nn_ofp);
fclose_if_non_null(sz_ofp);
fclose_if_non_null(cnt_ofp);
fclose_if_non_null(from_dst_ofp);
fclose_if_non_null(to_dst_ofp);
//-----------------------------------------------
// Delete temp file
unlink(tmp_opfile);
//-----------------------------------------------------------
status = is_tbl(dst_tbl, &dst_tbl_id); cBYE(status);
if ( dst_tbl_id >= 0 ) { /* Delete table */
status = del_tbl(NULL, dst_tbl_id);
}
sprintf(str_dst_nR, "%lld", dst_nR);
status = add_tbl(dst_tbl, str_dst_nR, &dst_tbl_id);
cBYE(status);
//-----------------------------------------------------------
/* TODO I think we can dispense with from_dst_fld and to_dst_fld */
sprintf(str_meta_data, "filename=%s:fldtype=int:n_sizeof=4", from_dst_opfile);
status = add_fld(src_tbl, from_dst_fld, str_meta_data, &itemp);
cBYE(status);
sprintf(str_meta_data, "filename=%s:fldtype=int:n_sizeof=4", to_dst_opfile);
status = add_fld(src_tbl, to_dst_fld, str_meta_data, &itemp);
cBYE(status);
sprintf(str_meta_data, "filename=%s:fldtype=int:n_sizeof=4", dst_fk_opfile);
status = add_fld(src_tbl, dst_fk_fld, str_meta_data, &itemp);
cBYE(status);
// Add output field to meta data
sprintf(str_meta_data, "filename=%s:fldtype=char string:n_sizeof=0", opfile);
status = add_fld(dst_tbl, src_fld, str_meta_data, &itemp);
cBYE(status);
fclose_if_non_null(cnt_ofp);
// Add nn field if needed
if ( is_any_null ) {
status = add_aux_fld(dst_tbl, src_fld, nn_opfile, "nn", &itemp);
cBYE(status);
}
else {
unlink(nn_opfile);
}
// Add sz field
status = add_aux_fld(dst_tbl, src_fld, sz_opfile, "sz", &itemp);
cBYE(status);
// Add count field to meta data
sprintf(str_meta_data, "filename=%s:fldtype=int:n_sizeof=4",
cnt_opfile);
status = add_fld(dst_tbl, "cnt", str_meta_data, &itemp);
cBYE(status);
//-----------------------------------------------------------
BYE:
free_if_non_null(null_str);
rs_munmap(X, nX);
rs_munmap(src_fld_X, src_fld_nX);
rs_munmap(cnt_X, cnt_nX);
rs_munmap(out_X, out_nX);
rs_munmap(dst_fk_X, dst_fk_nX);
rs_munmap(nn_src_fld_X, nn_src_fld_nX);
fclose_if_non_null(ofp); free_if_non_null(opfile);
fclose_if_non_null(nn_ofp); free_if_non_null(nn_opfile);
fclose_if_non_null(sz_ofp); free_if_non_null(sz_opfile);
fclose_if_non_null(cnt_ofp); free_if_non_null(cnt_opfile);
fclose_if_non_null(tmp_ofp); free_if_non_null(tmp_opfile);
fclose_if_non_null(from_dst_ofp); free_if_non_null(from_dst_opfile);
fclose_if_non_null(to_dst_ofp); free_if_non_null(to_dst_opfile);
free_if_non_null(sz_src_fld);
free_if_non_null(dst_fk_fld);
free_if_non_null(str_maxlen);
return(status);
}
// START FUNC DECL
int
read_csv(
FLD_META_TYPE *flds,
int n_flds,
char *infile,
char *str_load_fld,
char fld_delim, /* double quote character */
char fld_sep, /* comma */
char rec_delim, /* new line character */
bool ignore_hdr, /* whether to ignore header or not */
bool is_null_null, /* whether to convert "null" to null */
char ***ptr_nn_fnames,
char ***ptr_sz_fnames,
long long *ptr_num_rows
)
// STOP FUNC DECL
{
int status = 0;
int *load_fld = NULL; int n_load_fld = 0;
char *X = NULL; size_t nX = 0;
bool in_fld, is_fld_delim;
long long num_rows = 0; long long num_cells = 0, fld_idx;
long long lb, ub;
FILE **fps = NULL; char **fnames = NULL;
FILE **nn_fps = NULL; char **nn_fnames = NULL;
FILE **sz_fps = NULL; char **sz_fnames = NULL;
bool in_escape_mode = false, is_fld_delim_escaped = false;
g_num_rows = 0;
g_buflen = 8192;
g_buffer = (char *)malloc(g_buflen * sizeof(char));
return_if_malloc_failed(g_buffer);
zero_string(g_buffer, g_buflen);
/*----------------------------------------------------------------------------*/
if ( n_flds <= 0 ) { go_BYE(-1); }
fps = (FILE **)malloc(n_flds * sizeof(FILE *));
return_if_malloc_failed(fps);
fnames = (char **)malloc(n_flds * sizeof(char *));
return_if_malloc_failed(fnames);
nn_fps = (FILE **)malloc(n_flds * sizeof(FILE *));
return_if_malloc_failed(nn_fps);
nn_fnames = (char **)malloc(n_flds * sizeof(char *));
return_if_malloc_failed(nn_fnames);
sz_fps = (FILE **)malloc(n_flds * sizeof(FILE *));
return_if_malloc_failed(sz_fps);
sz_fnames = (char **)malloc(n_flds * sizeof(char *));
return_if_malloc_failed(sz_fnames);
for ( int i = 0; i < n_flds; i++ ) {
fps[i] = NULL;
nn_fps[i] = NULL;
sz_fps[i] = NULL;
fnames[i] = nn_fnames[i] = sz_fnames[i] = NULL;
}
/*----------------------------------------------------------------------------*/
if ( ( str_load_fld == NULL ) || ( *str_load_fld == '\0' ) ) {
load_fld = (int *)malloc(n_flds * sizeof(int));
return_if_malloc_failed(load_fld);
for ( int i = 0; i < n_flds; i++ ) {
load_fld[i]= 1;
}
}
status = parse_scalar_vals(str_load_fld, &load_fld, &n_load_fld);
cBYE(status);
if ( n_load_fld != n_flds ) { go_BYE(-1); }
status = rs_mmap(infile, &X, &nX, 0);
cBYE(status);
lb = 0; fld_idx = 0; in_fld = false; is_fld_delim = false;
for ( int i = 0; i < n_flds; i++ ) {
if ( load_fld[i] == TRUE ) {
status = open_temp_file(&(fps[i]), &(fnames[i]));
cBYE(status);
status = open_temp_file(&(nn_fps[i]), &(nn_fnames[i]));
cBYE(status);
if ( strcmp(flds[i].fldtype, "char string" ) == 0 ) {
status = open_temp_file(&(sz_fps[i]), &(sz_fnames[i]));
cBYE(status);
}
}
}
for ( long long i = 0; i < nX; i++ ) {
// START: Error handling for escape conventions
/* TODO: If I get an incorrect backslash, I simply replace it with a
* space. This is incorrect. But I need it to read files created
* with jspool utility from Jay Thomas. The correct thing to do is
* to fix the utility.
*/
if ( in_escape_mode ) {
if ( ( X[i] != '\\' ) && ( X[i] != '"' ) ) {
/* replace the backslash that put is into escape mode wih a space */
X[i-1] = ' ';
}
}
// STOP: Error handling for escape conventions
// START: Handling bslash and dquote
if ( X[i] == '\\' ) {
if ( in_escape_mode ) {
in_escape_mode = false;
}
else {
in_escape_mode = true;
}
}
// STOP: Handling bslash and dquote
if ( X[i] == '"' ) {
if ( in_escape_mode ) {
is_fld_delim_escaped = true;
in_escape_mode = false;
}
}
if ( in_fld == false ) {
in_fld = true;
if ( X[i] == fld_delim ) { /* consume this character */
is_fld_delim = true;
lb = i+1;
}
else {
lb = i;
if ( ( ( fld_idx < (n_flds-1) ) && ( X[i] == fld_sep ) ) ||
( ( fld_idx == n_flds-1 ) && ( X[i] == rec_delim ) ) ) {
/* field has terminated */
ub = i-1;
if ( load_fld[fld_idx] == 1 ) {
if ( ( ignore_hdr ) & ( num_rows == 0 ) ) {
/* Do nothing */
}
else {
status = print_cell(X, lb, ub+1, flds[fld_idx].fldtype,
flds[fld_idx].n_sizeof, is_null_null,
fps[fld_idx], nn_fps[fld_idx],
sz_fps[fld_idx]);
cBYE(status);
}
}
in_fld = false;
num_cells++;
if ( ( fld_idx == n_flds-1 ) && ( X[i] == rec_delim ) ) {
num_rows++;
g_num_rows = num_rows;
}
fld_idx++;
if ( fld_idx == n_flds ) { fld_idx = 0; }
}
}
}
else { /* Now within field. Decide whether to continue or stop. */
if ( is_fld_delim ) {
/* if field started with a fld_delim, it must end with one */
/* Note that we need to take care of escaping the dquote */
if ( ( X[i] == fld_delim ) && ( !is_fld_delim_escaped ) ) {
/* next char must be fld_sep or rec_delim or we have come to eof */
if ( i+1 > nX ) { go_BYE(-1); }
if ( ( fld_idx < (n_flds-1) ) && ( X[i+1] != fld_sep ) ) {
go_BYE(-1);
}
if ( fld_idx == n_flds-1 ) {
if ( X[i+1] != rec_delim ) {
go_BYE(-1);
}
else {
num_rows++;
g_num_rows = num_rows;
}
}
ub = i-1;
if ( load_fld[fld_idx] == 1 ) {
if ( ( ignore_hdr ) & ( num_rows == 0 ) ) {
/* Do nothing */
}
else {
status = print_cell(X, lb, ub+1, flds[fld_idx].fldtype,
flds[fld_idx].n_sizeof, is_null_null,
fps[fld_idx], nn_fps[fld_idx], sz_fps[fld_idx]);
cBYE(status);
}
}
in_fld = false;
is_fld_delim = false;
fld_idx++;
if ( fld_idx == n_flds ) { fld_idx = 0; }
num_cells++;
i++; /* Consume next char which must be fld_sep or rec_delim */
}
}
else {
if ( ( i == nX ) ||
( ( fld_idx < (n_flds-1) ) && ( X[i] == fld_sep ) ) ||
( ( fld_idx == n_flds-1 ) && ( X[i] == rec_delim ) ) ) {
/* field has terminated */
if ( i == nX ) {
ub = i;
}
else {
ub = i-1;
}
if ( load_fld[fld_idx] == 1 ) {
if ( ( ignore_hdr ) & ( num_rows == 0 ) ) {
/* Do nothing */
}
else {
status = print_cell(X, lb, ub+1, flds[fld_idx].fldtype,
flds[fld_idx].n_sizeof, is_null_null,
fps[fld_idx], nn_fps[fld_idx], sz_fps[fld_idx]);
cBYE(status);
}
}
in_fld = false;
num_cells++;
if ( ( fld_idx == n_flds-1 ) && ( X[i] == rec_delim ) ) {
num_rows++;
g_num_rows = num_rows;
}
fld_idx++;
if ( fld_idx == n_flds ) { fld_idx = 0; }
}
}
}
is_fld_delim_escaped = false;
}
if ( num_cells != ( n_flds * num_rows ) ) {
fprintf(stderr, "num_cells = %lld \n", num_cells);
fprintf(stderr, "n_flds = %d \n", n_flds );
fprintf(stderr, "num_rows = %lld \n", num_rows);
go_BYE(-1);
}
if ( ignore_hdr ) {
num_rows--; // Because you have eliminated first row
}
/* TODO: Copy information into meta data structure */
for ( int i = 0; i < n_flds; i++ ) {
if ( fnames[i] != NULL ) { // since not all fields are loaded
strcpy(flds[i].filename, fnames[i]);
}
}
for ( int i = 0; i < n_flds; i++ ) {
if ( fnames != NULL ) { free_if_non_null(fnames[i]); }
if ( fps != NULL ) { fclose_if_non_null(fps[i]); }
if ( nn_fps != NULL ) { fclose_if_non_null(nn_fps[i]); }
if ( sz_fps != NULL ) { fclose_if_non_null(sz_fps[i]); }
}
*ptr_nn_fnames = nn_fnames;
*ptr_sz_fnames = sz_fnames;
*ptr_num_rows = num_rows;
BYE:
rs_munmap(X, nX);
free_if_non_null(load_fld);
free_if_non_null(nn_fps);
free_if_non_null(sz_fps);
free_if_non_null(fnames);
free_if_non_null(fps);
free_if_non_null(g_buffer);
return(status);
}
int csync_rs_patch(const char *filename)
{
FILE *basis_file = 0, *delta_file = 0, *new_file = 0;
int backup_errno;
rs_stats_t stats;
rs_result result;
char *errstr = "?";
char tmpfname[MAXPATHLEN], newfname[MAXPATHLEN];
csync_debug(3, "Csync2 / Librsync: csync_rs_patch('%s')\n", filename);
csync_debug(3, "Receiving delta_file from peer..\n");
delta_file = open_temp_file(tmpfname, prefixsubst(filename));
if ( !delta_file ) { errstr="creating delta temp file"; goto io_error; }
if (unlink(tmpfname) < 0) { errstr="removing delta temp file"; goto io_error; }
if ( csync_recv_file(delta_file) ) goto error;
csync_debug(3, "Opening to be patched file on local host..\n");
basis_file = fopen(prefixsubst(filename), "rb");
if ( !basis_file ) {
basis_file = open_temp_file(tmpfname, prefixsubst(filename));
if ( !basis_file ) { errstr="opening data file for reading"; goto io_error; }
if (unlink(tmpfname) < 0) { errstr="removing data temp file"; goto io_error; }
}
csync_debug(3, "Opening temp file for new data on local host..\n");
new_file = open_temp_file(newfname, prefixsubst(filename));
if ( !new_file ) { errstr="creating new data temp file"; goto io_error; }
csync_debug(3, "Running rs_patch_file() from librsync..\n");
result = rs_patch_file(basis_file, delta_file, new_file, &stats);
if (result != RS_DONE) {
csync_debug(0, "Internal error from rsync library!\n");
goto error;
}
csync_debug(3, "Renaming tmp file to data file..\n");
fclose(basis_file);
#ifdef __CYGWIN__
/* TODO: needed? */
// This creates the file using the native windows API, bypassing
// the cygwin wrappers and so making sure that we do not mess up the
// permissions..
{
char winfilename[MAX_PATH];
HANDLE winfh;
cygwin_conv_to_win32_path(prefixsubst(filename), winfilename);
winfh = CreateFile(TEXT(winfilename),
GENERIC_WRITE, // open for writing
0, // do not share
NULL, // default security
CREATE_ALWAYS, // overwrite existing
FILE_ATTRIBUTE_NORMAL | // normal file
FILE_FLAG_OVERLAPPED, // asynchronous I/O
NULL); // no attr. template
if (winfh == INVALID_HANDLE_VALUE) {
csync_debug(0, "Win32 I/O Error %d in rsync-patch: %s\n",
(int)GetLastError(), winfilename);
errno = EACCES;
goto error;
}
CloseHandle(winfh);
}
#endif
if (rename(newfname, prefixsubst(filename)) < 0) { errstr="renaming tmp file to to be patched file"; goto io_error; }
csync_debug(3, "File has been patched successfully.\n");
fclose(delta_file);
fclose(new_file);
return 0;
io_error:
csync_debug(0, "I/O Error '%s' while %s in rsync-patch: %s\n",
strerror(errno), errstr, prefixsubst(filename));
error:;
backup_errno = errno;
if ( delta_file ) fclose(delta_file);
if ( basis_file ) fclose(basis_file);
if ( new_file ) fclose(new_file);
errno = backup_errno;
return -1;
}
static int get_data(
char *tbl,
char *fld,
long long *ptr_nR,
char **ptr_X,
size_t *ptr_nX,
char **ptr_opfile,
int *ptr_fldtype,
int *ptr_fldsz
)
{
int status = 0;
int tbl_id = INT_MIN, fld_id = INT_MIN, nn_fld_id = INT_MIN;
char *X = NULL; size_t nX = 0;
char *nn_X = NULL; size_t nn_nX = 0;
char *op_X = NULL; size_t op_nX = 0;
char *endptr = NULL;
char buffer[32];
long long nR = 0;
FLD_TYPE *fld_meta = NULL, *nn_fld_meta = NULL;
FILE *ofp = NULL; char *opfile = NULL;
zero_string(buffer, 32);
status = is_tbl(tbl, &tbl_id); cBYE(status);
chk_range(tbl_id, 0, g_n_tbl);
status = is_fld(NULL, tbl_id, fld, &fld_id); cBYE(status);
chk_range(fld_id, 0, g_n_fld);
nR = g_tbl[tbl_id].nR;
fld_meta = &(g_fld[fld_id]);
nn_fld_id = g_fld[fld_id].nn_fld_id;
if ( nn_fld_id >= 0 ) {
nn_fld_meta = &(g_fld[nn_fld_id]);
status = rs_mmap(nn_fld_meta->filename, &nn_X, &nn_nX, 0);
cBYE(status);
}
*ptr_fldsz = fld_meta->n_sizeof;
status = mk_ifldtype(fld_meta->fldtype, ptr_fldtype); cBYE(status);
switch ( *ptr_fldtype ) {
case FLDTYPE_INT :
case FLDTYPE_LONGLONG :
/* all is well */
break;
default :
go_BYE(-1);
}
if ( nn_X == NULL ) { /* no nn field */
status = rs_mmap(fld_meta->filename, &X, &nX, 0);
cBYE(status);
*ptr_nR = nR;
*ptr_X = X;
*ptr_nX = nX;
*ptr_opfile = NULL;
}
else {
long long nn_nR = 0;
status = f_to_s(tbl, nn_fld_meta->name, "sum", buffer);
cBYE(status);
nn_nR = strtol(buffer, &endptr, 10);
if ( *endptr != '\0' ) { go_BYE(-1); }
if ( ( nn_nR <= 0 ) || ( nn_nR >= nR ) ) { go_BYE(-1); }
/* Now make a copy of just the non-null values */
status = rs_mmap(fld_meta->filename, &X, &nX, 0);
cBYE(status);
status = rs_mmap(nn_fld_meta->filename, &nn_X, &nn_nX, 0);
cBYE(status);
long long filesz = nn_nR * fld_meta->n_sizeof;
status = open_temp_file(&ofp, &opfile, filesz); cBYE(status);
fclose_if_non_null(ofp);
status = mk_file(opfile, filesz); cBYE(status);
status = rs_mmap(opfile, &op_X, &op_nX, 1); cBYE(status);
status = copy_nn_vals(X, nR, nn_X, op_X, nn_nR, fld_meta->n_sizeof);
cBYE(status);
/* Cleanup and return values */
rs_munmap(X, nX);
rs_munmap(nn_X, nn_nX);
*ptr_nR = nn_nR;
*ptr_X = op_X;
*ptr_nX = op_nX;
*ptr_opfile = opfile;
}
BYE:
return(status);
}
int csync_rs_delta(const char *filename)
{
FILE *sig_file = 0, *new_file = 0, *delta_file = 0;
rs_result result;
rs_signature_t *sumset;
rs_stats_t stats;
char tmpfname[MAXPATHLEN];
csync_debug(3, "Csync2 / Librsync: csync_rs_delta('%s')\n", filename);
csync_debug(3, "Receiving sig_file from peer..\n");
sig_file = open_temp_file(tmpfname, prefixsubst(filename));
if ( !sig_file ) goto io_error;
if (unlink(tmpfname) < 0) goto io_error;
if ( csync_recv_file(sig_file) ) {
fclose(sig_file);
return -1;
}
result = rs_loadsig_file(sig_file, &sumset, &stats);
if (result != RS_DONE)
csync_fatal("Got an error from librsync, too bad!\n");
fclose(sig_file);
csync_debug(3, "Opening new_file and delta_file..\n");
new_file = fopen(prefixsubst(filename), "rb");
if ( !new_file ) {
int backup_errno = errno;
csync_debug(0, "I/O Error '%s' while %s in rsync-delta: %s\n",
strerror(errno), "opening data file for reading", filename);
csync_send_error();
fclose(new_file);
errno = backup_errno;
return -1;
}
delta_file = open_temp_file(tmpfname, prefixsubst(filename));
if ( !delta_file ) goto io_error;
if (unlink(tmpfname) < 0) goto io_error;
csync_debug(3, "Running rs_build_hash_table() from librsync..\n");
result = rs_build_hash_table(sumset);
if (result != RS_DONE)
csync_fatal("Got an error from librsync, too bad!\n");
csync_debug(3, "Running rs_delta_file() from librsync..\n");
result = rs_delta_file(sumset, new_file, delta_file, &stats);
if (result != RS_DONE)
csync_fatal("Got an error from librsync, too bad!\n");
csync_debug(3, "Sending delta_file to peer..\n");
csync_send_file(delta_file);
csync_debug(3, "Delta has been created successfully.\n");
rs_free_sumset(sumset);
fclose(delta_file);
fclose(new_file);
return 0;
io_error:
csync_debug(0, "I/O Error '%s' in rsync-delta: %s\n",
strerror(errno), prefixsubst(filename));
if (new_file) fclose(new_file);
if (delta_file) fclose(delta_file);
if (sig_file) fclose(sig_file);
return -1;
}
//---------------------------------------------------------------
// START FUNC DECL
int
f1opf2(
char *tbl,
char *f1,
char *str_op_spec,
char *f2
)
// STOP FUNC DECL
{
int status = 0;
char *f1_X = NULL;
size_t f1_nX = 0;
char *nn_f1_X = NULL;
size_t nn_f1_nX = 0;
char *sz_f1_X = NULL;
size_t sz_f1_nX = 0;
FLD_TYPE *f1_meta = NULL, *nn_f1_meta = NULL, *sz_f1_meta = NULL;
long long nR;
int tbl_id = INT_MIN, f1_id = INT_MIN, nn_f1_id = INT_MIN, sz_f1_id = INT_MIN;
int f2_id = INT_MIN, nn_f2_id = INT_MIN, sz_f2_id = INT_MIN;
char str_meta_data[1024];
int f1type;
char *op = NULL, *str_val = NULL;
FILE *ofp = NULL;
char *opfile = NULL;
FILE *nn_ofp = NULL;
char *nn_opfile = NULL;
FILE *sz_ofp = NULL;
char *sz_opfile = NULL;
char *xform_enum = NULL, *hash_algo = NULL;
//----------------------------------------------------------------
if ( ( tbl == NULL ) || ( *tbl == '\0' ) ) {
go_BYE(-1);
}
if ( ( f1 == NULL ) || ( *f1 == '\0' ) ) {
go_BYE(-1);
}
if ( ( f2 == NULL ) || ( *f2 == '\0' ) ) {
go_BYE(-1);
}
if ( ( str_op_spec == NULL ) || ( *str_op_spec == '\0' ) ) {
go_BYE(-1);
}
zero_string(str_meta_data, 1024);
//--------------------------------------------------------
status = extract_name_value(str_op_spec, "op=", ":", &op);
cBYE(status);
if ( op == NULL ) {
go_BYE(-1);
}
//--------------------------------------------------------
status = chk_if_ephemeral(&f2);
cBYE(status);
status = is_tbl(tbl, &tbl_id);
cBYE(status);
chk_range(tbl_id, 0, g_n_tbl);
nR = g_tbl[tbl_id].nR;
status = is_fld(NULL, tbl_id, f1, &f1_id);
cBYE(status);
chk_range(f1_id, 0, g_n_fld);
f1_meta = &(g_fld[f1_id]);
status = rs_mmap(f1_meta->filename, &f1_X, &f1_nX, 0);
cBYE(status);
// Get nn field for f1 if if it exists
nn_f1_id = g_fld[f1_id].nn_fld_id;
if ( nn_f1_id >= 0 ) {
nn_f1_meta = &(g_fld[nn_f1_id]);
status = rs_mmap(nn_f1_meta->filename, &nn_f1_X, &nn_f1_nX, 0);
cBYE(status);
}
//---------------------------------------------
status = mk_ifldtype(f1_meta->fldtype, &f1type);
/* TODO: Why do we get an error for 'char string' ? */
// If f1 type is 'char string', then get sz field
if ( strcmp(f1_meta->fldtype, "char string") == 0 ) {
sz_f1_id = g_fld[f1_id].sz_fld_id;
chk_range(sz_f1_id, 0, g_n_fld);
sz_f1_meta = &(g_fld[sz_f1_id]);
status = rs_mmap(sz_f1_meta->filename, &sz_f1_X, &sz_f1_nX, 0);
cBYE(status);
}
//--------------------------------------------------------
if ( strcmp(op, "xform") == 0 ) {
bool is_some_null;
status = open_temp_file(&ofp, &opfile, 0);
cBYE(status);
status = open_temp_file(&nn_ofp, &nn_opfile, 0);
cBYE(status);
status = open_temp_file(&sz_ofp, &sz_opfile, 0);
cBYE(status);
status = str_xform(nR, f1_X, (int *)sz_f1_X, nn_f1_X, str_op_spec,
ofp, nn_ofp, sz_ofp, &is_some_null);
cBYE(status);
fclose_if_non_null(ofp);
fclose_if_non_null(nn_ofp);
fclose_if_non_null(sz_ofp);
sprintf(str_meta_data, "filename=%s:fldtype=char string:n_sizeof=0",
opfile);
status = add_fld(tbl, f2, str_meta_data, &f2_id);
cBYE(status);
status = add_aux_fld(tbl, f2, sz_opfile, "sz", &sz_f2_id);
cBYE(status);
if ( is_some_null ) {
status = add_aux_fld(tbl, f2, nn_opfile, "nn", &nn_f2_id);
cBYE(status);
}
else {
unlink(nn_opfile);
}
}
else if ( strcmp(op, "hash") == 0 ) {
int hash_len;
char *endptr = NULL;
status = extract_name_value(str_op_spec, "len=", ":", &str_val);
cBYE(status);
if ( str_val == NULL ) {
go_BYE(-1);
}
hash_len = strtol(str_val, &endptr, 10);
if ( *endptr != '\0' ) {
go_BYE(-1);
}
status = extract_name_value(str_op_spec, "hash_algo=", ":", &hash_algo);
cBYE(status);
status = open_temp_file(&ofp, &opfile, 0);
cBYE(status);
status = f1opf2_hash(hash_algo, nR, f1_X, f1_nX, (int *)sz_f1_X,
f1_meta->n_sizeof, nn_f1_X, hash_len, ofp);
cBYE(status);
fclose_if_non_null(ofp);
sprintf(str_meta_data, "filename=%s", opfile);
switch ( hash_len ) {
case 4 :
strcat(str_meta_data, ":fldtype=int:n_sizeof=4");
break;
case 8 :
strcat(str_meta_data, ":fldtype=long long:n_sizeof=8");
break;
default :
go_BYE(-1);
break;
}
fclose_if_non_null(ofp);
status = add_fld(tbl, f2, str_meta_data, &f2_id);
cBYE(status);
}
else if ( ( strcmp(op, "conv") == 0 ) ||
( strcmp(op, "dateconv" ) == 0 ) ||
( strcmp(op, "bitcount" ) == 0 ) ||
( strcmp(op, "sqrt" ) == 0 ) ||
( strcmp(op, "!" ) == 0 ) ||
( strcmp(op, "++" ) == 0 ) ||
( strcmp(op, "--" ) == 0 ) ||
( strcmp(op, "~" ) == 0 )
) {
status = vec_f1opf2(nR, f1type, str_meta_data,
f1_X, nn_f1_X, sz_f1_X, op, str_op_spec, &opfile, &nn_opfile);
cBYE(status);
status = add_fld(tbl, f2, str_meta_data, &f2_id);
cBYE(status);
if ( nn_opfile != NULL ) {
status = add_aux_fld(tbl, f2, nn_opfile, "nn", &nn_f2_id);
cBYE(status);
}
}
else if ( strcmp(op, "shift") == 0 ) {
int ival;
char *endptr = NULL;
status = extract_name_value(str_op_spec, "val=", ":", &str_val);
cBYE(status);
ival = strtol(str_val, &endptr, 10);
if ( *endptr != '\0' ) {
fprintf(stderr, "invalid shift specifier = [%s] \n", str_val);
go_BYE(-1);
}
if ( ( ival == 0 ) || ( ival > MAX_SHIFT ) ||
( ival < -MAX_SHIFT ) || (ival >= nR )) {
fprintf(stderr, "shift = [%d] is out of bounds \n", ival);
go_BYE(-1);
}
status = open_temp_file(&ofp, &opfile, 0);
cBYE(status);
status = open_temp_file(&nn_ofp, &nn_opfile, 0);
cBYE(status);
status = f1opf2_shift(f1_X, nn_f1_X, nR, f1_meta->fldtype, ival,
ofp, nn_ofp);
cBYE(status);
fclose_if_non_null(ofp);
fclose_if_non_null(nn_ofp);
sprintf(str_meta_data, "filename=%s:fldtype=%s:n_sizeof=%d",opfile,
f1_meta->fldtype, f1_meta->n_sizeof);
status = add_fld(tbl, f2, str_meta_data, &f2_id);
cBYE(status);
status = add_aux_fld(tbl, f2, nn_opfile, "nn", &nn_f2_id);
cBYE(status);
}
else if ( strcmp(op, "cum") == 0 ) {
/* TODO: Document. If you do a cum, resultant field is all def */
char *new_fld_type = NULL;
int optype, n_sizeof;
char *str_reset_on = NULL, *str_reset_to = NULL, *endptr = NULL;
long long reset_on = 0, reset_to = 0;
bool is_reset = false;
status = extract_name_value(str_op_spec, "newtype=", ":", &new_fld_type);
cBYE(status);
status = extract_name_value(str_op_spec, "reset_on=", ":",
&str_reset_on);
cBYE(status);
if ( str_reset_on != NULL ) {
reset_on = strtoll(str_reset_on, &endptr, 10);
if ( *endptr != '\0' ) {
go_BYE(-1);
}
is_reset = true;
status = extract_name_value(str_op_spec, "reset_to=", ":",
&str_reset_to);
cBYE(status);
if ( str_reset_to == NULL ) {
go_BYE(-1);
}
reset_to = strtoll(str_reset_to, &endptr, 10);
if ( *endptr != '\0' ) {
go_BYE(-1);
}
}
if ( new_fld_type == NULL ) {
optype = f1type;
new_fld_type = strdup(f1_meta->fldtype);
n_sizeof = f1_meta->n_sizeof;
}
else {
if ( strcmp(new_fld_type, "int") == 0 ) {
optype = FLDTYPE_INT;
n_sizeof = sizeof(int);
}
else if ( strcmp(new_fld_type, "long long") == 0 ) {
optype = FLDTYPE_LONGLONG;
n_sizeof = sizeof(long long);
}
else {
go_BYE(-1);
}
}
status = f1opf2_cum(f1_X, nR, f1type, optype, is_reset,
reset_on, reset_to, &opfile);
cBYE(status);
sprintf(str_meta_data, "filename=%s:fldtype=%s:n_sizeof=%d",opfile,
new_fld_type, n_sizeof);
status = add_fld(tbl, f2, str_meta_data, &f2_id);
cBYE(status);
free_if_non_null(new_fld_type);
free_if_non_null(str_reset_on);
free_if_non_null(str_reset_to);
}
else {
fprintf(stderr, "Invalid op = [%s] \n", op);
go_BYE(-1);
}
BYE:
fclose_if_non_null(ofp);
fclose_if_non_null(nn_ofp);
g_write_to_temp_dir = false;
rs_munmap(f1_X, f1_nX);
rs_munmap(nn_f1_X, nn_f1_nX);
rs_munmap(sz_f1_X, sz_f1_nX);
free_if_non_null(op);
free_if_non_null(xform_enum);
free_if_non_null(hash_algo);
free_if_non_null(str_val);
free_if_non_null(opfile);
free_if_non_null(nn_opfile);
free_if_non_null(sz_opfile);
return(status);
}
int csync_rs_check(const char *filename, int isreg)
{
FILE *basis_file = 0, *sig_file = 0;
char buffer1[512], buffer2[512];
int rc, chunk, found_diff = 0;
int backup_errno;
rs_stats_t stats;
rs_result result;
long size;
char tmpfname[MAXPATHLEN];
csync_debug(3, "Csync2 / Librsync: csync_rs_check('%s', %d [%s])\n",
filename, isreg, isreg ? "regular file" : "non-regular file");
csync_debug(3, "Opening basis_file and sig_file..\n");
sig_file = open_temp_file(tmpfname, prefixsubst(filename));
if ( !sig_file ) goto io_error;
if (unlink(tmpfname) < 0) goto io_error;
basis_file = fopen(prefixsubst(filename), "rb");
if ( !basis_file ) { /* ?? why a tmp file? */
basis_file = open_temp_file(tmpfname, prefixsubst(filename));
if ( !basis_file ) goto io_error;
if (unlink(tmpfname) < 0) goto io_error;
}
if ( isreg ) {
csync_debug(3, "Running rs_sig_file() from librsync....\n");
result = rs_sig_file(basis_file, sig_file,
RS_DEFAULT_BLOCK_LEN, RS_DEFAULT_STRONG_LEN, &stats);
if (result != RS_DONE) {
csync_debug(0, "Internal error from rsync library!\n");
goto error;
}
}
fclose(basis_file);
basis_file = 0;
{
char line[100];
csync_debug(3, "Reading signature size from peer....\n");
if ( !conn_gets(line, 100) || sscanf(line, "octet-stream %ld\n", &size) != 1 )
csync_fatal("Format-error while receiving data.\n");
}
fflush(sig_file);
if ( size != ftell(sig_file) ) {
csync_debug(2, "Signature size differs: local=%d, peer=%d\n",
ftell(sig_file), size);
found_diff = 1;
}
rewind(sig_file);
csync_debug(3, "Receiving %ld bytes ..\n", size);
while ( size > 0 ) {
chunk = size > 512 ? 512 : size;
rc = conn_read(buffer1, chunk);
if ( rc <= 0 )
csync_fatal("Read-error while receiving data.\n");
chunk = rc;
if ( fread(buffer2, chunk, 1, sig_file) != 1 ) {
csync_debug(2, "Found EOF in local sig file.\n");
found_diff = 1;
}
if ( memcmp(buffer1, buffer2, chunk) ) {
csync_debug(2, "Found diff in sig at -%d:-%d\n",
size, size-chunk);
found_diff = 1;
}
size -= chunk;
csync_debug(3, "Got %d bytes, %ld bytes left ..\n",
chunk, size);
}
csync_debug(3, "File has been checked successfully (%s).\n",
found_diff ? "difference found" : "files are equal");
fclose(sig_file);
return found_diff;
io_error:
csync_debug(0, "I/O Error '%s' in rsync-check: %s\n",
strerror(errno), prefixsubst(filename));
error:;
backup_errno = errno;
if ( basis_file ) fclose(basis_file);
if ( sig_file ) fclose(sig_file);
errno = backup_errno;
return -1;
}
//---------------------------------------------------------------
// START FUNC DECL
int
srt_join(
char *docroot,
sqlite3 *in_db,
char *src_tbl,
char *src_lnk,
char *src_val,
char *dst_tbl,
char *dst_lnk,
char *dst_fld,
char *op
)
// STOP FUNC DECL
{
int status = 0;
sqlite3 *db = NULL;
char *src_val_X = NULL; size_t src_val_nX = 0;
char *src_lnk_X = NULL; size_t src_lnk_nX = 0;
char *nn_dst_val_X = NULL; size_t nn_dst_val_nX = 0;
char *dst_val_X = NULL; size_t dst_val_nX = 0;
char *dst_lnk_X = NULL; size_t dst_lnk_nX = 0;
FLD_META_TYPE src_val_meta, src_lnk_meta;
FLD_META_TYPE dst_lnk_meta;
long long src_nR, dst_nR; // counts all rows
char str_meta_data[1024];
int ijoin_op; bool is_any_null = false;
int src_tbl_id, dst_tbl_id; bool b_is_tbl;
FILE *ofp = NULL, *nn_ofp = NULL;
char *opfile = NULL, *nn_opfile = NULL;
//----------------------------------------------------------------
zero_string(str_meta_data, 1024);
status = mk_mjoin_op(op, &ijoin_op);
cBYE(status);
status = open_db_if_needed(docroot, in_db, &db);
cBYE(status);
zero_fld_meta(&src_val_meta);
zero_fld_meta(&src_lnk_meta);
zero_fld_meta(&dst_lnk_meta);
//----------------------------------------------------------------
// Get meta-data for all necessary fields
status = is_tbl(docroot, db, src_tbl, &b_is_tbl, &src_tbl_id);
cBYE(status);
if ( b_is_tbl == false ) {
fprintf(stderr, "tbl [%s] not found\n", src_tbl); go_BYE(-1);
}
status = is_tbl(docroot, db, dst_tbl, &b_is_tbl, &dst_tbl_id);
cBYE(status);
if ( b_is_tbl == false ) {
fprintf(stderr, "tbl [%s] not found\n", src_tbl); go_BYE(-1);
}
status = fld_meta(docroot, db, src_tbl, src_lnk, -1, &src_lnk_meta);
cBYE(status);
if ( ( src_val != NULL ) && ( *src_val != '\0' ) ) {
status = fld_meta(docroot, db, src_tbl, src_val, -1, &src_val_meta);
cBYE(status);
}
status = fld_meta(docroot, db, dst_tbl, dst_lnk, -1, &dst_lnk_meta);
cBYE(status);
status = internal_get_nR(db, src_tbl_id, &src_nR);
cBYE(status);
status = internal_get_nR(db, dst_tbl_id, &dst_nR);
cBYE(status);
//----------------------------------------------------------------
// Get pointer access to all necessary fields
status = rs_mmap(src_lnk_meta.filename, &src_lnk_X, &src_lnk_nX, 0);
cBYE(status);
if ( ( src_val != NULL ) && ( *src_val != '\0' ) ) {
status = rs_mmap(src_val_meta.filename, &src_val_X, &src_val_nX, 0);
cBYE(status);
}
status = rs_mmap(dst_lnk_meta.filename, &dst_lnk_X, &dst_lnk_nX, 0);
cBYE(status);
//--------------------------------------------------------
// Create output data files
status = open_temp_file(&ofp, &opfile); cBYE(status);
fclose_if_non_null(ofp);
status = open_temp_file(&nn_ofp, &nn_opfile); cBYE(status);
fclose_if_non_null(nn_ofp);
if ( ( src_val != NULL ) && ( *src_val != '\0' ) ) {
status = mk_file(opfile, src_val_meta.n_sizeof * dst_nR);
cBYE(status);
status = rs_mmap(opfile, &dst_val_X, &dst_val_nX, 1); cBYE(status);
}
status = mk_file(nn_opfile, sizeof(char) * dst_nR);
cBYE(status);
status = rs_mmap(nn_opfile, &nn_dst_val_X, &nn_dst_val_nX, 1); cBYE(status);
//--------------------------------------------------------
// Core join
if ( ( strcmp(src_lnk_meta.fldtype, "int") == 0 ) &&
( strcmp(src_lnk_meta.fldtype, "int") == 0 ) &&
( strcmp(dst_lnk_meta.fldtype, "int") == 0 ) ) {
status = core_srt_join_I_I_I(
(int *)src_lnk_X, (int *)src_val_X, src_nR,
(int *)dst_lnk_X, (int *)dst_val_X, nn_dst_val_X,
0, dst_nR, ijoin_op, &is_any_null);
}
else if ( ( strcmp(src_lnk_meta.fldtype, "int") == 0 ) &&
( strcmp(src_lnk_meta.fldtype, "int") == 0 ) &&
( strcmp(dst_lnk_meta.fldtype, "long long") == 0 ) ) {
status = core_srt_join_I_I_L(
(int *)src_lnk_X, (int *)src_val_X, src_nR,
(long long *)dst_lnk_X, (int *)dst_val_X, nn_dst_val_X,
0, dst_nR, ijoin_op, &is_any_null);
}
else if ( ( strcmp(src_lnk_meta.fldtype, "int") == 0 ) &&
( strcmp(src_lnk_meta.fldtype, "long long") == 0 ) &&
( strcmp(dst_lnk_meta.fldtype, "int") == 0 ) ) {
status = core_srt_join_I_L_I(
(int *)src_lnk_X, (long long
*)src_val_X, src_nR,
(int *)dst_lnk_X, (long long *)dst_val_X, nn_dst_val_X,
0, dst_nR, ijoin_op, &is_any_null);
}
else if ( ( strcmp(src_lnk_meta.fldtype, "int") == 0 ) &&
( strcmp(src_lnk_meta.fldtype, "long long") == 0 ) &&
( strcmp(dst_lnk_meta.fldtype, "long long") == 0 ) ) {
status = core_srt_join_I_L_L((int *)src_lnk_X,
(long long *)src_val_X, src_nR, (long long *)dst_lnk_X,
(long long *)dst_val_X, nn_dst_val_X, 0, dst_nR, ijoin_op,
&is_any_null);
}
else if ( ( strcmp(src_lnk_meta.fldtype, "long long") == 0 ) &&
( strcmp(src_lnk_meta.fldtype, "int") == 0 ) &&
( strcmp(dst_lnk_meta.fldtype, "int") == 0 ) ) {
status = core_srt_join_L_I_I(
(long long *)src_lnk_X, (int
*)src_val_X, src_nR,
(int *)dst_lnk_X, (int *)dst_val_X, nn_dst_val_X,
0, dst_nR, ijoin_op, &is_any_null);
}
else if ( ( strcmp(src_lnk_meta.fldtype, "long long") == 0 ) &&
( strcmp(src_lnk_meta.fldtype, "int") == 0 ) &&
( strcmp(dst_lnk_meta.fldtype, "long long") == 0 ) ) {
status = core_srt_join_L_I_L(
(long long *)src_lnk_X, (int
*)src_val_X, src_nR,
(long long *)dst_lnk_X, (int *)dst_val_X, nn_dst_val_X,
0, dst_nR, ijoin_op, &is_any_null);
}
else if ( ( strcmp(src_lnk_meta.fldtype, "long long") == 0 ) &&
( strcmp(src_lnk_meta.fldtype, "long long") == 0 ) &&
( strcmp(dst_lnk_meta.fldtype, "int") == 0 ) ) {
status = core_srt_join_L_L_I(
(long long *)src_lnk_X, (long long
*)src_val_X, src_nR,
(int *)dst_lnk_X, (long long *)dst_val_X, nn_dst_val_X,
0, dst_nR, ijoin_op, &is_any_null);
}
else if ( ( strcmp(src_lnk_meta.fldtype, "long long") == 0 ) &&
( strcmp(src_lnk_meta.fldtype, "long long") == 0 ) &&
( strcmp(dst_lnk_meta.fldtype, "long long") == 0 ) ) {
status = core_srt_join_L_L_L(
(long long *)src_lnk_X, (long long
*)src_val_X, src_nR,
(long long *)dst_lnk_X, (long long *)dst_val_X, nn_dst_val_X,
0, dst_nR, ijoin_op, &is_any_null);
}
else { go_BYE(-1); }
//--------------------------------------------------------
// Add output field to meta data
if ( ( src_val != NULL ) && ( *src_val != '\0' ) ) {
sprintf(str_meta_data,"filename=%s:n_sizeof=%d:fldtype=%s", opfile,
src_val_meta.n_sizeof, src_val_meta.fldtype);
status = add_fld(docroot, db, dst_tbl, dst_fld, str_meta_data);
cBYE(status);
if ( is_any_null == true ) {
status = add_aux_fld(docroot, db, dst_tbl, dst_fld, nn_opfile, "nn");
cBYE(status);
}
else {
unlink(nn_opfile);
free_if_non_null(nn_opfile);
}
}
else {
sprintf(str_meta_data,"filename=%s:n_sizeof=%ld:fldtype=bool",
nn_opfile, sizeof(char));
status = add_fld(docroot, db, dst_tbl, dst_fld, str_meta_data);
cBYE(status);
}
BYE:
if ( in_db == NULL ) { sqlite3_close(db); }
rs_munmap(src_val_X, src_val_nX);
rs_munmap(src_lnk_X, src_lnk_nX);
rs_munmap(dst_val_X, dst_val_nX);
rs_munmap(nn_dst_val_X, nn_dst_val_nX);
rs_munmap(dst_lnk_X, dst_lnk_nX);
free_if_non_null(nn_opfile);
free_if_non_null(opfile);
return(status);
}
off_t oggedit_write_flac_metadata(DB_FILE *in, const char *fname, const off_t offset, const int num_tags, char **tags)
{
off_t res;
char tempname[PATH_MAX] = "";
ogg_packet **headers = NULL;
char *vendor = NULL;
ogg_sync_state oy;
ogg_sync_init(&oy);
/* Original file must be writable whichever way we update it */
FILE *out = fopen(fname, "r+b");
if (!out) {
res = OGGEDIT_CANNOT_UPDATE_FILE;
goto cleanup;
}
/* See if we can write the tags packet directly into the existing file ... */
if (!(headers = metadata_block_packets(in, &oy, offset, &vendor, (int *)&res)))
goto cleanup;
const off_t stream_size_k = in->vfs->getlength(in) / 1000; // use file size for now
const size_t metadata_size = 4 + vc_size(vendor, num_tags, tags);
ptrdiff_t padding = headers[0]->bytes - metadata_size;
if (stream_size_k < 1000 || padding < 0 || (headers[1] && padding > 0) || padding > stream_size_k+metadata_size) {
res = open_temp_file(fname, tempname, &out);
if (res) {
goto cleanup;
}
}
/* Re-pad if writing the whole file */
if (*tempname) {
size_t i = 0, j = 0;
while (headers[i]) {
headers[j++] = headers[i];
while (headers[++i] && (headers[i]->packet[0] & 0x3F) == PADTYPE) {
ogg_packet_clear(headers[i]);
free(headers[i]);
}
}
headers[j] = NULL;
padding = headers[1] || stream_size_k < 900 ? 0 : stream_size_k < 10000 ? 1024 : stream_size_k < 100000 ? 8192 : 65536;
}
/* Write pages until we reach the correct comment header */
ogg_page og;
const int flac_serial = copy_up_to_codec(in, out, &oy, &og, *tempname ? 0 : offset, offset, FLACNAME);
if (flac_serial <= OGGEDIT_EOF) {
res = flac_serial;
goto cleanup;
}
if ((res = copy_up_to_header(in, out, &oy, &og, flac_serial)) <= OGGEDIT_EOF)
goto cleanup;
const long pageno = write_metadata_block_packets(out, flac_serial, vendor, num_tags, tags, padding, headers);
if (pageno < OGGEDIT_EOF) {
res = pageno;
goto cleanup;
}
/* If we have tempfile, copy the remaining pages */
if (*tempname) {
if ((res = copy_remaining_pages(in, out, &oy, flac_serial, pageno)) <= OGGEDIT_EOF)
goto cleanup;
if (rename(tempname, fname)) {
res = OGGEDIT_RENAME_FAILED;
goto cleanup;
}
}
res = file_size(fname);
cleanup:
clear_header_list(headers);
cleanup(in, out, &oy, vendor);
if (res < OGGEDIT_OK)
unlink(tempname);
return res;
}
/* TODO: This routine needs more work to deal with escaping dquotes and
* backslashes and dealing with eoln within the quotes */
int
main(
int argc,
char **argv
)
{
int status = 0;
char *infile = NULL;
char *outfile_str = NULL;
char *outfile_sz = NULL;
char *outfile_nn = NULL;
FILE *ofp = NULL, *szfp = NULL, *nnfp = NULL;
char *X = NULL; size_t nX = 0;
long long nR = 0, start_loc = -1, stop_loc =-1, len,fld_len;
char nullc = '\0';
bool is_in_str, pr_str;
char c_one = 1; char c_zero = 0;
if ( argc != 2 ) { go_BYE(-1); }
infile = argv[1];
status = open_temp_file(&ofp, &outfile_str, 0); cBYE(status);
status = open_temp_file(&szfp, &outfile_sz, 0); cBYE(status);
status = open_temp_file(&nnfp, &outfile_nn, 0); cBYE(status);
status = rs_mmap(infile, &X, &nX, 0);
cBYE(status);
nR = 0;
is_in_str = false;
pr_str = false;
for ( size_t i = 0; i < nX; i++ ) {
if ( X[i] == '"' ) {
if ( is_in_str == false ) {
is_in_str = true;
start_loc = i+1;
}
else {
is_in_str = false;
stop_loc = i-1;
pr_str = true;
}
}
if ( pr_str == true ) {
if ( ( stop_loc < 0 ) || ( start_loc < 0 ) ||
( ( start_loc - stop_loc ) > 1 ) ) {
fprintf(stderr, "Error at line %lld, position %lld of %lld \n",
nR, (long long)i, (long long)nX);
go_BYE(-1);
}
nR++;
pr_str = false;
len = stop_loc - start_loc + 1; // + 1 bcos both are inclusive
if ( len > 0 ) {
fwrite(X+start_loc, sizeof(char), len, ofp);
}
if ( len > 0 ) {
fwrite(&c_one, sizeof(char), 1, nnfp);
}
else {
fwrite(&c_zero, sizeof(char), 1, nnfp);
}
fwrite(&nullc, sizeof(char), 1, ofp);
fld_len = len + 1;
fwrite(&fld_len, sizeof(int), 1, szfp);
// Consume the eoln if it exists
if ( i+1 < nX ) {
if ( X[i+1] != '\n' ) {
fprintf(stderr, "Error at line %lld, position %lld of %lld \n",
nR, (long long)i, (long long)nX);
go_BYE(-1);
}
i++;
}
}
}
fprintf(stdout, "%lld:%s:%s:%s", nR, outfile_str, outfile_sz, outfile_nn);
BYE:
fclose_if_non_null(ofp);
fclose_if_non_null(szfp);
fclose_if_non_null(nnfp);
rs_munmap(X, nX);
return(status);
}
//---------------------------------------------------------------
// START FUNC DECL
int
xfer_string(
FLD_TYPE *dst_idx_meta,
char *src_fld_X,
char *sz_src_fld_X,
long long src_nR,
char *dst_tbl,
char *dst_fld,
char *dst_idx_X,
char *nn_dst_idx_X,
long long dst_nR
)
// STOP FUNC DECL
{
int status = 0;
char str_meta_data[256];
char *szX = NULL; size_t n_szX= 0; int *szptr = NULL;
char *nnX = NULL; size_t n_nnX= 0; char *nnptr = NULL;
int dst_fld_id, nn_dst_fld_id, sz_dst_fld_id;
FILE *ofp = NULL; char *opfile = NULL;
FILE *nn_ofp = NULL; char *nn_opfile = NULL;
FILE *sz_ofp = NULL; char *sz_opfile = NULL;
long long *offset_src_fld = NULL;
long long *ll_dst_idx = NULL;
bool is_any_null = false;
//--------------------------------------------------------
status = open_temp_file(&ofp, &opfile, 0); cBYE(status);
/* We cannot allocate storage for this since we do not know how much
* to allocate a-priori */
/* Open output fields for size and nn in mmmap mode */
long long filesz = dst_nR * sizeof(int);
status = open_temp_file(&sz_ofp, &sz_opfile, filesz); cBYE(status);
fclose_if_non_null(sz_ofp);
status = mk_file(sz_opfile, filesz); cBYE(status);
status = rs_mmap(sz_opfile, &szX, &n_szX, 1); cBYE(status);
szptr = (int *)szX;
long long nnfilesz = dst_nR * sizeof(char);
status = open_temp_file(&nn_ofp, &nn_opfile, nnfilesz); cBYE(status);
fclose_if_non_null(nn_ofp);
status = mk_file(nn_opfile, nnfilesz); cBYE(status);
status = rs_mmap(nn_opfile, &nnX, &n_nnX, 1); cBYE(status);
nnptr = (char *)nnX;
/*-------------------------------------------------------------*/
status = mk_offset((int *)sz_src_fld_X, src_nR, &offset_src_fld);
cBYE(status);
/* Convert from int to longlong if needed */
if ( strcmp(dst_idx_meta->fldtype, "int") == 0 ) {
ll_dst_idx = malloc(dst_nR * sizeof(long long));
return_if_malloc_failed(ll_dst_idx);
if ( dst_nR >= INT_MAX ) {
fprintf(stderr, "TO BE IMPLEMENTED\n"); go_BYE(-1);
}
conv_int_to_longlong((int *)dst_idx_X, (int)dst_nR, ll_dst_idx);
}
else {
ll_dst_idx = (long long *)dst_idx_X;
}
/*-------------------------------------------------------------*/
for ( long long i = 0; i < dst_nR; i++ ) {
int *i_sz_src_fld_ptr = (int *)sz_src_fld_X;
long long idx;
size_t nw;
char *testptr;
char nullc = 0;
long long offset; int sz;
if ( ( nn_dst_idx_X != NULL ) && ( nn_dst_idx_X[i] == FALSE ) ) {
// Output null value
is_any_null = true;
nnptr[i] = FALSE;
szptr[i] = 1;
nw = fwrite(&nullc, sizeof(char), 1, ofp); // Null terminate
if ( nw != 1 ) { go_BYE(-1); }
}
else {
idx = ll_dst_idx[i];
if ( ( idx < 0 ) || ( idx >= src_nR ) ) { go_BYE(-1); }
sz = i_sz_src_fld_ptr[idx];
offset = offset_src_fld[idx];
// This checks that strings are null terminated
testptr = src_fld_X + offset + sz - 1;
if ( *testptr != '\0' ) {
go_BYE(-1);
}
szptr[i] = sz;
nnptr[i] = TRUE;
nw = fwrite(src_fld_X + offset, sizeof(char), sz, ofp);
if ( nw != sz ) { go_BYE(-1); }
}
}
fclose_if_non_null(ofp);
rs_munmap(szX, n_szX);
rs_munmap(nnX, n_nnX);
// Add output field to meta data
zero_string(str_meta_data, 256);
strcpy(str_meta_data, "fldtype=char string:n_sizeof=0");
strcat(str_meta_data, ":filename=");
strcat(str_meta_data, opfile);
status = add_fld(dst_tbl, dst_fld, str_meta_data, &dst_fld_id); cBYE(status);
// Add size field
status = add_aux_fld(dst_tbl, dst_fld, sz_opfile, "sz", &sz_dst_fld_id);
cBYE(status);
if ( is_any_null ) {
status = add_aux_fld(dst_tbl, dst_fld, nn_opfile, "nn", &nn_dst_fld_id);
cBYE(status);
}
else {
unlink(nn_opfile);
}
BYE:
if ( strcmp(dst_idx_meta->fldtype, "int") == 0 ) {
free_if_non_null(ll_dst_idx);
}
fclose_if_non_null(ofp);
fclose_if_non_null(nn_ofp);
fclose_if_non_null(sz_ofp);
free_if_non_null(opfile);
free_if_non_null(nn_opfile);
free_if_non_null(sz_opfile);
free_if_non_null(offset_src_fld);
return(status);
}
//---------------------------------------------------------------
// START FUNC DECL
int
crossprod(
char *t1,
char *f1,
char *t2,
char *f2,
char *t3
)
// STOP FUNC DECL
{
int status = 0;
char *Y1 = NULL; size_t nY1 = 0;
char *Y2 = NULL; size_t nY2 = 0;
char *f1_X = NULL; size_t f1_nX = 0; char *f1_opfile = NULL;
int f1type, f2type;
char *f2_X = NULL; size_t f2_nX = 0; char *f2_opfile = NULL;
long long f1size, f2size;
int t2f1_fld_id = INT_MIN, t2f2_fld_id = INT_MIN;
int t3_id = INT_MIN, itemp;
long long chk_nR1 = 0, nR1, nR2, nR3;
char str_meta_data[1024];
char *t3f1_opfile = NULL, *t3f2_opfile = NULL;
FILE *ofp = NULL;
char buffer[32];
// For multi-threading
int rc; // result code for thread create
pthread_t threads[MAX_NUM_THREADS];
pthread_attr_t attr;
void *thread_status;
//----------------------------------------------------------------
zero_string(str_meta_data, 1024);
zero_string(buffer, 32);
if ( strcmp(f1, f2) == 0 ) { go_BYE(-1); }
/* Remove f1 != f2 restriction later. To do so, we need to specify
* fields of t3 explicitly */
//----------------------------------------------------------------
status = get_data(t1, f1, &nR1, &f1_X, &f1_nX, &f1_opfile, &f1type, &f1size);
cBYE(status);
status = get_data(t2, f2, &nR2, &f2_X, &f2_nX, &f2_opfile, &f2type, &f2size);
cBYE(status);
nR3 = nR1 * nR2;
if ( nR3 == 0 ) {
fprintf(stderr, "No data to create t3 \n");
goto BYE;
}
// Create storage for field 1 in Table t3 */
long long filesz = nR3 * f1size;
status = open_temp_file(&ofp, &t3f1_opfile, filesz); cBYE(status);
fclose_if_non_null(ofp);
status = mk_file(t3f1_opfile, filesz); cBYE(status);
status = rs_mmap(t3f1_opfile, &Y1, &nY1, 1); cBYE(status);
// Create storage for field 2 in Table t3 */
filesz = nR3 * f2size;
status = open_temp_file(&ofp, &t3f2_opfile, filesz); cBYE(status);
fclose_if_non_null(ofp);
status = mk_file(t3f2_opfile, filesz); cBYE(status);
status = rs_mmap(t3f2_opfile, &Y2, &nY2, 1); cBYE(status);
//----------------------------------------------------------------
/* Set up parallelism computations. Parallelization strategy is
* simple. Partition field 1 (nR1 rows) among the threads */
g_nR1 = nR1;
g_nR2 = nR2;
g_nR3 = nR3;
g_f1type = f1type;
g_f2type = f2type;
g_f1size = f1size;
g_f2size = f2size;
g_f1_X = f1_X;
g_f2_X = f2_X;
g_Y1 = Y1;
g_Y2 = Y2;
for ( int i = 0; i < MAX_NUM_THREADS; i++ ) {
g_thread_id[i] = i;
g_num_rows_processed[i] = 0;
}
status = get_num_threads(&g_nT);
cBYE(status);
//--------------------------------------------
#define MIN_ROWS_FOR_CROSSPROD 4 // 1024
if ( nR1 <= MIN_ROWS_FOR_CROSSPROD ) {
g_nT = 1;
}
/* Don't create more threads than you can use */
if ( g_nT > nR1 ) { g_nT = nR1; }
if ( g_nT == 1 ) {
core_crossprod(&(g_thread_id[0]));
chk_nR1 = g_num_rows_processed[0];
}
else { /* Create threads */
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
for ( int t = 0; t < g_nT; t++ ) {
rc = pthread_create(&threads[t], NULL, core_crossprod,
&(g_thread_id[t]));
if ( rc ) { go_BYE(-1); }
}
/* Free attribute and wait for the other threads */
pthread_attr_destroy(&attr);
for ( int t = 0; t < g_nT; t++ ) {
rc = pthread_join(threads[t], &thread_status);
if ( rc ) { go_BYE(-1); }
chk_nR1 += g_num_rows_processed[t];
}
}
if ( chk_nR1 != nR1 ) { go_BYE(-1); }
//----------------------------------------------------------------
// Add output fields to t3 meta data
status = is_tbl(t3, &t3_id); cBYE(status);
if ( t3_id >= 0 ) {
status = del_tbl(NULL, t3_id);
cBYE(status);
}
sprintf(buffer, "%lld", nR3);
status = add_tbl(t3, buffer, &itemp); cBYE(status);
sprintf(str_meta_data, "fldtype=int:n_sizeof=%u:filename=%s",
f1size, t3f1_opfile);
status = add_fld(t3, f1, str_meta_data, &t2f1_fld_id); cBYE(status);
zero_string(str_meta_data, 1024);
sprintf(str_meta_data, "fldtype=int:n_sizeof=%u:filename=%s",
f2size, t3f2_opfile);
status = add_fld(t3, f2, str_meta_data, &t2f2_fld_id); cBYE(status);
BYE:
fclose_if_non_null(ofp);
rs_munmap(f1_X, f1_nX);
rs_munmap(f2_X, f2_nX);
if ( f1_opfile != NULL ) {
unlink(f1_opfile); free_if_non_null(f1_opfile);
}
if ( f2_opfile != NULL ) {
unlink(f2_opfile); free_if_non_null(f2_opfile);
}
free_if_non_null(t3f1_opfile);
free_if_non_null(t3f2_opfile);
return(status);
}
//---------------------------------------------------------------
// START FUNC DECL
int
dup_fld(
char *tbl,
char *f1,
char *f2
)
// STOP FUNC DECL
{
int status = 0;
char *Y = NULL; size_t nY = 0;
char *X = NULL; size_t nX = 0;
char *nn_X = NULL; size_t nn_nX = 0;
char *sz_X = NULL; size_t sz_nX = 0;
FILE *ofp = NULL; char *opfile = NULL;
FILE *nn_ofp = NULL; char *nn_opfile = NULL;
FILE *sz_ofp = NULL; char *sz_opfile = NULL;
int tbl_id, f1_fld_id, f2_fld_id, nn_fld_id, sz_fld_id;
//----------------------------------------------------------------
if ( ( tbl == NULL ) || ( *tbl == '\0' ) ) { go_BYE(-1); }
if ( ( f1 == NULL ) || ( *f1 == '\0' ) ) { go_BYE(-1); }
if ( ( f2 == NULL ) || ( *f2 == '\0' ) ) { go_BYE(-1); }
if ( strcmp(f1, f2) == 0 ) { go_BYE(-1); }
//--------------------------------------------------------
status = is_tbl(tbl, &tbl_id); cBYE(status);
chk_range(tbl_id, 0, g_n_tbl);
status = is_fld(NULL, tbl_id, f1, &f1_fld_id); cBYE(status);
chk_range(f1_fld_id, 0, g_n_fld);
status = is_fld(NULL, tbl_id, f2, &f2_fld_id); cBYE(status);
if ( f2_fld_id >= 0 ) {
status = del_fld(NULL, tbl_id, NULL, f2_fld_id); cBYE(status);
}
// Open input file
status = rs_mmap(g_fld[f1_fld_id].filename, &X, &nX, 0); cBYE(status);
// Create a copy of the data
status = open_temp_file(&ofp, &opfile, nX); cBYE(status);
fclose_if_non_null(ofp);
status = mk_file(opfile, nX); cBYE(status);
status = rs_mmap(opfile, &Y, &nY, 1);
memcpy(Y, X, nX);
rs_munmap(Y, nY);
// Get empty spot for f2
status = get_empty_fld(&f2_fld_id); cBYE(status);
// f2 is same as f1 except for data and "is_external"
g_fld[f2_fld_id] = g_fld[f1_fld_id];
g_fld[f2_fld_id].is_external = false;
g_fld[f2_fld_id].nn_fld_id = -1;
g_fld[f2_fld_id].sz_fld_id = -1;
status = chk_fld_name(f2, 0);
zero_string(g_fld[f2_fld_id].name, MAX_LEN_FLD_NAME+1);
zero_string(g_fld[f2_fld_id].filename, MAX_LEN_FILE_NAME+1);
strcpy(g_fld[f2_fld_id].name, f2);
strcpy(g_fld[f2_fld_id].filename, opfile);
// Add to hash table
//--------------------------------------------------------
// Add auxiliary field nn if if it exists
nn_fld_id = g_fld[f1_fld_id].nn_fld_id;
if ( nn_fld_id >= 0 ) {
// open input file
status = rs_mmap(g_fld[nn_fld_id].filename, &nn_X, &nn_nX, 0); cBYE(status);
// create copy of data
status = open_temp_file(&nn_ofp, &nn_opfile, nn_nX); cBYE(status);
fclose_if_non_null(nn_ofp);
status = mk_file(nn_opfile, nn_nX); cBYE(status);
status = rs_mmap(nn_opfile, &Y, &nY, 1);
memcpy(Y, nn_X, nn_nX);
rs_munmap(Y, nY);
rs_munmap(nn_X, nn_nX);
nn_fld_id = INT_MIN;
status = add_aux_fld(tbl, f2, nn_opfile, "nn", &nn_fld_id);
cBYE(status);
}
//--------------------------------------------------------
// Add auxiliary field sz if if it exists
sz_fld_id = g_fld[f1_fld_id].sz_fld_id;
if ( sz_fld_id >= 0 ) {
status = open_temp_file(&sz_ofp, &sz_opfile, sz_nX); cBYE(status);
fclose_if_non_null(sz_ofp);
status = mk_file(sz_opfile, sz_nX); cBYE(status);
status = rs_mmap(sz_opfile, &Y, &nY, 1);
status = rs_mmap(g_fld[sz_fld_id].filename, &sz_X, &sz_nX, 0);
memcpy(Y, sz_X, sz_nX);
rs_munmap(Y, nY);
rs_munmap(sz_X, sz_nX);
status = add_aux_fld(tbl, f2, sz_opfile, "sz", &sz_fld_id);
cBYE(status);
}
//--------------------------------------------------------
BYE:
rs_munmap(X, nX);
rs_munmap(nn_X, nn_nX);
rs_munmap(sz_X, sz_nX);
rs_munmap(Y, nY);
fclose_if_non_null(ofp);
fclose_if_non_null(nn_ofp);
fclose_if_non_null(sz_ofp);
free_if_non_null(opfile);
free_if_non_null(nn_opfile);
free_if_non_null(sz_opfile);
return(status);
}
off_t oggedit_write_vorbis_metadata(DB_FILE *in, const char *fname, const off_t offset, const size_t stream_size, const int num_tags, char **tags)
{
off_t res;
char tempname[PATH_MAX] = "";
char *vendor = NULL;
ogg_packet codebooks;
memset(&codebooks, '\0', sizeof(codebooks));
ogg_sync_state oy;
ogg_sync_init(&oy);
/* Original file must be writable whichever way we update it */
FILE *out = fopen(fname, "r+b");
if (!out) {
res = OGGEDIT_CANNOT_UPDATE_FILE;
goto cleanup;
}
/* See if we can write the tags packet directly into the existing file ... */
const ptrdiff_t tags_packet_size = check_vorbis_headers(in, &oy, offset, &vendor, &codebooks);
if (tags_packet_size <= OGGEDIT_EOF) {
res = tags_packet_size;
goto cleanup;
}
const size_t metadata_size = strlen(VCMAGIC) + vc_size(vendor, num_tags, tags) + 1;
ptrdiff_t padding = tags_packet_size - metadata_size;
const off_t file_size_k = in->vfs->getlength(in) / 1000;
const size_t stream_size_k = stream_size ? stream_size / 1000 : file_size_k;
if (file_size_k < 100 || padding < 0 || padding > file_size_k/10+stream_size_k+metadata_size) {
res = open_temp_file(fname, tempname, &out);
if (res) {
goto cleanup;
}
}
/* Re-pad if writing the whole file */
if (*tempname)
padding = stream_size_k < 90 ? 0 : stream_size_k < 1000 ? 128 : stream_size_k < 10000 ? 1024 : 8192;
/* Write pages until the correct comment header */
ogg_page og;
int64_t vorbis_serial = copy_up_to_codec(in, out, &oy, &og, *tempname ? 0 : offset, offset, VORBISNAME);
if (vorbis_serial <= OGGEDIT_EOF) {
res = vorbis_serial;
goto cleanup;
}
vorbis_serial = copy_up_to_header(in, out, &oy, &og, vorbis_serial);
if (vorbis_serial <= OGGEDIT_EOF) {
res = vorbis_serial;
goto cleanup;
}
const long pageno = write_vorbis_tags(out, vorbis_serial, vendor, num_tags, tags, padding, &codebooks);
if (pageno < OGGEDIT_EOF) {
res = pageno;
goto cleanup;
}
/* If we have tempfile, copy the remaining pages */
if (*tempname) {
vorbis_serial = copy_remaining_pages(in, out, &oy, vorbis_serial, pageno);
if (vorbis_serial <= OGGEDIT_EOF) {
res = vorbis_serial;
goto cleanup;
}
if (rename(tempname, fname)) {
res = OGGEDIT_RENAME_FAILED;
goto cleanup;
}
}
res = file_size(fname);
cleanup:
ogg_packet_clear(&codebooks);
cleanup(in, out, &oy, vendor);
if (res < OGGEDIT_OK)
unlink(tempname);
return res;
}
//---------------------------------------------------------------
// START FUNC DECL
int
join(
char *docroot,
sqlite3 *in_db,
char *src_tbl,
char *src_lnk,
char *src_val,
char *dst_tbl,
char *dst_lnk,
char *dst_fld,
char *op
)
// STOP FUNC DECL
{
int status = 0;
sqlite3 *db = NULL;
char *src_val_X = NULL; size_t src_val_nX = 0;
char *src_lnk_X = NULL; size_t src_lnk_nX = 0;
char *src_idx_X = NULL; size_t src_idx_nX = 0;
char *nn_dst_val_X = NULL; size_t nn_dst_val_nX = 0;
char *dst_val_X = NULL; size_t dst_val_nX = 0;
char *dst_lnk_X = NULL; size_t dst_lnk_nX = 0;
char *dst_idx_X = NULL; size_t dst_idx_nX = 0;
FLD_META_TYPE src_val_meta, src_lnk_meta, src_idx_meta;
FLD_META_TYPE dst_val_meta, dst_lnk_meta, dst_idx_meta;
long long src_nR, dst_nR; // counts all rows
char str_meta_data[1024];
int sizeof_src_idx, sizeof_dst_idx;
int *iptr;
int ijoin_op;
char *x_src_tbl = NULL, *x_dst_tbl = NULL;
FILE *ofp = NULL, *nn_ofp = NULL;
char *opfile = NULL, *nn_opfile = NULL;
//----------------------------------------------------------------
zero_string(str_meta_data, 1024);
status = mk_mjoin_op(op, &ijoin_op);
cBYE(status);
status = open_db_if_needed(docroot, in_db, &db);
cBYE(status);
zero_fld_meta(&src_val_meta);
zero_fld_meta(&src_lnk_meta);
zero_fld_meta(&src_idx_meta);
zero_fld_meta(&dst_val_meta);
zero_fld_meta(&dst_lnk_meta);
zero_fld_meta(&dst_idx_meta);
//----------------------------------------------------------------
/* Given the src_tbl, we create x_src_tbl which has no null
values in the link fields and which has an index field called "id"
with values 0, 1, .... |x_src_tbl| -1 */
status = elim_null_vals(docroot, db, src_tbl, src_lnk, &x_src_tbl);
cBYE(status);
/* Similarly for dst_tbl */
status = elim_null_vals(docroot, db, dst_tbl, dst_lnk, &x_dst_tbl);
cBYE(status);
status = sort_if_necessary(docroot, db, x_src_tbl, src_lnk, "id");
cBYE(status);
status = sort_if_necessary(docroot, db, x_dst_tbl, dst_lnk, "id");
cBYE(status);
// Get meta-data for all necessary fields
status = fld_meta(docroot, db, x_src_tbl, src_lnk, -1, &src_lnk_meta);
cBYE(status);
status = fld_meta(docroot, db, x_src_tbl, XXXXXXX, -1, &src_idx_meta);
cBYE(status);
status = fld_meta(docroot, db, x_src_tbl, src_val, -1, &src_val_meta);
cBYE(status);
status = fld_meta(docroot, db, x_dst_tbl, dst_lnk, -1, &dst_lnk_meta);
cBYE(status);
status = fld_meta(docroot, db, x_dst_tbl, XXXXXXX, -1, &dst_idx_meta);
cBYE(status);
// Get pointer access to all necessary fields
status = rs_mmap(src_lnk_meta.filename, &src_lnk_X, &src_lnk_nX, 0);
cBYE(status);
status = rs_mmap(src_idx_meta.filename, &src_idx_X, &src_idx_nX, 0);
cBYE(status);
status = rs_mmap(src_val_meta.filename, &src_val_X, &src_val_nX, 0);
cBYE(status);
status = rs_mmap(dst_lnk_meta.filename, &dst_lnk_X, &dst_lnk_nX, 0);
cBYE(status);
status = rs_mmap(dst_idx_meta.filename, &dst_idx_X, &dst_idx_nX, 0);
cBYE(status);
//--------------------------------------------------------
// Create output data files
status = open_temp_file(&ofp, &opfile); cBYE(status);
fclose_if_non_null(ofp);
status = open_temp_file(&nn_ofp, &nn_opfile); cBYE(status);
fclose_if_non_null(nn_ofp);
status = mk_file(opfile, src_val_meta.n_sizeof * dst_nR);
cBYE(status);
status = mk_file(nn_opfile, sizeof(char) * dst_nR);
cBYE(status);
status = rs_mmap(opfile, &dst_val_X, &dst_val_nX, 1); cBYE(status);
status = rs_mmap(nn_opfile, &nn_dst_val_X, &nn_dst_val_nX, 1); cBYE(status);
//--------------------------------------------------------
// Core join
status = core_join_I_I_I_I_I_I(
(int *)cpy_src_lnk_X, (int *)cpy_src_idx_X,
(int *)src_val_X,
src_nR, nn_src_nR,
(int *)cpy_dst_lnk_X, (int *)cpy_dst_idx_X,
(int *)dst_val_X, nn_dst_val_X,
dst_nR, nn_dst_nR,
ijoin_op, &is_any_null);
//--------------------------------------------------------
// Clean up aux data structures
rs_munmap(cpy_src_lnk_X, cpy_src_lnk_nX);
rs_munmap(cpy_src_idx_X, cpy_src_idx_nX);
rs_munmap(cpy_dst_lnk_X, cpy_dst_lnk_nX);
rs_munmap(cpy_dst_idx_X, cpy_dst_idx_nX);
if ( cpy_src_lnk_file == NULL ) {
status = del_fld(docroot, db, src_tbl,"_cpy_lnk"); cBYE(status);
}
else {
unlink(cpy_src_lnk_file); free_if_non_null(cpy_src_lnk_file);
}
if ( cpy_src_idx_file == NULL ) {
status = del_fld(docroot, db, src_tbl,"_cpy_idx"); cBYE(status);
}
else {
unlink(cpy_src_idx_file); free_if_non_null(cpy_src_idx_file);
}
if ( cpy_dst_lnk_file == NULL ) {
status = del_fld(docroot, db, dst_tbl,"_cpy_lnk"); cBYE(status);
}
else {
unlink(cpy_dst_lnk_file); free_if_non_null(cpy_dst_lnk_file);
}
if ( cpy_dst_idx_file == NULL ) {
status = del_fld(docroot, db, dst_tbl,"_cpy_idx"); cBYE(status);
}
else {
unlink(cpy_dst_idx_file); free_if_non_null(cpy_dst_idx_file);
}
// Add output field to meta data
sprintf(str_meta_data,"filename=%s:n_sizeof=%d:fldtype=%s", opfile,
src_val_meta.n_sizeof, src_val_meta.fldtype);
status = add_fld(docroot, db, dst_tbl, dst_fld, str_meta_data);
cBYE(status);
if ( is_any_null == true ) {
status = add_aux_fld(docroot, db, dst_tbl, dst_fld, nn_opfile, "nn");
cBYE(status);
}
else {
unlink(nn_opfile);
free_if_non_null(nn_opfile);
}
BYE:
if ( in_db == NULL ) { sqlite3_close(db); }
rs_munmap(dst_val_X, dst_nX);
rs_munmap(nn_dst_val_X, nn_dst_nX);
rs_munmap(src_val_X, src_val_nX);
rs_munmap(src_lnk_X, src_lnk_nX);
rs_munmap(src_lnk_X, src_lnk_nX);
rs_munmap(dst_lnk_X, dst_lnk_nX);
rs_munmap(dst_idx_X, dst_idx_nX);
free_if_non_null(nn_opfile);
free_if_non_null(opfile);
return(status);
}
//---------------------------------------------------------------
// START FUNC DECL
int
num_in_range(
char *t1,
char *f1,
char *t2,
char *lb,
char *ub,
char *cnt
)
// STOP FUNC DECL
{
int status = 0;
char *f1_X = NULL; size_t f1_nX = 0;
char *lb_X = NULL; size_t lb_nX = 0;
char *ub_X = NULL; size_t ub_nX = 0;
char *cnt_X = NULL; size_t cnt_nX = 0;
int t1_id = INT_MIN, t2_id = INT_MIN;
int f1_id = INT_MIN, lb_id = INT_MIN, ub_id = INT_MIN, cnt_id = INT_MIN;
FLD_TYPE *f1_meta = NULL, *lb_meta = NULL, *ub_meta = NULL;
long long nR1 = INT_MIN, nR2 = INT_MIN, chk_nR1 = INT_MIN;
long long **cntptrs = NULL;
// For multi-threading
int nT;
int rc; // result code for thread create
pthread_t threads[MAX_NUM_THREADS];
pthread_attr_t attr;
void *thread_status;
char str_meta_data[1024];
char *opfile = NULL; FILE *ofp = NULL;
//----------------------------------------------------------------
if ( ( t1 == NULL ) || ( *t1 == '\0' ) ) { go_BYE(-1); }
if ( ( f1 == NULL ) || ( *f1 == '\0' ) ) { go_BYE(-1); }
if ( ( t2 == NULL ) || ( *t2 == '\0' ) ) { go_BYE(-1); }
if ( ( lb == NULL ) || ( *lb == '\0' ) ) { go_BYE(-1); }
if ( ( ub == NULL ) || ( *ub == '\0' ) ) { go_BYE(-1); }
if ( ( cnt == NULL ) || ( *cnt == '\0' ) ) { go_BYE(-1); }
if ( strcmp(t1, t2) == 0 ) { go_BYE(-1); }
zero_string(str_meta_data, 1024);
//--------------------------------------------------------
status = is_tbl(t1, &t1_id); cBYE(status);
chk_range(t1_id, 0, g_n_tbl);
nR1 = g_tbl[t1_id].nR;
//--------------------------------------------------------
status = is_fld(NULL, t1_id, f1, &f1_id); cBYE(status);
chk_range(f1_id, 0, g_n_fld);
f1_meta = &(g_fld[f1_id]);
status = rs_mmap(f1_meta->filename, &f1_X, &f1_nX, 0); cBYE(status);
// Have not implemented case where f1 has null field
if ( f1_meta->nn_fld_id >= 0 ) { go_BYE(-1); }
// Have implemented only for int
if ( strcmp(f1_meta->fldtype, "int") != 0 ) { cBYE(-1); }
//--------------------------------------------------------
status = is_tbl(t2, &t2_id); cBYE(status);
chk_range(t2_id, 0, g_n_tbl);
nR2 = g_tbl[t2_id].nR;
//--------------------------------------------------------
status = is_fld(NULL, t2_id, lb, &lb_id); cBYE(status);
chk_range(lb_id, 0, g_n_fld);
lb_meta = &(g_fld[lb_id]);
status = rs_mmap(lb_meta->filename, &lb_X, &lb_nX, 0); cBYE(status);
// Have not implemented case where lb has null field
if ( lb_meta->nn_fld_id >= 0 ) { go_BYE(-1); }
// Have implemented only for int
if ( strcmp(lb_meta->fldtype, "int") != 0 ) { cBYE(-1); }
//--------------------------------------------------------
status = is_fld(NULL, t2_id, ub, &ub_id); cBYE(status);
chk_range(ub_id, 0, g_n_fld);
ub_meta = &(g_fld[ub_id]);
status = rs_mmap(ub_meta->filename, &ub_X, &ub_nX, 0); cBYE(status);
// Have not implemented case where ub has null field
if ( ub_meta->nn_fld_id >= 0 ) { go_BYE(-1); }
// Have implemented only for int
if ( strcmp(ub_meta->fldtype, "int") != 0 ) { cBYE(-1); }
//--------------------------------------------------------
// Set up access to input
int *inptr = (int *)f1_X;
int *lbptr = (int *)lb_X;
int *ubptr = (int *)ub_X;
//--------------------------------------------------------
//--- Decide on how much parallelism to use
for ( int i = 0; i < MAX_NUM_THREADS; i++ ) {
g_thread_id[i] = i;
g_num_rows[i] = 0;
}
status = get_num_threads(&nT);
cBYE(status);
//--------------------------------------------
#define MIN_ROWS_FOR_SUBSAMPLE 10000 // 1048576
if ( nR1 <= MIN_ROWS_FOR_SUBSAMPLE ) {
nT = 1;
}
/* Don't create more threads than you can use */
if ( nT > nR1 ) { nT = nR1; }
//--------------------------------------------
/* Make space for output */
long long filesz = nR2 * sizeof(long long);
status = open_temp_file(&ofp, &opfile, filesz); cBYE(status);
fclose_if_non_null(ofp);
status = mk_file(opfile, filesz); cBYE(status);
status = rs_mmap(opfile, &cnt_X, &cnt_nX, 1);
long long *cntptr = (long long *)cnt_X;
/* Make a holding tank for partial results */
cntptrs = malloc(nT * sizeof(long long *));
return_if_malloc_failed(cntptrs);
for ( int i = 0; i < nT; i++ ) {
cntptrs[i] = malloc(nR2 * sizeof(long long));
return_if_malloc_failed(cntptrs[i]);
for ( long long j = 0; j <nR2; j++ ) {
cntptrs[i][j] = 0;
}
}
// Add count field to meta data
sprintf(str_meta_data, "fldtype=long long:n_sizeof=8:filename=%s", opfile);
status = add_fld(t2, cnt, str_meta_data, &cnt_id); cBYE(status);
chk_range(cnt_id, 0, g_n_fld);
//-----------------------------------------------------------
// Now we count how much there is in each range
// Set up global variables
g_nT = nT;
g_inptr = inptr;
g_lbptr = lbptr;
g_ubptr = ubptr;
g_cntptrs = cntptrs;
g_nR1 = nR1;
g_nR2 = nR2;
if ( g_nT == 1 ) {
core_num_in_range(&(g_thread_id[0]));
chk_nR1 = g_num_rows[0];
}
else {
chk_nR1 = 0;
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
for ( int t = 0; t < g_nT; t++ ) {
rc = pthread_create(&threads[t], NULL, core_num_in_range,
&(g_thread_id[t]));
if ( rc ) { go_BYE(-1); }
}
/* Free attribute and wait for the other threads */
pthread_attr_destroy(&attr);
for ( int t = 0; t < g_nT; t++ ) {
rc = pthread_join(threads[t], &thread_status);
if ( rc ) { go_BYE(-1); }
chk_nR1 += g_num_rows[t];
}
}
if ( chk_nR1 != nR1 ) { go_BYE(-1); }
// Accumulate partial results
for ( long long i = 0; i < nR2; i++ ) {
cntptr[i] = 0;
for ( int j= 0; j < nT; j++ ) {
cntptr[i] += cntptrs[j][i];
}
}
BYE:
g_write_to_temp_dir = false;
rs_munmap(f1_X, f1_nX);
rs_munmap(lb_X, lb_nX);
rs_munmap(ub_X, ub_nX);
rs_munmap(cnt_X, cnt_nX);
free_if_non_null(opfile);
return(status);
}
/*
Example multipart form-data request content format:
--AaB03x
Content-Disposition: form-data; name="submit-name"
Larry
--AaB03x
Content-Disposition: form-data; name="file"; filename="file1.txt"
Content-Type: text/plain
... contents of file1.txt ...
--AaB03x--
*/
static evhtp_res
upload_read_cb (evhtp_request_t *req, evbuf_t *buf, void *arg)
{
RecvFSM *fsm = arg;
char *line;
size_t len;
gboolean no_line = FALSE;
int res = EVHTP_RES_OK;
if (fsm->state == RECV_ERROR)
return EVHTP_RES_OK;
/* Update upload progress. */
fsm->progress->uploaded += (gint64)evbuffer_get_length(buf);
seaf_debug ("progress: %lld/%lld\n",
fsm->progress->uploaded, fsm->progress->size);
evbuffer_add_buffer (fsm->line, buf);
/* Drain the buffer so that evhtp don't copy it to another buffer
* after this callback returns.
*/
evbuffer_drain (buf, evbuffer_get_length (buf));
while (!no_line) {
switch (fsm->state) {
case RECV_INIT:
line = evbuffer_readln (fsm->line, &len, EVBUFFER_EOL_CRLF_STRICT);
if (line != NULL) {
seaf_debug ("[upload] boundary line: %s.\n", line);
if (!strstr (line, fsm->boundary)) {
seaf_warning ("[upload] no boundary found in the first line.\n");
free (line);
res = EVHTP_RES_BADREQ;
goto out;
} else {
fsm->state = RECV_HEADERS;
free (line);
}
} else {
no_line = TRUE;
}
break;
case RECV_HEADERS:
line = evbuffer_readln (fsm->line, &len, EVBUFFER_EOL_CRLF_STRICT);
if (line != NULL) {
seaf_debug ("[upload] mime header line: %s.\n", line);
if (len == 0) {
/* Read an blank line, headers end. */
free (line);
if (g_strcmp0 (fsm->input_name, "file") == 0) {
if (open_temp_file (fsm) < 0) {
seaf_warning ("[upload] Failed open temp file.\n");
res = EVHTP_RES_SERVERR;
goto out;
}
}
seaf_debug ("[upload] Start to recv %s.\n", fsm->input_name);
fsm->state = RECV_CONTENT;
} else if (parse_mime_header (line, fsm) < 0) {
free (line);
res = EVHTP_RES_BADREQ;
goto out;
} else {
free (line);
}
} else {
no_line = TRUE;
}
break;
case RECV_CONTENT:
if (g_strcmp0 (fsm->input_name, "file") == 0)
res = recv_file_data (fsm, &no_line);
else
res = recv_form_field (fsm, &no_line);
if (res != EVHTP_RES_OK)
goto out;
break;
}
}
out:
if (res != EVHTP_RES_OK) {
/* Don't receive any data before the connection is closed. */
evhtp_request_pause (req);
/* Set keepalive to 0. This will cause evhtp to close the
* connection after sending the reply.
*/
req->keepalive = 0;
fsm->state = RECV_ERROR;
}
if (res == EVHTP_RES_BADREQ) {
evhtp_send_reply (req, EVHTP_RES_BADREQ);
} else if (res == EVHTP_RES_SERVERR) {
evbuffer_add_printf (req->buffer_out, "Internal server error\n");
evhtp_send_reply (req, EVHTP_RES_SERVERR);
}
return EVHTP_RES_OK;
}
//---------------------------------------------------------------
// START FUNC DECL
int
f1f2f3f4opf5(
char *tbl,
char *f1,
char *f2,
char *f3,
char *f4,
char *str_op_spec,
char *f5
)
// STOP FUNC DECL
{
int status = 0;
char *f1_X = NULL; size_t f1_nX = 0;
char *nn_f1_X = NULL; size_t nn_f1_nX = 0;
char *f2_X = NULL; size_t f2_nX = 0;
char *nn_f2_X = NULL; size_t nn_f2_nX = 0;
char *f3_X = NULL; size_t f3_nX = 0;
char *nn_f3_X = NULL; size_t nn_f3_nX = 0;
char *f4_X = NULL; size_t f4_nX = 0;
char *nn_f4_X = NULL; size_t nn_f4_nX = 0;
char *out_X = NULL; size_t out_nX = 0;
char *nn_out_X = NULL; size_t nn_out_nX = 0;
char *opfile = NULL, *nn_opfile = NULL; FILE *ofp = NULL;
FLD_TYPE *f1_meta = NULL, *nn_f1_meta = NULL;
FLD_TYPE *f2_meta = NULL, *nn_f2_meta = NULL;
FLD_TYPE *f3_meta = NULL, *nn_f3_meta = NULL;
FLD_TYPE *f4_meta = NULL, *nn_f4_meta = NULL;
long long nR;
int tbl_id = INT_MIN;
int f1_id = INT_MIN, f2_id = INT_MIN, f3_id = INT_MIN,
f4_id = INT_MIN, f5_id = INT_MIN;
int nn_f1_id = INT_MIN, nn_f2_id = INT_MIN, nn_f3_id = INT_MIN,
nn_f4_id = INT_MIN, nn_f5_id = INT_MIN;
char str_meta_data[1024];
bool is_any_null = false;
float *f1ptr = NULL, *f2ptr = NULL, *f3ptr = NULL, *f4ptr = NULL;
float *foutptr = NULL; char *nn_out = NULL;
char *op = NULL;
//----------------------------------------------------------------
if ( ( tbl == NULL ) || ( *tbl == '\0' ) ) { go_BYE(-1); }
if ( ( f1 == NULL ) || ( *f1 == '\0' ) ) { go_BYE(-1); }
if ( ( f2 == NULL ) || ( *f2 == '\0' ) ) { go_BYE(-1); }
if ( ( f3 == NULL ) || ( *f3 == '\0' ) ) { go_BYE(-1); }
if ( ( f4 == NULL ) || ( *f4 == '\0' ) ) { go_BYE(-1); }
if ( ( f5 == NULL ) || ( *f5 == '\0' ) ) { go_BYE(-1); }
if ( ( str_op_spec == NULL ) || ( *str_op_spec == '\0' ) ) { go_BYE(-1); }
zero_string(str_meta_data, 1024);
//--------------------------------------------------------
status = extract_name_value(str_op_spec, "op=", ":", &op);
cBYE(status);
if ( op == NULL ) { go_BYE(-1); }
//--------------------------------------------------------
status = is_tbl(tbl, &tbl_id); cBYE(status);
chk_range(tbl_id, 0, g_n_tbl);
nR = g_tbl[tbl_id].nR;
//--------------------------------------------------------
/* Get information for field f1 */
status = is_fld(NULL, tbl_id, f1, &f1_id);
f1_meta = &(g_fld[f1_id]);
status = rs_mmap(f1_meta->filename, &f1_X, &f1_nX, 0); cBYE(status);
nn_f1_id = f1_meta->nn_fld_id;
if ( nn_f1_id >= 0 ) {
nn_f1_meta = &(g_fld[nn_f1_id]);
status = rs_mmap(nn_f1_meta->filename, &nn_f1_X, &nn_f1_nX, 0);
cBYE(status);
}
//--------------------------------------------------------
/* Get information for field f2 */
status = is_fld(NULL, tbl_id, f2, &f2_id);
f2_meta = &(g_fld[f2_id]);
status = rs_mmap(f2_meta->filename, &f2_X, &f2_nX, 0); cBYE(status);
nn_f2_id = f2_meta->nn_fld_id;
if ( nn_f2_id >= 0 ) {
nn_f2_meta = &(g_fld[nn_f2_id]);
status = rs_mmap(nn_f2_meta->filename, &nn_f2_X, &nn_f2_nX, 0);
cBYE(status);
}
//--------------------------------------------------------
/* Get information for field f3 */
status = is_fld(NULL, tbl_id, f3, &f3_id);
f3_meta = &(g_fld[f3_id]);
status = rs_mmap(f3_meta->filename, &f3_X, &f3_nX, 0); cBYE(status);
nn_f3_id = f3_meta->nn_fld_id;
if ( nn_f3_id >= 0 ) {
nn_f3_meta = &(g_fld[nn_f3_id]);
status = rs_mmap(nn_f3_meta->filename, &nn_f3_X, &nn_f3_nX, 0);
cBYE(status);
}
//--------------------------------------------------------
/* Get information for field f4 */
status = is_fld(NULL, tbl_id, f4, &f4_id);
f4_meta = &(g_fld[f4_id]);
status = rs_mmap(f4_meta->filename, &f4_X, &f4_nX, 0); cBYE(status);
nn_f4_id = f4_meta->nn_fld_id;
if ( nn_f4_id >= 0 ) {
nn_f4_meta = &(g_fld[nn_f4_id]);
status = rs_mmap(nn_f4_meta->filename, &nn_f4_X, &nn_f4_nX, 0);
cBYE(status);
}
//--------------------------------------------------------
if ( strcmp(f1_meta->fldtype, "float") != 0 ) { go_BYE(-1); }
if ( strcmp(f2_meta->fldtype, "float") != 0 ) { go_BYE(-1); }
if ( strcmp(f3_meta->fldtype, "float") != 0 ) { go_BYE(-1); }
if ( strcmp(f4_meta->fldtype, "float") != 0 ) { go_BYE(-1); }
//--------------------------------------------------------
f1ptr = (float *)f1_X;
f2ptr = (float *)f2_X;
f3ptr = (float *)f3_X;
f4ptr = (float *)f4_X;
//--------------------------------------------------------
// Create output file
long long filesz = sizeof(float) * nR;
status = open_temp_file(&ofp, &opfile, filesz); cBYE(status); fclose_if_non_null(ofp);
status = mk_file(opfile, filesz); cBYE(status);
status = rs_mmap(opfile, &out_X, &out_nX, 1); cBYE(status);
foutptr = (float*)out_X;
//--------------------------------------------------------
// Create nn field for output file
long long nnfilesz = sizeof(char) * nR;
status = open_temp_file(&ofp, &nn_opfile, nnfilesz); cBYE(status);
fclose_if_non_null(ofp);
status = mk_file(nn_opfile, filesz); cBYE(status);
status = rs_mmap(nn_opfile, &nn_out_X, &nn_out_nX, 1); cBYE(status);
nn_out = (char*)nn_out_X;
//--------------------------------------------------------
if ( strcmp(op, "lat_long_dist") == 0 ) {
for ( long long i = 0; i < nR; i++ ) {
if ( ( nn_f1_X != NULL ) && ( nn_f1_X[i] == 0 ) ) {
is_any_null = true; foutptr[i] = 0; nn_out[i] = 0; continue;
}
if ( ( nn_f2_X != NULL ) && ( nn_f2_X[i] == 0 ) ) {
is_any_null = true; foutptr[i] = 0; nn_out[i] = 0; continue;
}
if ( ( nn_f3_X != NULL ) && ( nn_f3_X[i] == 0 ) ) {
is_any_null = true; foutptr[i] = 0; nn_out[i] = 0; continue;
}
if ( ( nn_f4_X != NULL ) && ( nn_f4_X[i] == 0 ) ) {
is_any_null = true; foutptr[i] = 0; nn_out[i] = 0; continue;
}
if ( ( f1ptr[i] < -90 ) || ( f1ptr[i] > 90 ) ) { go_BYE(-1); }
if ( ( f2ptr[i] < -180 ) || ( f2ptr[i] > 180 ) ) { go_BYE(-1); }
if ( ( f3ptr[i] < -90 ) || ( f3ptr[i] > 90 ) ) { go_BYE(-1); }
if ( ( f4ptr[i] < -180 ) || ( f4ptr[i] > 180 ) ) { go_BYE(-1); }
status = latlong_distance(f1ptr[i], f2ptr[i], f3ptr[i], f4ptr[i],
&(foutptr[i]));
cBYE(status);
nn_out[i] = TRUE;
}
}
sprintf(str_meta_data, "filename=%s:fldtype=float:n_sizeof=4", opfile);
status = add_fld(tbl, f5, str_meta_data, &f5_id);
cBYE(status);
if ( is_any_null ) {
status = add_aux_fld(tbl, f5, nn_opfile, "nn", &nn_f5_id);
cBYE(status);
}
else {
unlink(nn_opfile);
free_if_non_null(nn_opfile);
}
BYE:
rs_munmap(f1_X, f1_nX); rs_munmap(nn_f1_X, nn_f1_nX);
rs_munmap(f2_X, f2_nX); rs_munmap(nn_f2_X, nn_f2_nX);
rs_munmap(f3_X, f3_nX); rs_munmap(nn_f3_X, nn_f3_nX);
rs_munmap(f4_X, f4_nX); rs_munmap(nn_f4_X, nn_f4_nX);
rs_munmap(out_X, out_nX); rs_munmap(nn_out_X, nn_out_nX);
free_if_non_null(opfile);
free_if_non_null(nn_opfile);
free_if_non_null(op);
return(status);
}
int
collect_lnk_idx_vals_int(
char *src_lnk_X,
char *nn_src_lnk_X,
char *nn_src_fld_X,
int src_nR,
char *dst_lnk_X,
char *nn_dst_lnk_X,
int dst_nR,
INT_INT_TYPE **ptr_src_pair,
int *ptr_nn_src_nR,
INT_INT_TYPE **ptr_dst_pair,
int *ptr_nn_dst_nR,
char **ptr_temp_src_X,
size_t *ptr_temp_src_nX,
char **ptr_temp_dst_X,
size_t *ptr_temp_dst_nX
)
{
int status = 0;
int nn_src_nR, nn_dst_nR;
INT_INT_TYPE *src_pair = NULL, *dst_pair = NULL;
int rec_size = sizeof(int) + sizeof(int);
char *temp_src_file = NULL, *temp_dst_file = NULL;
FILE *tfp = NULL;
int *ll_dst_lnk_ptr = NULL, *ll_src_lnk_ptr = NULL;
char *temp_src_X = NULL; size_t temp_src_nX = 0;
char *temp_dst_X = NULL; size_t temp_dst_nX = 0;
/* START: Collect nn source values and sort them on link field */
status = open_temp_file(&tfp, &temp_src_file, 0);
cBYE(status);
nn_src_nR = 0;
ll_src_lnk_ptr = (int*)src_lnk_X;
for ( int i = 0; i < src_nR; i++ ) {
if ( ( nn_src_fld_X != NULL ) && ( nn_src_fld_X[i] == FALSE ) ) { continue; }
if ( ( nn_src_lnk_X != NULL ) && ( nn_src_lnk_X[i] == FALSE ) ) { continue; }
nn_src_nR++;
fwrite(ll_src_lnk_ptr+i, 1, sizeof(int), tfp);
fwrite(&i, 1, sizeof(int), tfp);
}
fclose_if_non_null(tfp);
if ( nn_src_nR == 0 ) { /* No values to process */
go_BYE(-1);
}
status = rs_mmap(temp_src_file, &temp_src_X, &temp_src_nX, 1);
cBYE(status);
if ( ( ( temp_src_nX/ sizeof(INT_INT_TYPE) ) * sizeof(INT_INT_TYPE) ) != temp_src_nX) {
go_BYE(-1);
}
qsort(temp_src_X, nn_src_nR, rec_size, sort2_asc_int);
src_pair = (INT_INT_TYPE *)temp_src_X;
/* STOP: Collect nn source values and sort them on link field */
/*------------------------------------------------*/
/* START: Sort destination link fields */
status = open_temp_file(&tfp, &temp_dst_file, 0);
cBYE(status);
nn_dst_nR = 0;
ll_dst_lnk_ptr = (int *)dst_lnk_X;
for ( int i = 0; i < dst_nR; i++ ) {
if ( ( nn_dst_lnk_X != NULL ) && ( nn_dst_lnk_X[i] == FALSE ) ) { continue; }
nn_dst_nR++;
fwrite(ll_dst_lnk_ptr+i, 1, sizeof(int), tfp);
fwrite(&i, 1, sizeof(int), tfp);
}
fclose_if_non_null(tfp);
if ( nn_dst_nR == 0 ) { /* No values to process */
go_BYE(-1);
}
status = rs_mmap(temp_dst_file, &temp_dst_X, &temp_dst_nX, 1);
cBYE(status);
if ( ( ( temp_dst_nX/ sizeof(INT_INT_TYPE) ) * sizeof(INT_INT_TYPE) ) != temp_dst_nX) {
go_BYE(-1);
}
qsort(temp_dst_X, nn_dst_nR, rec_size, sort2_asc_int);
dst_pair = (INT_INT_TYPE *)temp_dst_X;
/*------------------------------------------------*/
/* STOP : Sort destination link fields */
//--------------------------------------------------------
*ptr_nn_src_nR = nn_src_nR;
*ptr_nn_dst_nR = nn_dst_nR;
*ptr_src_pair = src_pair;
*ptr_dst_pair = dst_pair;
*ptr_temp_src_X = temp_src_X;
*ptr_temp_src_nX = temp_src_nX;
*ptr_temp_dst_X = temp_dst_X;
*ptr_temp_dst_nX = temp_dst_nX;
BYE:
unlink(temp_src_file);
unlink(temp_dst_file);
free_if_non_null(temp_src_file);
free_if_non_null(temp_dst_file);
return(status);
}