bool test(long int f0) {
double s0 = f_to_s(f0);
double s1 = s0 + (1.0 / SRATE);
long int f1 = f0 + 1;
long int fh = s_to_f(s1);
printf("%20ld %20ld %20f %20f\n", f1, fh, s0, s1);
return f1 == fh;
}
static VALUE
nurat_to_s(VALUE self)
{
get_dat1(self);
if (f_one_p(dat->den))
return f_to_s(dat->num);
else
return rb_funcall(rb_mKernel, id_format, 3,
rb_str_new2("%d/%d"), dat->num, dat->den);
}
/*
* call-seq:
* cmp.to_r -> rational
*
* Returns the value as a rational if possible.
*/
static VALUE
nucomp_to_r(VALUE self)
{
get_dat1(self);
if (k_inexact_p(dat->imag) || f_nonzero_p(dat->imag)) {
VALUE s = f_to_s(self);
rb_raise(rb_eRangeError, "can't convert %s into Rational",
StringValuePtr(s));
}
return f_to_r(dat->real);
}
static int get_data(
char *docroot,
sqlite3 *db,
char *tbl,
char *fld,
long long *ptr_nR,
FLD_META_TYPE *ptr_fld_meta,
char **ptr_X,
size_t *ptr_nX,
FLD_META_TYPE *ptr_nn_fld_meta,
char **ptr_nn_X,
size_t *ptr_nn_nX,
long long *ptr_nn_nR
)
{
int status = 0;
int tbl_id, fld_id;
bool exists;
char *X = NULL; size_t nX = 0;
char *nn_X = NULL; size_t nn_nX = 0;
char *endptr = NULL;
char buffer[32];
zero_string(buffer, 32);
status = is_tbl_fld(docroot, db, tbl, fld, &tbl_id, &fld_id, &exists);
cBYE(status);
if ( !exists ) { go_BYE(-1); }
status = internal_get_nR(db, tbl_id, ptr_nR);
status = fld_meta(docroot, db, tbl, fld, -1, ptr_fld_meta);
cBYE(status);
status = rs_mmap(ptr_fld_meta->filename, &X, &nX, 0);
cBYE(status);
status = get_aux_field_if_it_exists(docroot, db, tbl, fld_id,
"nn", ptr_nn_fld_meta, &nn_X, &nn_nX);
if ( nn_X == NULL ) {
*ptr_nn_nR = *ptr_nR;
}
else {
status = f_to_s(docroot, db, tbl, ptr_nn_fld_meta->name, "sum", buffer);
cBYE(status);
*ptr_nn_nR = strtol(buffer, &endptr, 10);
if ( *endptr != '\0' ) { go_BYE(-1); }
if ( *ptr_nn_nR <= 0 ) { go_BYE(-1); }
}
*ptr_X = X; *ptr_nX = nX;
*ptr_nn_X = nn_X; *ptr_nn_nX = nn_nX;
BYE:
return(status);
}
//---------------------------------------------------------------
// START FUNC DECL
int
count_strings(
char *src_tbl,
char *src_fld,
char *str_trunc_len,
// truncate strings beyond this len
char *dst_tbl
)
// STOP FUNC DECL
{
int status = 0;
char *X = NULL; size_t nX = 0;
char *src_fld_X = NULL; size_t src_fld_nX = 0;
char *nn_src_fld_X = NULL; size_t nn_src_fld_nX = 0;
char *sz_src_fld_X = NULL; size_t sz_src_fld_nX = 0;
FLD_TYPE *src_fld_meta = NULL;
FLD_TYPE *nn_src_fld_meta = NULL;
FLD_TYPE *sz_src_fld_meta = NULL;
long long src_nR;
int src_tbl_id = INT_MIN, src_fld_id = INT_MIN;
int dst_tbl_id = INT_MIN;
int nn_src_fld_id = INT_MIN, sz_src_fld_id = INT_MIN;
int itemp;
char str_dst_nR[32]; long long dst_nR = 0;
char str_meta_data[4096];
char *out_X = NULL; char *cnt_X = NULL;
char *dst_fk_X = NULL; size_t dst_fk_nX = 0;
FILE *ofp = NULL; char *opfile = NULL;
FILE *tmp_ofp = NULL; char *tmp_opfile = NULL;
FILE *nn_ofp = NULL; char *nn_opfile = NULL;
FILE *sz_ofp = NULL; char *sz_opfile = NULL;
FILE *cnt_ofp = NULL; char *cnt_opfile = NULL;
FILE *from_dst_ofp = NULL; char *from_dst_opfile = NULL;
FILE *dst_fk_ofp = NULL; char *dst_fk_opfile = NULL; int *idst_fk = NULL;
FILE *to_dst_ofp = NULL; char *to_dst_opfile = NULL;
size_t out_nX = 0, cnt_nX = 0;
int *szptr = NULL; char *nnptr = NULL;
char *cptr = NULL, *prev_cptr = NULL;
char *sz_src_fld = NULL; char *str_maxlen = NULL;
char *from_dst_fld = NULL, *to_dst_fld = NULL, *dst_fk_fld = NULL;
char *null_str = NULL, *endptr = NULL;
int maxlen, rec_size, trunc_len = -1; // no truncation by default
long long cnt = 1; int icnt = 1;
bool is_any_null = false;
char is_nn; int sz;
//----------------------------------------------------------------
if ( str_trunc_len != NULL ) {
trunc_len = strtol(str_trunc_len, &endptr, 10);
if ( *endptr != '\0' ) { go_BYE(-1); }
if ( trunc_len <= 1 ) { go_BYE(-1); }
trunc_len++; // to include null character
}
if ( ( src_tbl == NULL ) || ( *src_tbl == '\0' ) ) { go_BYE(-1); }
if ( ( src_fld == NULL ) || ( *src_fld == '\0' ) ) { go_BYE(-1); }
if ( ( dst_tbl == NULL ) || ( *dst_tbl == '\0' ) ) { go_BYE(-1); }
zero_string(str_meta_data, 4096);
zero_string(str_dst_nR, 32);
//--------------------------------------------------------
status = is_tbl(src_tbl, &src_tbl_id); cBYE(status);
chk_range(src_tbl_id, 0, g_n_tbl);
status = is_fld(NULL, src_tbl_id, src_fld, &src_fld_id); cBYE(status);
chk_range(src_fld_id, 0, g_n_fld);
src_nR = g_tbl[src_tbl_id].nR;
if ( src_nR > INT_MAX ) {
fprintf(stderr, "NOT IMPLEMENTED\n"); go_BYE(-1);
}
//--------------------------------------------------------
src_fld_meta = &(g_fld[src_fld_id]);
status = rs_mmap(src_fld_meta->filename, &src_fld_X, &src_fld_nX, 0);
cBYE(status);
//--------------------------------------------------------
if ( strcmp(src_fld_meta->fldtype, "char string") != 0 ) { go_BYE(-1); }
cptr = src_fld_X;
// Get nn field for source if if it exists
nn_src_fld_id = g_fld[src_fld_id].nn_fld_id;
if ( nn_src_fld_id >= 0 ) {
nn_src_fld_meta = &(g_fld[nn_src_fld_id]);
status = rs_mmap(nn_src_fld_meta->filename, &nn_src_fld_X, &nn_src_fld_nX, 0);
cBYE(status);
}
nnptr = (char *)nn_src_fld_X;
// Get sz field for source
sz_src_fld_id = g_fld[src_fld_id].sz_fld_id;
chk_range(sz_src_fld_id, 0, g_n_fld);
sz_src_fld_meta = &(g_fld[sz_src_fld_id]);
status = rs_mmap(sz_src_fld_meta->filename, &sz_src_fld_X, &sz_src_fld_nX, 0);
cBYE(status);
szptr = (int *)sz_src_fld_X;
//------------------------------------
mcr_alloc_null_str(from_dst_fld, (MAX_LEN_FILE_NAME+16));
mcr_alloc_null_str(to_dst_fld, (MAX_LEN_FILE_NAME+16));
mcr_alloc_null_str(dst_fk_fld, (MAX_LEN_FILE_NAME+16));
mcr_alloc_null_str(sz_src_fld, (MAX_LEN_FILE_NAME+16));
mcr_alloc_null_str(str_maxlen, 32);
strcpy(sz_src_fld, "_sz_"); strcat(sz_src_fld, src_fld);
strcpy(from_dst_fld, src_tbl); strcat(from_dst_fld, "_from");
strcpy(to_dst_fld, dst_tbl); strcat(to_dst_fld, "_to");
strcpy(dst_fk_fld, dst_tbl); strcat(dst_fk_fld, "_id");
// Get max length of src field
status = f_to_s(src_tbl, sz_src_fld, "max", str_maxlen);
cBYE(status);
maxlen = strtol(str_maxlen, &endptr, 10);
if ( *endptr != '\0' ) { go_BYE(-1); }
if ( maxlen <= 0 ) { go_BYE(-1); }
if ( trunc_len < 0 ) {
trunc_len = maxlen;
}
mcr_alloc_null_str(null_str, trunc_len);
//--------------------------------------------------------
// Write out src field as constant length field
status = open_temp_file(&tmp_ofp, &tmp_opfile, 0);
cBYE(status);
for ( int i = 0; i < src_nR; i++ ) {
int sz, fill_sz;
sz = szptr[i]; // includes terminating null character
if ( sz <= trunc_len ) {
fwrite(cptr, sizeof(char), sz, tmp_ofp);
fill_sz = trunc_len - sz;
fwrite(null_str, sizeof(char), fill_sz, tmp_ofp);
}
else {
fwrite(cptr, sizeof(char), trunc_len-1, tmp_ofp);
fwrite(null_str, sizeof(char), 1, tmp_ofp);
}
// Append origin of this string
fwrite(&i, sizeof(int), 1, tmp_ofp);
cptr += sz;
// TODO: Check about nullc and sz
}
fclose_if_non_null(tmp_ofp);
//---------------------------------------------------------
// Sort output file so that similar titles occur together
status = rs_mmap(tmp_opfile, &X, &nX, 1); // 1 for writing
cBYE(status);
// Note that we added the origin of the string. Hence the size of a
// record is trunc_len + sizeof(int)
rec_size = trunc_len + sizeof(int);
qsort(X, src_nR, rec_size, str_compare);
rs_munmap(X, nX);
//--------------------------------------------------------
// Now we need to compact the sorted output so that it is not padded
// Open 4 files for
// output string, sz(output string), nn(output string), cnt
status = open_temp_file(&ofp, &opfile, 0); cBYE(status);
status = open_temp_file(&nn_ofp, &nn_opfile, 0); cBYE(status);
status = open_temp_file(&sz_ofp, &sz_opfile, 0); cBYE(status);
status = open_temp_file(&cnt_ofp, &cnt_opfile, 0); cBYE(status);
status = open_temp_file(&from_dst_ofp, &from_dst_opfile, 0); cBYE(status);
status = open_temp_file(&to_dst_ofp, &to_dst_opfile, 0); cBYE(status);
status = open_temp_file(&dst_fk_ofp, &dst_fk_opfile, 0); cBYE(status);
fclose_if_non_null(dst_fk_ofp);
mk_file(dst_fk_opfile, src_nR * sizeof(int)); cBYE(status);
rs_mmap(dst_fk_opfile, &dst_fk_X, &dst_fk_nX, 1); cBYE(status);
idst_fk = (int *)dst_fk_X;
status = rs_mmap(tmp_opfile, &X, &nX, 0);
cBYE(status);
cptr = X;
prev_cptr = cptr;
cptr += rec_size;
cnt = icnt = 1;
for ( int i = 0; i < src_nR; i++ ) {
char *xptr; int *iptr;
xptr = prev_cptr + trunc_len;
iptr = (int *)xptr;
if ( ( *iptr < 0 ) || ( *iptr >= src_nR ) ) { go_BYE(-1); }
fwrite(iptr, sizeof(int), 1, from_dst_ofp);
fwrite(&dst_nR, sizeof(int), 1, to_dst_ofp);
idst_fk[*iptr] = dst_nR;
if ( strcmp(cptr, prev_cptr) != 0 ) {
sz = strlen(prev_cptr);
if ( sz == 0 ) { is_nn = FALSE; } else { is_nn = TRUE; }
sz++; // for null character termination
fwrite(prev_cptr, sizeof(char), sz, ofp);
fwrite(&sz, sizeof(int), 1, sz_ofp);
fwrite(&is_nn, sizeof(char), 1, nn_ofp);
if ( cnt > INT_MAX ) { go_BYE(-1); }
icnt = cnt;
fwrite(&icnt, sizeof(int), 1, cnt_ofp);
cnt = 1;
dst_nR++;
}
else {
cnt++;
}
prev_cptr = cptr;
cptr += rec_size;
}
// START: Now for the last guy
cnt--; // this is needed
sz = strlen(prev_cptr);
if ( sz == 0 ) { is_nn = FALSE; } else { is_nn = TRUE; }
if ( is_nn == FALSE ) {
is_any_null = true;
}
sz++; // for null character termination
fwrite(prev_cptr, sizeof(char), sz, ofp);
fwrite(&sz, sizeof(int), 1, sz_ofp);
fwrite(&is_nn, sizeof(char), 1, nn_ofp);
if ( cnt > INT_MAX ) { go_BYE(-1); }
icnt = cnt;
fwrite(&icnt, sizeof(int), 1, cnt_ofp);
dst_nR++;
// STOP: Now for the last guy
//---------------------------
fclose_if_non_null(ofp);
fclose_if_non_null(nn_ofp);
fclose_if_non_null(sz_ofp);
fclose_if_non_null(cnt_ofp);
fclose_if_non_null(from_dst_ofp);
fclose_if_non_null(to_dst_ofp);
//-----------------------------------------------
// Delete temp file
unlink(tmp_opfile);
//-----------------------------------------------------------
status = is_tbl(dst_tbl, &dst_tbl_id); cBYE(status);
if ( dst_tbl_id >= 0 ) { /* Delete table */
status = del_tbl(NULL, dst_tbl_id);
}
sprintf(str_dst_nR, "%lld", dst_nR);
status = add_tbl(dst_tbl, str_dst_nR, &dst_tbl_id);
cBYE(status);
//-----------------------------------------------------------
/* TODO I think we can dispense with from_dst_fld and to_dst_fld */
sprintf(str_meta_data, "filename=%s:fldtype=int:n_sizeof=4", from_dst_opfile);
status = add_fld(src_tbl, from_dst_fld, str_meta_data, &itemp);
cBYE(status);
sprintf(str_meta_data, "filename=%s:fldtype=int:n_sizeof=4", to_dst_opfile);
status = add_fld(src_tbl, to_dst_fld, str_meta_data, &itemp);
cBYE(status);
sprintf(str_meta_data, "filename=%s:fldtype=int:n_sizeof=4", dst_fk_opfile);
status = add_fld(src_tbl, dst_fk_fld, str_meta_data, &itemp);
cBYE(status);
// Add output field to meta data
sprintf(str_meta_data, "filename=%s:fldtype=char string:n_sizeof=0", opfile);
status = add_fld(dst_tbl, src_fld, str_meta_data, &itemp);
cBYE(status);
fclose_if_non_null(cnt_ofp);
// Add nn field if needed
if ( is_any_null ) {
status = add_aux_fld(dst_tbl, src_fld, nn_opfile, "nn", &itemp);
cBYE(status);
}
else {
unlink(nn_opfile);
}
// Add sz field
status = add_aux_fld(dst_tbl, src_fld, sz_opfile, "sz", &itemp);
cBYE(status);
// Add count field to meta data
sprintf(str_meta_data, "filename=%s:fldtype=int:n_sizeof=4",
cnt_opfile);
status = add_fld(dst_tbl, "cnt", str_meta_data, &itemp);
cBYE(status);
//-----------------------------------------------------------
BYE:
free_if_non_null(null_str);
rs_munmap(X, nX);
rs_munmap(src_fld_X, src_fld_nX);
rs_munmap(cnt_X, cnt_nX);
rs_munmap(out_X, out_nX);
rs_munmap(dst_fk_X, dst_fk_nX);
rs_munmap(nn_src_fld_X, nn_src_fld_nX);
fclose_if_non_null(ofp); free_if_non_null(opfile);
fclose_if_non_null(nn_ofp); free_if_non_null(nn_opfile);
fclose_if_non_null(sz_ofp); free_if_non_null(sz_opfile);
fclose_if_non_null(cnt_ofp); free_if_non_null(cnt_opfile);
fclose_if_non_null(tmp_ofp); free_if_non_null(tmp_opfile);
fclose_if_non_null(from_dst_ofp); free_if_non_null(from_dst_opfile);
fclose_if_non_null(to_dst_ofp); free_if_non_null(to_dst_opfile);
free_if_non_null(sz_src_fld);
free_if_non_null(dst_fk_fld);
free_if_non_null(str_maxlen);
return(status);
}
static int get_data(
char *tbl,
char *fld,
long long *ptr_nR,
char **ptr_X,
size_t *ptr_nX,
char **ptr_opfile,
int *ptr_fldtype,
int *ptr_fldsz
)
{
int status = 0;
int tbl_id = INT_MIN, fld_id = INT_MIN, nn_fld_id = INT_MIN;
char *X = NULL; size_t nX = 0;
char *nn_X = NULL; size_t nn_nX = 0;
char *op_X = NULL; size_t op_nX = 0;
char *endptr = NULL;
char buffer[32];
long long nR = 0;
FLD_TYPE *fld_meta = NULL, *nn_fld_meta = NULL;
FILE *ofp = NULL; char *opfile = NULL;
zero_string(buffer, 32);
status = is_tbl(tbl, &tbl_id); cBYE(status);
chk_range(tbl_id, 0, g_n_tbl);
status = is_fld(NULL, tbl_id, fld, &fld_id); cBYE(status);
chk_range(fld_id, 0, g_n_fld);
nR = g_tbl[tbl_id].nR;
fld_meta = &(g_fld[fld_id]);
nn_fld_id = g_fld[fld_id].nn_fld_id;
if ( nn_fld_id >= 0 ) {
nn_fld_meta = &(g_fld[nn_fld_id]);
status = rs_mmap(nn_fld_meta->filename, &nn_X, &nn_nX, 0);
cBYE(status);
}
*ptr_fldsz = fld_meta->n_sizeof;
status = mk_ifldtype(fld_meta->fldtype, ptr_fldtype); cBYE(status);
switch ( *ptr_fldtype ) {
case FLDTYPE_INT :
case FLDTYPE_LONGLONG :
/* all is well */
break;
default :
go_BYE(-1);
}
if ( nn_X == NULL ) { /* no nn field */
status = rs_mmap(fld_meta->filename, &X, &nX, 0);
cBYE(status);
*ptr_nR = nR;
*ptr_X = X;
*ptr_nX = nX;
*ptr_opfile = NULL;
}
else {
long long nn_nR = 0;
status = f_to_s(tbl, nn_fld_meta->name, "sum", buffer);
cBYE(status);
nn_nR = strtol(buffer, &endptr, 10);
if ( *endptr != '\0' ) { go_BYE(-1); }
if ( ( nn_nR <= 0 ) || ( nn_nR >= nR ) ) { go_BYE(-1); }
/* Now make a copy of just the non-null values */
status = rs_mmap(fld_meta->filename, &X, &nX, 0);
cBYE(status);
status = rs_mmap(nn_fld_meta->filename, &nn_X, &nn_nX, 0);
cBYE(status);
long long filesz = nn_nR * fld_meta->n_sizeof;
status = open_temp_file(&ofp, &opfile, filesz); cBYE(status);
fclose_if_non_null(ofp);
status = mk_file(opfile, filesz); cBYE(status);
status = rs_mmap(opfile, &op_X, &op_nX, 1); cBYE(status);
status = copy_nn_vals(X, nR, nn_X, op_X, nn_nR, fld_meta->n_sizeof);
cBYE(status);
/* Cleanup and return values */
rs_munmap(X, nX);
rs_munmap(nn_X, nn_nX);
*ptr_nR = nn_nR;
*ptr_X = op_X;
*ptr_nX = op_nX;
*ptr_opfile = opfile;
}
BYE:
return(status);
}
//---------------------------------------------------------------
// START FUNC DECL
int
parsort1(
char *tbl,
char *f1,
char *f2,
char *up_or_down /* not used right now */
)
// STOP FUNC DECL
{
int status = 0;
char *f1_X = NULL; size_t f1_nX = 0;
char *op_X = NULL; size_t op_nX = 0;
char *cnt_X = NULL; size_t cnt_nX = 0;
char *t2f2_X = NULL; size_t t2f2_nX = 0;
FLD_TYPE *f1_meta = NULL;
FLD_TYPE *f2_meta = NULL;
FLD_TYPE *t2f2_meta = NULL;
FLD_TYPE *cnt_meta = NULL;
long long nR, nR2;
int tbl_id = INT_MIN, f1_id = INT_MIN, f2_id = INT_MIN, cnt_id = INT_MIN;
int t2f2_id = INT_MIN;
char str_meta_data[1024];
FILE *ofp = NULL; char *opfile = NULL;
FILE *tfp = NULL; char *tempfile = NULL;
char str_rslt[32]; zero_string(str_rslt, 32);
char t2[MAX_LEN_TBL_NAME];
int itemp;
int *xxx = NULL, *f1lb = NULL, *f1ub = NULL;
long long *count = NULL, *chk_count = NULL;
int **offsets = NULL, **bak_offsets = NULL;
int *inptr = NULL;
// For multi-threading
int rc; // result code for thread create
pthread_t threads[MAX_NUM_THREADS];
pthread_attr_t attr;
void *thread_status;
// START: For timing
struct timeval Tps;
struct timezone Tpf;
void *Tzp = NULL;
long long t_before_sec = 0, t_before_usec = 0, t_before = 0;
long long t_after_sec, t_after_usec, t_after;
long long t_delta_usec;
// STOP : For timing
//----------------------------------------------------------------
if ( ( tbl == NULL ) || ( *tbl == '\0' ) ) { go_BYE(-1); }
if ( ( f1 == NULL ) || ( *f1 == '\0' ) ) { go_BYE(-1); }
if ( ( f2 == NULL ) || ( *f2 == '\0' ) ) { go_BYE(-1); }
zero_string(str_meta_data, 1024);
/* t2 isa temporary table */
zero_string(t2, MAX_LEN_TBL_NAME);
status = qd_uq_str(t2, MAX_LEN_TBL_NAME);
strcpy(t2, "t2"); // TODO DELETE THIS
g_offsets = NULL;
g_count = NULL;
//--------------------------------------------------------
status = is_tbl(tbl, &tbl_id); cBYE(status);
chk_range(tbl_id, 0, g_n_tbl);
nR = g_tbl[tbl_id].nR;
status = is_fld(NULL, tbl_id, f1, &f1_id); cBYE(status);
chk_range(f1_id, 0, g_n_fld);
f1_meta = &(g_fld[f1_id]);
status = rs_mmap(f1_meta->filename, &f1_X, &f1_nX, 0);
cBYE(status);
// Not implemented for following cases
if ( g_fld[f1_id].nn_fld_id >= 0 ) { go_BYE(-1); }
if ( strcmp(f1_meta->fldtype, "int") != 0 ) { go_BYE(-1); }
if ( nR <= 1048576 ) { go_BYE(-1); }
//---------------------------------------------
status = gettimeofday(&Tps, &Tpf); cBYE(status);
t_before_sec = (long long)Tps.tv_sec;
t_before_usec = (long long)Tps.tv_usec;
t_before = t_before_sec * 1000000 + t_before_usec;
int reduction_factor = (int)(sqrt((double)nR));
sprintf(str_rslt, "%d", reduction_factor);
status = subsample(tbl, f1, str_rslt, t2, "f2"); cBYE(status);
status = gettimeofday(&Tps, &Tpf); cBYE(status);
t_after_sec = (long long)Tps.tv_sec;
t_after_usec = (long long)Tps.tv_usec;
t_after = t_after_sec * 1000000 + t_after_usec;
fprintf(stderr, "TIME0 = %lld \n", t_after - t_before);
t_before = t_after;
// Must have sufficient diversity of values
status = f1opf2(t2, "f2", "op=shift:val=-1", "nextf2"); cBYE(status);
status = drop_nn_fld(t2, "nextf2"); cBYE(status);
status = f1f2opf3(t2, "f2", "nextf2", "==", "x"); cBYE(status);
status = f_to_s(t2, "x", "sum", str_rslt);
char *endptr;
long long lltemp = strtoll(str_rslt, &endptr, 10);
if ( lltemp != 0 ) { go_BYE(-1); }
//-------------------------------------------------
// Get range of values of f1
status = f_to_s(tbl, f1, "max", str_rslt);
int f1max = strtoll(str_rslt, &endptr, 10);
status = f_to_s(tbl, f1, "min", str_rslt);
int f1min = strtoll(str_rslt, &endptr, 10);
//-------------------------------------------------
// Now we sort the values that we sampled
status = fop(t2, "f2", "sortA"); cBYE(status);
// status = pr_fld(t2, "f2", "", stdout);
status = get_nR(t2, &nR2);
// Now each thread selects a range to work on
int nT;
for ( int i = 0; i < MAX_NUM_THREADS; i++ ) {
g_thread_id[i] = i;
}
status = get_num_threads(&nT);
cBYE(status);
//--------------------------------------------
#define MIN_ROWS_FOR_PARSORT1 1048576
if ( nR <= MIN_ROWS_FOR_PARSORT1 ) {
nT = 1;
}
/* Don't create more threads than you can use */
if ( nT > nR ) { nT = nR; }
//--------------------------------------------
double block_size = (double)nR2 / (double)nT;
status = is_fld(t2, -1, "f2", &t2f2_id); cBYE(status);
chk_range(t2f2_id, 0, g_n_fld);
t2f2_meta = &(g_fld[t2f2_id]);
status = rs_mmap(t2f2_meta->filename, &t2f2_X, &t2f2_nX, 0);
cBYE(status);
int *iptr = (int *)t2f2_X;
xxx = malloc(nT * sizeof(int)); return_if_malloc_failed(xxx);
f1lb = malloc(nT * sizeof(int)); return_if_malloc_failed(f1lb);
f1ub = malloc(nT * sizeof(int)); return_if_malloc_failed(f1ub);
/* FOR OLD_WAY
count = malloc(nT * sizeof(long long)); return_if_malloc_failed(count);
*/
chk_count = malloc(nT * sizeof(long long));
return_if_malloc_failed(chk_count);
g_count = malloc(nT * sizeof(long long)); return_if_malloc_failed(g_count);
for ( int i = 0; i < nT; i++ ) {
// FOR OLD_WAY count[i]= 0;
chk_count[i]= 0;
int j = i+1;
long long idx = j * block_size;
if ( idx >= nR2 ) { idx = nR2 -1 ; }
int y = iptr[idx];
xxx[i] = y;
// fprintf(stdout,"idx = %lld: j = %d: y = %d \n", idx, j, y);
}
for ( int i = 0; i < nT; i++ ) {
if ( ( i == 0 ) && ( i == (nT - 1 ) ) ) {
f1lb[i] = f1min;
f1ub[i] = f1max;
}
else if ( i == 0 ) {
f1lb[i] = f1min;
f1ub[i] = xxx[i];
}
else if ( i == (nT -1 ) ) {
f1lb[i] = xxx[i-1] + 1;
f1ub[i] = f1max;
}
else {
f1lb[i] = xxx[i-1] + 1;
f1ub[i] = xxx[i];
}
}
// STOP: Each thread has now a range to work on
// Create a temporary table t3 to store ranges
char t3[MAX_LEN_TBL_NAME]; int t3_id;
zero_string(t3, MAX_LEN_TBL_NAME);
status = qd_uq_str(t3, MAX_LEN_TBL_NAME);
strcpy(t3, "t3"); // TODO DELETE THIS
sprintf(str_rslt, "%d", nT);
status = add_tbl(t3, str_rslt, &t3_id);
// Add lower bound to t3
status = open_temp_file(&tfp, &tempfile, -1); cBYE(status);
fclose_if_non_null(tfp);
tfp = fopen(tempfile, "wb"); return_if_fopen_failed(tfp, tempfile, "wb");
fwrite(f1lb, sizeof(int), nT, tfp);
fclose_if_non_null(tfp);
sprintf(str_meta_data, "fldtype=%s:n_sizeof=%d:filename=%s",
f1_meta->fldtype, f1_meta->n_sizeof, tempfile);
status = add_fld(t3, "lb", str_meta_data, &itemp); cBYE(status);
free_if_non_null(tempfile);
// Add upper bound to t3
status = open_temp_file(&tfp, &tempfile, -1); cBYE(status);
fclose_if_non_null(tfp);
tfp = fopen(tempfile, "wb"); return_if_fopen_failed(tfp, tempfile, "wb");
fwrite(f1ub, sizeof(int), nT, tfp);
fclose_if_non_null(tfp);
sprintf(str_meta_data, "fldtype=%s:n_sizeof=%d:filename=%s",
f1_meta->fldtype, f1_meta->n_sizeof, tempfile);
status = add_fld(t3, "ub", str_meta_data, &itemp); cBYE(status);
free_if_non_null(tempfile);
#undef OLD_WAY
#ifdef OLD_WAY
// Now we count how much there is in each range
inptr = (int *)f1_X;
for ( long long i = 0; i < nR; i++ ) {
int ival = *inptr++;
int range_idx = INT_MIN;
// TODO: Improve sequential search
for ( int j = 0; j < nT; j++ ) {
if ( ival >= f1lb[j] && ( ival <= f1ub[j] ) ) {
range_idx = j;
break;
}
}
count[range_idx]++;
}
/*
for ( int i = 0; i < nT; i++ ) {
fprintf(stdout,"%d: (%d, %d) = %lld \n", i, f1lb[i], f1ub[i], count[i]);
}
*/
#else
status = num_in_range(tbl, f1, t3, "lb", "ub", "cnt"); cBYE(status);
// Get a pointer to the count field
status = is_tbl(t3, &t3_id);
chk_range(t3_id, 0, g_n_tbl);
status = is_fld(NULL, t3_id, "cnt", &cnt_id);
chk_range(cnt_id, 0, g_n_fld);
cnt_meta = &(g_fld[cnt_id]);
status = rs_mmap(cnt_meta->filename, &cnt_X, &cnt_nX, 0); cBYE(status);
count = (long long *)cnt_X;
#endif
status = gettimeofday(&Tps, &Tpf); cBYE(status);
t_after_sec = (long long)Tps.tv_sec;
t_after_usec = (long long)Tps.tv_usec;
t_after = t_after_sec * 1000000 + t_after_usec;
fprintf(stderr, "TIME1 = %lld \n", t_after - t_before);
t_before = t_after;
bak_offsets = malloc(nT * sizeof(int *)); return_if_malloc_failed(bak_offsets);
g_offsets = malloc(nT * sizeof(int *)); return_if_malloc_failed(g_offsets);
#ifdef OLD_WAY
// Make space for output
long long filesz = nR * f1_meta->n_sizeof;
status = open_temp_file(&ofp, &opfile, filesz); cBYE(status);
status = mk_file(opfile, filesz); cBYE(status);
status = rs_mmap(opfile, &op_X, &op_nX, 1); cBYE(status);
offsets = malloc(nT * sizeof(int *)); return_if_malloc_failed(offsets);
long long cum_count = 0;
for ( int i = 0; i < nT; i++ ) {
bak_offsets[i] = offsets[i] = (int *)op_X;
if ( i > 0 ) {
cum_count += count[i-1];
offsets[i] += cum_count;
bak_offsets[i] = offsets[i];
}
}
inptr = (int *)f1_X;
// Now we place each item into its thread bucket
for ( long long i = 0; i < nR; i++ ) {
int ival = *inptr++;
int range_idx = INT_MIN;
// TODO: Improve sequential search
for ( int j = 0; j < nT; j++ ) {
if ( ival >= f1lb[j] && ( ival <= f1ub[j] ) ) {
range_idx = j;
break;
}
}
int *xptr = offsets[range_idx];
*xptr = ival;
offsets[range_idx]++;
chk_count[range_idx]++;
if ( chk_count[range_idx] > count[range_idx] ) {
go_BYE(-1);
}
}
cum_count = 0;
for ( int i = 0; i < nT-1; i++ ) {
if ( offsets[i] != bak_offsets[i+1] ) {
go_BYE(-1);
}
}
#else
status = mv_range(tbl, f1, f2, t3, "lb", "ub", "cnt");
cBYE(status);
status = is_fld(NULL, tbl_id, f2, &f2_id);
chk_range(f2_id, 0, g_n_fld);
f2_meta = &(g_fld[f2_id]);
status = rs_mmap(f2_meta->filename, &op_X, &op_nX, 1); cBYE(status);
#endif
long long cum_count = 0;
for ( int i = 0; i < nT; i++ ) {
bak_offsets[i] = (int *)op_X;
if ( i > 0 ) {
cum_count += count[i-1];
bak_offsets[i] += cum_count;
}
}
status = gettimeofday(&Tps, &Tpf); cBYE(status);
t_after_sec = (long long)Tps.tv_sec;
t_after_usec = (long long)Tps.tv_usec;
t_after = t_after_sec * 1000000 + t_after_usec;
fprintf(stderr, "TIME2 = %lld \n", t_after - t_before);
t_before = t_after;
// Set up global variables
g_nT = nT;
for ( int i = 0; i < nT; i++ ) {
g_offsets[i] = bak_offsets[i];
g_count[i] = count[i];
}
if ( g_nT == 1 ) {
core_parsort1(&(g_thread_id[0]));
}
else {
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
for ( int t = 0; t < g_nT; t++ ) {
rc = pthread_create(&threads[t], NULL, core_parsort1,
&(g_thread_id[t]));
if ( rc ) { go_BYE(-1); }
}
/* Free attribute and wait for the other threads */
pthread_attr_destroy(&attr);
for ( int t = 0; t < g_nT; t++ ) {
rc = pthread_join(threads[t], &thread_status);
if ( rc ) { go_BYE(-1); }
}
}
/* SEQUENTIAL CODE
for ( int i = 0; i < nT; i++ ) {
qsort_asc_int(bak_offsets[i], count[i], sizeof(int), NULL);
}
*/
status = gettimeofday(&Tps, &Tpf); cBYE(status);
t_after_sec = (long long)Tps.tv_sec;
t_after_usec = (long long)Tps.tv_usec;
t_after = t_after_sec * 1000000 + t_after_usec;
fprintf(stderr, "TIME3 = %lld \n", t_after - t_before);
// Indicate the dst_fld is sorted ascending
status = set_fld_info(tbl, f2, "sort=1");
rs_munmap(op_X, op_nX);
status = del_tbl(t2, -1); cBYE(status);
status = del_tbl(t3, -1); cBYE(status);
BYE:
rs_munmap(op_X, op_nX);
rs_munmap(cnt_X, cnt_nX);
free_if_non_null(xxx);
free_if_non_null(f1lb);
free_if_non_null(f1ub);
// Do not delete unless using OLD_WAY free_if_non_null(count);
free_if_non_null(g_count);
free_if_non_null(g_offsets);
free_if_non_null(offsets);
free_if_non_null(bak_offsets);
free_if_non_null(chk_count);
fclose_if_non_null(ofp);
g_write_to_temp_dir = false;
rs_munmap(f1_X, f1_nX);
rs_munmap(op_X, op_nX);
free_if_non_null(opfile);
return(status);
}
//---------------------------------------------------------------
// START FUNC DECL
int
count_vals(
char *src_tbl,
char *src_fld,
char *in_cnt_fld,
char *dst_tbl,
char *dst_fld,
char *out_cnt_fld
)
// STOP FUNC DECL
{
int status = 0;
char *src_fld_X = NULL; size_t src_fld_nX = 0;
char *in_cnt_fld_X = NULL; size_t in_cnt_fld_nX = 0;
FLD_TYPE *src_fld_meta = NULL, *in_cnt_fld_meta = NULL;
FLD_TYPE *cnt_fld_meta = NULL, *nn_src_fld_meta = NULL;
long long src_nR;
int src_tbl_id = INT_MIN, dst_tbl_id = INT_MIN;
int src_fld_id = INT_MIN, nn_src_fld_id = INT_MIN;
int dst_fld_id = INT_MIN;
int cnt_fld_id = INT_MIN, nn_cnt_fld_id = INT_MIN;
long long dst_nR, n_out;
char buffer[32];
char str_meta_data[256];
char *nn_src_fld_X = NULL; size_t nn_src_fld_nX = 0;
char *out_X = NULL; char *cnt_X = NULL;
char *opfile = NULL, *tmp_opfile = NULL; FILE *ofp = 0;
char *cnt_opfile = NULL, *tmp_cnt_opfile = NULL; FILE *cnt_ofp = 0;
size_t n_out_X = 0, n_cnt_X = 0;
unsigned int n_sizeof = INT_MAX;
//----------------------------------------------------------------
if ( ( src_tbl == NULL ) || ( *src_tbl == '\0' ) ) { go_BYE(-1); }
if ( ( src_fld == NULL ) || ( *src_fld == '\0' ) ) { go_BYE(-1); }
if ( ( dst_tbl == NULL ) || ( *dst_tbl == '\0' ) ) { go_BYE(-1); }
if ( ( dst_fld == NULL ) || ( *dst_fld == '\0' ) ) { go_BYE(-1); }
if ( strcmp(src_tbl, dst_tbl) == 0 ) { go_BYE(-1); }
zero_string(str_meta_data, 256);
zero_string(buffer, 32);
//--------------------------------------------------------
status = is_tbl(src_tbl, &src_tbl_id); cBYE(status);
chk_range(src_tbl_id, 0, g_n_tbl);
src_nR = g_tbl[src_tbl_id].nR;
//--------------------------------------------------------
status = is_fld(NULL, src_tbl_id, src_fld, &src_fld_id); cBYE(status);
chk_range(src_fld_id, 0, g_n_fld);
src_fld_meta = &(g_fld[src_fld_id]);
status = rs_mmap(src_fld_meta->filename, &src_fld_X, &src_fld_nX, 0); cBYE(status);
nn_src_fld_id = src_fld_meta->nn_fld_id;
if ( nn_src_fld_id >= 0 ) {
nn_src_fld_meta = &(g_fld[nn_src_fld_id]);
status = rs_mmap(nn_src_fld_meta->filename, &nn_src_fld_X, &nn_src_fld_nX, 0);
cBYE(status);
}
// Get the count field if specified. It cannot have null values
if ( *in_cnt_fld != '\0' ) {
status = is_fld(NULL, src_tbl_id, src_fld, &src_fld_id); cBYE(status);
chk_range(src_fld_id, 0, g_n_fld);
src_fld_meta = &(g_fld[src_fld_id]);
status = rs_mmap(src_fld_meta->filename, &src_fld_X, &src_fld_nX, 0); cBYE(status);
nn_cnt_fld_id = cnt_fld_meta->nn_fld_id;
chk_range(nn_cnt_fld_id, 0, g_n_fld);
}
/* Make sure src_fld field is sorted ascending */
zero_string(buffer, 32);
status = f_to_s(src_tbl, src_fld, "is_sorted", buffer);
cBYE(status);
if ( strcmp(buffer, "ascending") != 0 ) {
fprintf(stderr, "Field [%s] in Table [%s] not sorted ascending\n",
src_fld, src_tbl);
go_BYE(-1);
}
//--------------------------------------------------------
// Create 2 temporary files to store the results. We allocate space
// differently based on field types
g_write_to_temp_dir = true;
status = open_temp_file(&ofp, &tmp_opfile, 0); cBYE(status);
fclose_if_non_null(ofp);
status = open_temp_file(&cnt_ofp, &tmp_cnt_opfile, 0); cBYE(status);
fclose_if_non_null(cnt_ofp);
//------------------------------------------------------
n_out = src_nR * sizeof(int); // upper bound
if ( strcmp(src_fld_meta->fldtype, "int") == 0 ) {
status = mk_file(tmp_cnt_opfile, sizeof(int) * src_nR);
cBYE(status);
rs_mmap(tmp_cnt_opfile, &cnt_X, &n_cnt_X, 1);
cBYE(status);
status = mk_file(tmp_opfile, sizeof(int) * src_nR);
cBYE(status);
rs_mmap(tmp_opfile, &out_X, &n_out_X, 1);
cBYE(status);
if ( nn_src_fld_X == NULL ) {
if ( *in_cnt_fld == '\0' ) {
uniq_alldef_int((int *)src_fld_X, src_nR, (int *)out_X,
&dst_nR, (int *)cnt_X);
}
else {
uniq_cnt_alldef_i_i((int *)src_fld_X, (int *)in_cnt_fld_X,
src_nR, (int *)out_X, &dst_nR, (int *)cnt_X);
}
}
else {
uniq_int((int *)src_fld_X, src_nR, nn_src_fld_X,
(int *)out_X, &dst_nR, (int *)cnt_X);
}
}
else if ( strcmp(src_fld_meta->fldtype, "long long") == 0 ) {
status = mk_file(tmp_opfile, sizeof(long long) * src_nR);
cBYE(status);
rs_mmap(tmp_opfile, &out_X, &n_out_X, 1);
cBYE(status);
if ( nn_src_fld_X == NULL ) {
if ( *in_cnt_fld == '\0' ) {
status = mk_file(tmp_cnt_opfile, sizeof(long long) * src_nR);
cBYE(status);
rs_mmap(tmp_cnt_opfile, &cnt_X, &n_cnt_X, 1);
cBYE(status);
uniq_alldef_longlong((long long *)src_fld_X, src_nR,
(long long *)out_X, &dst_nR, (long long *)cnt_X);
}
else {
if ( strcmp(in_cnt_fld_meta->fldtype, "long long") == 0 ) {
status = mk_file(tmp_cnt_opfile, sizeof(long long) * src_nR);
cBYE(status);
rs_mmap(tmp_cnt_opfile, &cnt_X, &n_cnt_X, 1);
cBYE(status);
uniq_cnt_alldef_ll_ll((long long *)src_fld_X,
(long long *)in_cnt_fld_X, src_nR, (long long *)out_X, &dst_nR,
(long long *)cnt_X);
}
else if ( strcmp(in_cnt_fld_meta->fldtype, "double") == 0 ) {
status = mk_file(tmp_cnt_opfile, sizeof(double) * src_nR);
cBYE(status);
rs_mmap(tmp_cnt_opfile, &cnt_X, &n_cnt_X, 1);
cBYE(status);
uniq_cnt_alldef_ll_dbl((long long *)src_fld_X,
(double *)in_cnt_fld_X, src_nR, (long long *)out_X, &dst_nR,
(double *)cnt_X);
}
}
}
else {
if ( *in_cnt_fld == '\0' ) {
status = mk_file(tmp_cnt_opfile, sizeof(long long) * src_nR);
cBYE(status);
rs_mmap(tmp_cnt_opfile, &cnt_X, &n_cnt_X, 1);
cBYE(status);
uniq_longlong((long long *)src_fld_X, src_nR, nn_src_fld_X,
(long long *)out_X, &dst_nR, (long long *)cnt_X);
}
else {
fprintf(stderr, "NOT IMPLEMENTED\n"); go_BYE(-1);
}
}
}
else { go_BYE(-1); }
if ( dst_nR == 0 ) { goto BYE; }
//-----------------------------------------------------------
status = is_tbl(dst_tbl, &dst_tbl_id); cBYE(status);
if ( dst_tbl_id >= 0 ) {
status = del_tbl(NULL, dst_tbl_id);
}
sprintf(buffer, "%lld", dst_nR);
status = add_tbl(dst_tbl, buffer, &dst_tbl_id);
cBYE(status);
//-----------------------------------------------------------
// Now copy the temporary files over to real files
status = open_temp_file(&ofp, &opfile, 0);
cBYE(status);
if ( strcmp(src_fld_meta->fldtype, "int") == 0 ) {
fwrite(out_X, sizeof(int), dst_nR, ofp);
n_sizeof = sizeof(int);
}
else if ( strcmp(src_fld_meta->fldtype, "long long") == 0 ) {
fwrite(out_X, sizeof(long long), dst_nR, ofp);
n_sizeof = sizeof(long long);
}
fclose_if_non_null(ofp);
// Add output field to meta data
sprintf(str_meta_data, "filename=%s:fldtype=%s:n_sizeof=%u", opfile,
src_fld_meta->fldtype, n_sizeof);
status = add_fld(dst_tbl, dst_fld, str_meta_data, &dst_fld_id);
cBYE(status);
// Since src_fld is sorted ascending, so is dst_fld
g_fld[dst_fld_id].sorttype = 1;
// Now copy the temporary count file to the real one
status = open_temp_file(&cnt_ofp, &cnt_opfile, 0);
cBYE(status);
if ( *in_cnt_fld == '\0' ) {
if ( strcmp(src_fld_meta->fldtype, "int") == 0 ) {
fwrite(cnt_X, sizeof(int), dst_nR, cnt_ofp);
sprintf(str_meta_data, "filename=%s:fldtype=int:n_sizeof=%lu",
cnt_opfile, sizeof(int));
}
else if ( strcmp(src_fld_meta->fldtype, "long long") == 0 ) {
fwrite(cnt_X, sizeof(long long), dst_nR, cnt_ofp);
sprintf(str_meta_data, "filename=%s:fldtype=long long:n_sizeof=%lu",
cnt_opfile, sizeof(long long));
}
else { go_BYE(-1); }
}
else {
if ( strcmp(in_cnt_fld_meta->fldtype, "int") == 0 ) {
fwrite(cnt_X, sizeof(int), dst_nR, cnt_ofp);
sprintf(str_meta_data, "filename=%s:fldtype=int:n_sizeof=%lu",
cnt_opfile, sizeof(int));
}
else if ( strcmp(in_cnt_fld_meta->fldtype, "long long") == 0 ) {
fwrite(cnt_X, sizeof(long long), dst_nR, cnt_ofp);
sprintf(str_meta_data, "filename=%s:fldtype=long long:n_sizeof=%lu",
cnt_opfile, sizeof(long long));
}
else if ( strcmp(in_cnt_fld_meta->fldtype, "double") == 0 ) {
fwrite(cnt_X, sizeof(double), dst_nR, cnt_ofp);
sprintf(str_meta_data, "filename=%s:fldtype=double:n_sizeof=%lu",
cnt_opfile, sizeof(double));
}
else { go_BYE(-1); }
}
fclose_if_non_null(cnt_ofp);
// Add count field to meta data
status = add_fld(dst_tbl, out_cnt_fld, str_meta_data, &cnt_fld_id);
cBYE(status);
//-----------------------------------------------------------
BYE:
g_write_to_temp_dir = false;
rs_munmap(src_fld_X, src_fld_nX);
free_if_non_null(opfile);
rs_munmap(src_fld_X, src_fld_nX);
rs_munmap(in_cnt_fld_X, in_cnt_fld_nX);
rs_munmap(cnt_X, n_cnt_X);
rs_munmap(out_X, n_out_X);
rs_munmap(nn_src_fld_X, nn_src_fld_nX);
unlink(tmp_opfile);
unlink(tmp_cnt_opfile);
free_if_non_null(tmp_opfile);
free_if_non_null(tmp_cnt_opfile);
free_if_non_null(cnt_opfile);
return(status);
}
// last review 9/11/2013
//---------------------------------------------------------------
// START FUNC DECL
int
num_in_range(
char *t1,
char *f1,
char *t2, /* We expect this table to be small */
char *lb,
char *ub,
char *cnt,
char *rslt_buf
)
// STOP FUNC DECL
{
int status = 0;
char *f1_X = NULL; size_t f1_nX = 0;
char *lb_X = NULL; size_t lb_nX = 0;
char *ub_X = NULL; size_t ub_nX = 0;
char *cnt_X = NULL; size_t cnt_nX = 0;
int t1_id = INT_MIN, t2_id = INT_MIN;
int f1_id = INT_MIN, lb_id = INT_MIN, ub_id = INT_MIN, cnt_id = INT_MIN;
int nn_f1_id = INT_MIN, nn_lb_id = INT_MIN, nn_ub_id = INT_MIN;
TBL_REC_TYPE t1_rec, t2_rec;
FLD_REC_TYPE f1_rec, lb_rec, ub_rec, cnt_rec;
FLD_REC_TYPE nn_f1_rec, nn_lb_rec, nn_ub_rec;
long long nR1 = INT_MIN, nR2 = INT_MIN;
long long *offsets = NULL;
#define BUFLEN 32
char f1_buf[BUFLEN], lb_ub_buf[BUFLEN];
int ddir_id = -1, fileno = -1 ;
FLD_TYPE cnt_fldtype = I4;
int fldsz = 0; size_t filesz = 0;
//----------------------------------------------------------------
if ( ( t1 == NULL ) || ( *t1 == '\0' ) ) { go_BYE(-1); }
if ( ( f1 == NULL ) || ( *f1 == '\0' ) ) { go_BYE(-1); }
if ( ( t2 == NULL ) || ( *t2 == '\0' ) ) { go_BYE(-1); }
if ( ( lb == NULL ) || ( *lb == '\0' ) ) { go_BYE(-1); }
if ( ( ub == NULL ) || ( *ub == '\0' ) ) { go_BYE(-1); }
if ( ( cnt == NULL ) || ( *cnt == '\0' ) ) { go_BYE(-1); }
if ( strcmp(t1, t2) == 0 ) { go_BYE(-1); }
if ( strcmp(lb, ub) == 0 ) { go_BYE(-1); }
if ( strcmp(ub, cnt) == 0 ) { go_BYE(-1); }
if ( strcmp(cnt, lb) == 0 ) { go_BYE(-1); }
zero_string(f1_buf, BUFLEN);
zero_string(lb_ub_buf, BUFLEN);
//--------------------------------------------------------
status = is_tbl(t1, &t1_id, &t1_rec); cBYE(status);
chk_range(t1_id, 0, g_n_tbl);
nR1 = t1_rec.nR;
if ( nR1 > INT_MAX ) { go_BYE(-1); }
//--------------------------------------------------------
status = is_fld(NULL, t1_id, f1, &f1_id, &f1_rec, &nn_f1_id, &nn_f1_rec);
cBYE(status);
chk_range(f1_id, 0, g_n_fld);
status = get_data(f1_rec, &f1_X, &f1_nX, 0); cBYE(status);
if ( nn_f1_id >= 0 ) { go_BYE(-1); }
if ( f1_rec.fldtype != I4 ) { go_BYE(-1); }
//--------------------------------------------------------
status = is_tbl(t2, &t2_id, &t2_rec); cBYE(status);
chk_range(t2_id, 0, g_n_tbl);
nR2 = t2_rec.nR;
//--------------------------------------------------------
status = is_fld(NULL, t2_id, lb, &lb_id, &lb_rec, &nn_lb_id, &nn_lb_rec);
cBYE(status);
chk_range(lb_id, 0, g_n_fld);
status = get_data(lb_rec, &lb_X, &lb_nX, 0); cBYE(status);
if ( nn_lb_id >= 0 ) { go_BYE(-1); }
if ( lb_rec.fldtype != I4 ) { cBYE(-1); }
//--------------------------------------------------------
status = is_fld(NULL, t2_id, ub, &ub_id, &ub_rec, &nn_ub_id, &nn_ub_rec);
cBYE(status);
chk_range(ub_id, 0, g_n_fld);
status = get_data(ub_rec, &ub_X, &ub_nX, 0); cBYE(status);
if ( nn_ub_id >= 0 ) { go_BYE(-1); }
if ( ub_rec.fldtype != I4 ) { cBYE(-1); }
//--------------------------------------------------------
/*-- Current implementation assumes that numbers sorted in particular
* manner. This is checked below */
status = f_to_s(t1, f1, "is_sorted", f1_buf, BUFLEN); cBYE(status);
if ( ( strcmp(f1_buf, "ascending") != 0 ) &&
( strcmp(f1_buf, "descending") != 0 ) ) {
fprintf(stderr , "fld [%s] in tbl [%s] should be sorted\n", f1, t1);
go_BYE(-1);
}
status = f_to_s(t2, lb, "is_sorted", lb_ub_buf, BUFLEN); cBYE(status);
if ( strcmp(lb_ub_buf, f1_buf) != 0 ) {
fprintf(stderr , "Expect fld %s in tbl %s to be sorted ", lb, t2);
fprintf(stderr , "same as fld %s in tbl %s\n", f1, t2);
go_BYE(-1);
}
status = f_to_s(t2, ub, "is_sorted", lb_ub_buf, BUFLEN); cBYE(status);
if ( strcmp(lb_ub_buf, f1_buf) != 0 ) {
fprintf(stderr , "Expect fld %s in tbl %s to be sorted ", ub, t2);
fprintf(stderr , "same as fld %s in tbl %s\n", f1, t2);
go_BYE(-1);
}
/* Make space for output */
cnt_fldtype = I4;
status = get_fld_sz(cnt_fldtype, &fldsz); cBYE(status);
filesz = fldsz * nR2;
status = mk_temp_file(filesz, &ddir_id, &fileno); cBYE(status);
status = q_mmap(ddir_id, fileno, &cnt_X, &cnt_nX, 1);
int *cntptr = (int *)cnt_X;
assign_const_I4(cntptr, nR2, 0);
long long chk_nR1 = 0;
//-----------------------------------------------------------
int *inptr = (int *)f1_X;
int *lbptr = (int *)lb_X;
int *ubptr = (int *)ub_X;
int t2idx = 0;
int lbval = lbptr[0];
int ubval = ubptr[0];
if ( strcmp(f1_buf, "ascending") == 0 ) {
for ( long long i = 0; i < nR1; i++ ) {
int valI4 = inptr[i];
if ( valI4 < lbval ) { continue; }
for ( ; t2idx < nR2 ; ) {
if ( valI4 < ubval ) {
cntptr[t2idx]++;
chk_nR1++;
break;
}
t2idx++;
if ( t2idx >= nR2 ) { break; }
lbval = lbptr[t2idx];
ubval = ubptr[t2idx];
if ( valI4 < lbval ) { break; }
}
if ( t2idx == nR2 ) { break; }
}
}
else if ( strcmp(f1_buf, "descending") == 0 ) {
for ( long long i = 0; i < nR1; i++ ) {
int valI4 = inptr[i];
if ( valI4 >= ubval ) { continue; }
for ( ; t2idx < nR2 ; ) {
if ( valI4 >= lbval ) {
cntptr[t2idx]++;
chk_nR1++;
break;
}
t2idx++;
if ( t2idx >= nR2 ) { break; }
lbval = lbptr[t2idx];
ubval = ubptr[t2idx];
if ( valI4 >= ubval ) { break; }
}
if ( t2idx == nR2 ) { break; }
}
}
else {
go_BYE(-1);
}
sprintf(rslt_buf, "%lld", chk_nR1); // return number of matches
zero_fld_rec(&cnt_rec); cnt_rec.fldtype = cnt_fldtype;
status = add_fld(t2_id, cnt, ddir_id, fileno, &cnt_id, &cnt_rec); cBYE(status);
BYE:
rs_munmap(f1_X, f1_nX);
rs_munmap(lb_X, lb_nX);
rs_munmap(ub_X, ub_nX);
rs_munmap(cnt_X, cnt_nX);
free_if_non_null(offsets);
return(status);
}
static VALUE
nurat_s_convert(int argc, VALUE *argv, VALUE klass)
{
VALUE a1, a2;
if (rb_scan_args(argc, argv, "02", &a1, &a2) == 1) {
a2 = ONE;
}
switch (TYPE(a1)) {
case T_COMPLEX:
if (k_float_p(RCOMPLEX(a1)->image) || !f_zero_p(RCOMPLEX(a1)->image)) {
VALUE s = f_to_s(a1);
rb_raise(rb_eRangeError, "can't accept %s",
StringValuePtr(s));
}
a1 = RCOMPLEX(a1)->real;
}
switch (TYPE(a2)) {
case T_COMPLEX:
if (k_float_p(RCOMPLEX(a2)->image) || !f_zero_p(RCOMPLEX(a2)->image)) {
VALUE s = f_to_s(a2);
rb_raise(rb_eRangeError, "can't accept %s",
StringValuePtr(s));
}
a2 = RCOMPLEX(a2)->real;
}
switch (TYPE(a1)) {
case T_FIXNUM:
case T_BIGNUM:
break;
case T_FLOAT:
a1 = f_to_r(a1);
break;
case T_STRING:
a1 = string_to_r_strict(a1);
break;
}
switch (TYPE(a2)) {
case T_FIXNUM:
case T_BIGNUM:
break;
case T_FLOAT:
a2 = f_to_r(a2);
break;
case T_STRING:
a2 = string_to_r_strict(a2);
break;
}
switch (TYPE(a1)) {
case T_RATIONAL:
if (NIL_P(a2) || f_zero_p(a2))
return a1;
else
return f_div(a1, a2);
}
switch (TYPE(a2)) {
case T_RATIONAL:
return f_div(a1, a2);
}
return nurat_s_new(klass, a1, a2);
}
//---------------------------------------------------------------
// START FUNC DECL
int
percentiles(
char *src_tbl,
char *src_fld,
char *dst_tbl,
char *str_n_out
)
// STOP FUNC DECL
{
int status = 0;
char *src_fld_X = NULL; size_t src_fld_nX = 0;
TBL_REC_TYPE src_tbl_rec; int src_tbl_id = -1; long long src_nR = -1;
TBL_REC_TYPE dst_tbl_rec; int dst_tbl_id = -1; long long dst_nR = -1;
FLD_REC_TYPE src_fld_rec; int src_fld_id = -1;
FLD_REC_TYPE nn_src_fld_rec; int nn_src_fld_id = -1;
FLD_REC_TYPE min_rec; int min_id = -1;
FLD_REC_TYPE max_rec; int max_id = -1;
FLD_REC_TYPE avg_rec; int avg_id = -1;
FLD_REC_TYPE cnt_rec; int cnt_id = -1;
char *minX = NULL; size_t min_nX = 0; int min_ddir_id = -1, min_fileno = 0;
char *maxX = NULL; size_t max_nX = 0; int max_ddir_id = -1, max_fileno = 0;
char *cntX = NULL; size_t cnt_nX = 0; int cnt_ddir_id = -1, cnt_fileno = 0;
char *avgX = NULL; size_t avg_nX = 0; int avg_ddir_id = -1, avg_fileno = 0;
int fldsz = 0; long long filesz = 0;
int bin_size;
//----------------------------------------------------------------
if ( ( src_tbl == NULL ) || ( *src_tbl == '\0' ) ) { go_BYE(-1); }
if ( ( src_fld == NULL ) || ( *src_fld == '\0' ) ) { go_BYE(-1); }
if ( ( dst_tbl == NULL ) || ( *dst_tbl == '\0' ) ) { go_BYE(-1); }
if ( strcmp(src_tbl, dst_tbl) == 0 ) { go_BYE(-1); }
//--------------------------------------------------------
status = stoI8(str_n_out, &dst_nR); cBYE(status);
if ( ( dst_nR >= MAX_BINS_FOR_PERCENTILE ) || ( dst_nR <= 1 ) ) {
fprintf(stderr, "num_bins = %s not in valid range [2, %d] \n",
str_n_out, MAX_BINS_FOR_PERCENTILE);
go_BYE(-1);
}
//--------------------------------------------------------
status = is_tbl(src_tbl, &src_tbl_id, &src_tbl_rec); cBYE(status);
if (src_tbl_id < 0 ) { go_BYE(-1); }
src_nR = src_tbl_rec.nR;
if ( dst_nR >= src_nR ) {
fprintf(stderr, "Source Table [%s] has insufficient rows [%lld]\n",
src_tbl, src_nR);
go_BYE(-1);
}
status = is_fld(NULL, src_tbl_id, src_fld, &src_fld_id, &src_fld_rec,
&nn_src_fld_id, &nn_src_fld_rec);
if ( src_fld_id < 0 ) { go_BYE(-1); }
switch ( src_fld_rec.fldtype ) {
case I4 : case I8 : case F4 : case F8 : break;
default : go_BYE(-1); break;
}
if ( nn_src_fld_id >= 0 ) { go_BYE(-1); }
/* Make sure src_fld field is sorted ascending */
char srttype[32]; zero_string(srttype, 32);
status = f_to_s(src_tbl, src_fld, "is_sorted", srttype, 32); cBYE(status);
if ( strcmp(srttype, "ascending") != 0 ) {
fprintf(stderr, "Field [%s] in Table [%s] not sorted ascending\n",
src_fld, src_tbl);
go_BYE(-1);
}
//--------------------------------------------------------
status = get_data(src_fld_rec, &src_fld_X, &src_fld_nX, 0); cBYE(status);
//--------------------------------------------------------
// Create 4 files for the 4 fields to be created
// min, max, cnt, avg
int n1, n2;
status = get_fld_sz(I8, &n1); cBYE(status);
status = get_fld_sz(F8, &n2); cBYE(status);
if ( n1 != n2 ) { go_BYE(-1); }
status = get_fld_sz(I8, &fldsz); cBYE(status);
filesz = fldsz * dst_nR;
status = mk_temp_file(filesz, &min_ddir_id, &min_fileno); cBYE(status);
status = mk_temp_file(filesz, &max_ddir_id, &max_fileno); cBYE(status);
status = mk_temp_file(filesz, &cnt_ddir_id, &cnt_fileno); cBYE(status);
status = get_fld_sz(F8, &fldsz); cBYE(status);
filesz = fldsz * dst_nR;
status = mk_temp_file(filesz, &avg_ddir_id, &avg_fileno); cBYE(status);
status = q_mmap(min_ddir_id, min_fileno, &minX, &min_nX, 1); cBYE(status);
status = q_mmap(max_ddir_id, max_fileno, &maxX, &max_nX, 1); cBYE(status);
status = q_mmap(cnt_ddir_id, cnt_fileno, &cntX, &cnt_nX, 1); cBYE(status);
status = q_mmap(avg_ddir_id, avg_fileno, &avgX, &avg_nX, 1); cBYE(status);
//------------------------------------------------------
/* Delete table if it exists. Create brand new table */
status = is_tbl(dst_tbl, &dst_tbl_id , &dst_tbl_rec); cBYE(status);
if ( dst_tbl_id >= 0 ) {
status = del_tbl(NULL, dst_tbl_id); cBYE(status);
}
zero_tbl_rec(&dst_tbl_rec);
status = add_tbl(dst_tbl, str_n_out, &dst_tbl_id, &dst_tbl_rec); cBYE(status);
//-----------------------------------------------------------
// START: Here starts the processing
/* b is the bin number */
long long *minI8 = (long long *)minX;
long long *maxI8 = (long long *)maxX;
double *minF8 = (double *)minX;
double *maxF8 = (double *)maxX;
long long *cntI8 = (long long *)cntX;
double *avgF8 = (double *)avgX;
bin_size = src_nR / dst_nR;
cilkfor ( int b = 0; b < dst_nR; b++ ) {
long long lb = b * bin_size;
long long ub = lb + bin_size;
if ( b == ( dst_nR -1 ) ) { ub = src_nR; }
int *valsI4 = (int *) src_fld_X; valsI4 += lb;
float *valsF4 = (float *) src_fld_X; valsF4 += lb;
long long *valsI8 = (long long *) src_fld_X; valsI8 += lb;
double *valsF8 = (double *) src_fld_X; valsF8 += lb;
//-----------------------------------------------
double lvalF8 = 0; long long lvalI8 = 0;
double lsumF8 = 0; long long lsumI8 = 0;
long long lminI8 = LLONG_MAX, lmaxI8 = LLONG_MIN;
double lminF8 = DBL_MIN, lmaxF8 = DBL_MIN;
//-----------------------------------------------
for ( long long i = 0; i < (ub - lb); i++ ) {
switch ( src_fld_rec.fldtype ) {
case I4 : lvalI8 = valsI4[i]; break;
case I8 : lvalI8 = valsI8[i]; break;
case F4 : lvalF8 = valsI4[i]; break;
case F8 : lvalF8 = valsI8[i]; break;
default : status = -1; continue; break;
}
switch ( src_fld_rec.fldtype ) {
case I4 : case I8 :
lminI8 = min(lminI8 , lvalI8);
lmaxI8 = max(lmaxI8 , lvalI8);
lsumI8 += lvalI8;
break;
case F4 : case F8 :
lminF8 = min(lminF8 , lvalF8);
lmaxF8 = max(lmaxF8 , lvalF8);
lsumF8 += lvalF8;
break;
default : status = -1; continue; break;
}
}
// Write out the values
cntI8[b] = (ub - lb);
double n = ub - lb;
switch ( src_fld_rec.fldtype ) {
case I4 : case I8 :
minI8[b] = lminI8;
maxI8[b] = lmaxI8;
avgF8[b] = (double)lsumI8 / n;
break;
case F4 : case F8 :
minF8[b] = lminF8;
maxF8[b] = lmaxF8;
avgF8[b] = (double)lsumF8 / n;
break;
default : status = -1; continue; break;
}
}
//-----------------------------------------------------------
status = add_tbl(dst_tbl, str_n_out, &dst_tbl_id, &dst_tbl_rec); cBYE(status);
// Add output field(s) to meta data
zero_fld_rec(&min_rec); zero_fld_rec(&max_rec);
switch ( src_fld_rec.fldtype ) {
case I4 : case I8 : min_rec.fldtype = I8; max_rec.fldtype = I8; break;
case F4 : case F8 : min_rec.fldtype = F8; max_rec.fldtype = F8; break;
default : go_BYE(-1); break;
}
status = add_fld(dst_tbl_id, "min", min_ddir_id, min_fileno,
&min_id, &min_rec); cBYE(status);
status = add_fld(dst_tbl_id, "max", max_ddir_id, max_fileno,
&max_id, &max_rec); cBYE(status);
zero_fld_rec(&avg_rec); avg_rec.fldtype = F8;
status = add_fld(dst_tbl_id, "avg", avg_ddir_id, avg_fileno,
&avg_id, &avg_rec); cBYE(status);
zero_fld_rec(&cnt_rec); cnt_rec.fldtype = I8;
status = add_fld(dst_tbl_id, "cnt", cnt_ddir_id, cnt_fileno,
&cnt_id, &cnt_rec); cBYE(status);
//-----------------------------------------------------------
BYE:
rs_munmap(src_fld_X, src_fld_nX);
rs_munmap(minX, min_nX);
rs_munmap(maxX, max_nX);
rs_munmap(cntX, cnt_nX);
rs_munmap(avgX, avg_nX);
return(status);
}
