void bin_load(uint4 begin, uint4 end)
{
unsigned char *ptr, *cp1, *cp2, *btop, *gvkey_char_ptr, *tmp_ptr, *tmp_key_ptr, *c, *ctop;
unsigned char hdr_lvl, src_buff[MAX_KEY_SZ + 1], dest_buff[MAX_ZWR_KEY_SZ],
cmpc_str[MAX_KEY_SZ + 1], dup_key_str[MAX_KEY_SZ + 1];
unsigned char *end_buff;
unsigned short rec_len, next_cmpc;
int len;
int current, last, length, max_blk_siz, max_key, status;
uint4 iter, max_data_len, max_subsc_len, key_count;
ssize_t rec_count, global_key_count, subsc_len,extr_std_null_coll;
boolean_t need_xlation, new_gvn, utf8_extract;
rec_hdr *rp, *next_rp;
mval v, tmp_mval;
mstr mstr_src, mstr_dest;
collseq *extr_collseq, *db_collseq, *save_gv_target_collseq;
coll_hdr extr_collhdr, db_collhdr;
gv_key *tmp_gvkey = NULL; /* null-initialize at start, will be malloced later */
char std_null_coll[BIN_HEADER_NUMSZ + 1];
# ifdef GTM_CRYPT
gtmcrypt_key_t *encr_key_handles;
char *inbuf;
int4 index;
int req_dec_blk_size, init_status, crypt_status;
muext_hash_hdr_ptr_t hash_array = NULL;
# endif
DCL_THREADGBL_ACCESS;
SETUP_THREADGBL_ACCESS;
assert(4 == SIZEOF(coll_hdr));
gvinit();
v.mvtype = MV_STR;
len = file_input_bin_get((char **)&ptr);
hdr_lvl = EXTR_HEADER_LEVEL(ptr);
if (!(((('4' == hdr_lvl) || ('5' == hdr_lvl)) && (BIN_HEADER_SZ == len)) || (('4' > hdr_lvl) && (V3_BIN_HEADER_SZ == len))))
{
rts_error(VARLSTCNT(1) ERR_LDBINFMT);
mupip_exit(ERR_LDBINFMT);
}
/* expecting the level in a single character */
assert(' ' == *(ptr + SIZEOF(BIN_HEADER_LABEL) - 3));
if (0 != memcmp(ptr, BIN_HEADER_LABEL, SIZEOF(BIN_HEADER_LABEL) - 2) || ('2' > hdr_lvl) || *(BIN_HEADER_VERSION) < hdr_lvl)
{ /* ignore the level check */
rts_error(VARLSTCNT(1) ERR_LDBINFMT);
mupip_exit(ERR_LDBINFMT);
}
/* check if extract was generated in UTF-8 mode */
utf8_extract = (0 == MEMCMP_LIT(&ptr[len - BIN_HEADER_LABELSZ], UTF8_NAME)) ? TRUE : FALSE;
if ((utf8_extract && !gtm_utf8_mode) || (!utf8_extract && gtm_utf8_mode))
{ /* extract CHSET doesn't match $ZCHSET */
if (utf8_extract)
rts_error(VARLSTCNT(4) ERR_LOADINVCHSET, 2, LEN_AND_LIT("UTF-8"));
else
rts_error(VARLSTCNT(4) ERR_LOADINVCHSET, 2, LEN_AND_LIT("M"));
mupip_exit(ERR_LDBINFMT);
}
if ('4' >= hdr_lvl)
{ /* Binary extracts in V50000-to-V52000 (label=4) and pre-V50000 (label=3) could have a '\0' byte (NULL byte)
* in the middle of the string. Replace it with ' ' (space) like it would be in V52000 binary extracts and above.
*/
for (c = ptr, ctop = c + len; c < ctop; c++)
{
if ('\0' == *c)
*c = ' ';
}
}
util_out_print("Label = !AD\n", TRUE, len, ptr);
new_gvn = FALSE;
if (hdr_lvl > '3')
{
memcpy(std_null_coll, ptr + BIN_HEADER_NULLCOLLOFFSET, BIN_HEADER_NUMSZ);
std_null_coll[BIN_HEADER_NUMSZ] = '\0';
extr_std_null_coll = STRTOUL(std_null_coll, NULL, 10);
if (0 != extr_std_null_coll && 1!= extr_std_null_coll)
{
rts_error(VARLSTCNT(5) ERR_TEXT, 2, RTS_ERROR_TEXT("Corrupted null collation field in header"),
ERR_LDBINFMT);
mupip_exit(ERR_LDBINFMT);
}
} else
extr_std_null_coll = 0;
# ifdef GTM_CRYPT
if ('5' <= hdr_lvl)
{
int i, num_indexes;
len = file_input_bin_get((char **)&ptr);
hash_array = (muext_hash_hdr *)malloc(len);
/* store hashes of all the files used during extract into muext_hash_hdr structure */
memcpy((char *)hash_array, ptr, len);
num_indexes = len / GTMCRYPT_HASH_LEN;
encr_key_handles = (gtmcrypt_key_t *)malloc(SIZEOF(gtmcrypt_key_t) * num_indexes);
INIT_PROC_ENCRYPTION(crypt_status);
GC_BIN_LOAD_ERR(crypt_status);
for (index = 0; index < num_indexes; index++)
{
if (0 == memcmp(hash_array[index].gtmcrypt_hash, EMPTY_GTMCRYPT_HASH, GTMCRYPT_HASH_LEN))
continue;
GTMCRYPT_GETKEY(hash_array[index].gtmcrypt_hash, encr_key_handles[index], crypt_status);
GC_BIN_LOAD_ERR(crypt_status);
}
}
# endif
if ('2' < hdr_lvl)
{
len = file_input_bin_get((char **)&ptr);
if (SIZEOF(coll_hdr) != len)
{
rts_error(VARLSTCNT(5) ERR_TEXT, 2, RTS_ERROR_TEXT("Corrupt collation header"), ERR_LDBINFMT);
mupip_exit(ERR_LDBINFMT);
}
extr_collhdr = *((coll_hdr *)(ptr));
new_gvn = TRUE;
} else
gtm_putmsg(VARLSTCNT(3) ERR_OLDBINEXTRACT, 1, hdr_lvl - '0');
if (begin < 2)
begin = 2;
for (iter = 2; iter < begin; iter++)
{
if (!(len = file_input_bin_get((char **)&ptr)))
{
gtm_putmsg(VARLSTCNT(3) ERR_LOADEOF, 1, begin);
util_out_print("Error reading record number: !UL\n", TRUE, iter);
mupip_error_occurred = TRUE;
return;
} else if (len == SIZEOF(coll_hdr))
{
extr_collhdr = *((coll_hdr *)(ptr));
assert(hdr_lvl > '2');
iter--;
}
}
assert(iter == begin);
util_out_print("Beginning LOAD at record number: !UL\n", TRUE, begin);
max_data_len = 0;
max_subsc_len = 0;
global_key_count = key_count = 0;
rec_count = begin - 1;
extr_collseq = db_collseq = NULL;
need_xlation = FALSE;
assert(NULL == tmp_gvkey); /* GVKEY_INIT macro relies on this */
GVKEY_INIT(tmp_gvkey, DBKEYSIZE(MAX_KEY_SZ)); /* tmp_gvkey will point to malloced memory after this */
for (; !mupip_DB_full ;)
{
if (++rec_count > end)
break;
next_cmpc = 0;
mupip_error_occurred = FALSE;
if (mu_ctrly_occurred)
break;
if (mu_ctrlc_occurred)
{
util_out_print("!AD:!_ Key cnt: !UL max subsc len: !UL max data len: !UL", TRUE,
LEN_AND_LIT(gt_lit), key_count, max_subsc_len, max_data_len);
util_out_print("Last LOAD record number: !UL", TRUE, key_count ? (rec_count - 1) : 0);
mu_gvis();
util_out_print(0, TRUE);
mu_ctrlc_occurred = FALSE;
}
/* reset the stringpool for every record in order to avoid garbage collection */
stringpool.free = stringpool.base;
if (!(len = file_input_bin_get((char **)&ptr)) || mupip_error_occurred)
break;
else if (len == SIZEOF(coll_hdr))
{
extr_collhdr = *((coll_hdr *)(ptr));
assert(hdr_lvl > '2');
new_gvn = TRUE; /* next record will contain a new gvn */
rec_count--; /* Decrement as this record does not count as a record for loading purposes */
continue;
}
rp = (rec_hdr*)(ptr);
# ifdef GTM_CRYPT
if ('5' <= hdr_lvl)
{ /* Getting index value from the extracted file. It indicates which database file this record belongs to */
GET_LONG(index, ptr);
if (-1 != index) /* Indicates that the record is encrypted. */
{
req_dec_blk_size = len - SIZEOF(int4);
inbuf = (char *)(ptr + SIZEOF(int4));
GTMCRYPT_DECODE_FAST(encr_key_handles[index], inbuf, req_dec_blk_size, NULL, crypt_status);
GC_BIN_LOAD_ERR(crypt_status);
}
rp = (rec_hdr*)(ptr + SIZEOF(int4));
}
# endif
btop = ptr + len;
cp1 = (unsigned char*)(rp + 1);
v.str.addr = (char*)cp1;
while (*cp1++)
;
v.str.len =INTCAST((char*)cp1 - v.str.addr - 1);
if (('2' >= hdr_lvl) || new_gvn)
{
if ((HASHT_GBLNAME_LEN == v.str.len) && (0 == memcmp(v.str.addr, HASHT_GBLNAME, HASHT_GBLNAME_LEN)))
continue;
bin_call_db(BIN_BIND, (INTPTR_T)gd_header, (INTPTR_T)&v.str);
max_key = gv_cur_region->max_key_size;
db_collhdr.act = gv_target->act;
db_collhdr.ver = gv_target->ver;
db_collhdr.nct = gv_target->nct;
}
GET_USHORT(rec_len, &rp->rsiz);
if (rp->cmpc != 0 || v.str.len > rec_len || mupip_error_occurred)
{
bin_call_db(ERR_COR, (INTPTR_T)rec_count, (INTPTR_T)global_key_count);
mu_gvis();
util_out_print(0, TRUE);
continue;
}
if (new_gvn)
{
global_key_count = 1;
if ((db_collhdr.act != extr_collhdr.act || db_collhdr.ver != extr_collhdr.ver
|| db_collhdr.nct != extr_collhdr.nct
|| gv_cur_region->std_null_coll != extr_std_null_coll))
{
if (extr_collhdr.act)
{
if (extr_collseq = ready_collseq((int)extr_collhdr.act))
{
if (!do_verify(extr_collseq, extr_collhdr.act, extr_collhdr.ver))
{
gtm_putmsg(VARLSTCNT(8) ERR_COLLTYPVERSION, 2, extr_collhdr.act,
extr_collhdr.ver, ERR_GVIS, 2, gv_altkey->end - 1, gv_altkey->base);
mupip_exit(ERR_COLLTYPVERSION);
}
} else
{
gtm_putmsg(VARLSTCNT(7) ERR_COLLATIONUNDEF, 1, extr_collhdr.act,
ERR_GVIS, 2, gv_altkey->end - 1, gv_altkey->base);
mupip_exit(ERR_COLLATIONUNDEF);
}
}
if (db_collhdr.act)
{
if (db_collseq = ready_collseq((int)db_collhdr.act))
{
if (!do_verify(db_collseq, db_collhdr.act, db_collhdr.ver))
{
gtm_putmsg(VARLSTCNT(8) ERR_COLLTYPVERSION, 2, db_collhdr.act,
db_collhdr.ver, ERR_GVIS, 2, gv_altkey->end - 1, gv_altkey->base);
mupip_exit(ERR_COLLTYPVERSION);
}
} else
{
gtm_putmsg(VARLSTCNT(7) ERR_COLLATIONUNDEF, 1, db_collhdr.act,
ERR_GVIS, 2, gv_altkey->end - 1, gv_altkey->base);
mupip_exit(ERR_COLLATIONUNDEF);
}
}
need_xlation = TRUE;
} else
need_xlation = FALSE;
}
new_gvn = FALSE;
for (; rp < (rec_hdr*)btop; rp = (rec_hdr*)((unsigned char *)rp + rec_len))
{
GET_USHORT(rec_len, &rp->rsiz);
if (rec_len + (unsigned char *)rp > btop)
{
bin_call_db(ERR_COR, (INTPTR_T)rec_count, (INTPTR_T)global_key_count);
mu_gvis();
util_out_print(0, TRUE);
break;
}
cp1 = (unsigned char*)(rp + 1);
cp2 = gv_currkey->base + rp->cmpc;
current = 1;
for (;;)
{
last = current;
current = *cp2++ = *cp1++;
if (0 == last && 0 == current)
break;
if (cp1 > (unsigned char *)rp + rec_len ||
cp2 > (unsigned char *)gv_currkey + gv_currkey->top)
{
bin_call_db(ERR_COR, (INTPTR_T)rec_count, (INTPTR_T)global_key_count);
mu_gvis();
util_out_print(0, TRUE);
break;
}
}
if (mupip_error_occurred)
break;
gv_currkey->end = cp2 - gv_currkey->base - 1;
if (need_xlation)
{
assert(hdr_lvl >= '3');
assert(extr_collhdr.act || db_collhdr.act || extr_collhdr.nct || db_collhdr.nct ||
extr_std_null_coll != gv_cur_region->std_null_coll);
/* gv_currkey would have been modified/translated in the earlier put */
memcpy(gv_currkey->base, cmpc_str, next_cmpc);
next_rp = (rec_hdr *)((unsigned char*)rp + rec_len);
if ((unsigned char*)next_rp < btop)
{
next_cmpc = next_rp->cmpc;
assert(next_cmpc <= gv_currkey->end);
memcpy(cmpc_str, gv_currkey->base, next_cmpc);
} else
next_cmpc = 0;
/* length of the key might change (due to nct variation),
* so get a copy of the original key from the extract */
memcpy(dup_key_str, gv_currkey->base, gv_currkey->end + 1);
gvkey_char_ptr = dup_key_str;
while (*gvkey_char_ptr++)
;
gv_currkey->prev = 0;
gv_currkey->end = gvkey_char_ptr - dup_key_str;
assert(gv_keysize <= tmp_gvkey->top);
while (*gvkey_char_ptr)
{
/* get next subscript (in GT.M internal subsc format) */
subsc_len = 0;
tmp_ptr = src_buff;
while (*gvkey_char_ptr)
*tmp_ptr++ = *gvkey_char_ptr++;
subsc_len = tmp_ptr - src_buff;
src_buff[subsc_len] = '\0';
if (extr_collseq)
{
/* undo the extract time collation */
TREF(transform) = TRUE;
save_gv_target_collseq = gv_target->collseq;
gv_target->collseq = extr_collseq;
} else
TREF(transform) = FALSE;
/* convert the subscript to string format */
end_buff = gvsub2str(src_buff, dest_buff, FALSE);
/* transform the string to the current subsc format */
TREF(transform) = TRUE;
tmp_mval.mvtype = MV_STR;
tmp_mval.str.addr = (char *)dest_buff;
tmp_mval.str.len = INTCAST(end_buff - dest_buff);
tmp_gvkey->prev = 0;
tmp_gvkey->end = 0;
if (extr_collseq)
gv_target->collseq = save_gv_target_collseq;
mval2subsc(&tmp_mval, tmp_gvkey);
/* we now have the correctly transformed subscript */
tmp_key_ptr = gv_currkey->base + gv_currkey->end;
memcpy(tmp_key_ptr, tmp_gvkey->base, tmp_gvkey->end + 1);
gv_currkey->prev = gv_currkey->end;
gv_currkey->end += tmp_gvkey->end;
gvkey_char_ptr++;
}
if ( gv_cur_region->std_null_coll != extr_std_null_coll && gv_currkey->prev)
{
if (extr_std_null_coll == 0)
{
GTM2STDNULLCOLL(gv_currkey->base, gv_currkey->end);
} else
{
STD2GTMNULLCOLL(gv_currkey->base, gv_currkey->end);
}
}
}
if (gv_currkey->end >= max_key)
{
bin_call_db(ERR_COR, (INTPTR_T)rec_count, (INTPTR_T)global_key_count);
mu_gvis();
util_out_print(0, TRUE);
continue;
}
if (max_subsc_len < (gv_currkey->end + 1))
max_subsc_len = gv_currkey->end + 1;
v.str.addr = (char*)cp1;
v.str.len =INTCAST(rec_len - (cp1 - (unsigned char *)rp));
if (max_data_len < v.str.len)
max_data_len = v.str.len;
bin_call_db(BIN_PUT, (INTPTR_T)&v, 0);
if (mupip_error_occurred)
{
if (!mupip_DB_full)
{
bin_call_db(ERR_COR, (INTPTR_T)rec_count, (INTPTR_T)global_key_count);
util_out_print(0, TRUE);
}
break;
}
key_count++;
global_key_count++;
}
}
GTMCRYPT_ONLY(
if (NULL != hash_array)
free(hash_array);
)
void go_load(uint4 begin, uint4 end)
{
char *ptr;
int len, fmt, keylength, keystate;
uint4 iter, max_data_len, max_subsc_len, key_count, max_rec_size;
mstr src, des;
unsigned char *rec_buff, ch;
boolean_t utf8_extract, format_error = FALSE, hasht_ignored = FALSE, hasht_gbl = FALSE;
char *val_off;
int val_len, val_off1, val_len1;
boolean_t is_setextract;
gvinit();
max_rec_size = DEFAULT_MAX_REC_SIZE;
rec_buff = (unsigned char *)malloc(max_rec_size);
fmt = MU_FMT_ZWR; /* by default, the extract format is ZWR (not GO) */
len = file_input_get(&ptr);
if (mupip_error_occurred)
{
free(rec_buff);
return;
}
if (len >= 0)
{
util_out_print("!AD", TRUE, len, ptr);
utf8_extract = ((len >= STR_LIT_LEN(UTF8_NAME)) &&
(0 == MEMCMP_LIT(ptr + len - STR_LIT_LEN("UTF-8"), "UTF-8"))) ? TRUE : FALSE;
if ((utf8_extract && !gtm_utf8_mode) || (!utf8_extract && gtm_utf8_mode))
{ /* extract CHSET doesn't match $ZCHSET */
if (utf8_extract)
gtm_putmsg(VARLSTCNT(4) ERR_LOADINVCHSET, 2, LEN_AND_LIT("UTF-8"));
else
gtm_putmsg(VARLSTCNT(4) ERR_LOADINVCHSET, 2, LEN_AND_LIT("M"));
mupip_error_occurred = TRUE;
free(rec_buff);
return;
}
} else
mupip_exit(ERR_LOADFILERR);
len = file_input_get(&ptr);
if (mupip_error_occurred)
{
free(rec_buff);
return;
}
if (len >= 0)
{
util_out_print("!AD", TRUE, len, ptr);
fmt = (0 == memcmp(ptr + len - STR_LIT_LEN("ZWR"), "ZWR", STR_LIT_LEN("ZWR"))) ? MU_FMT_ZWR : MU_FMT_GO;
} else
mupip_exit(ERR_LOADFILERR);
if (begin < 3)
begin = 3;
for (iter = 3; iter < begin; iter++)
{
len = file_input_get(&ptr);
if (len < 0) /* The IO device has signalled an end of file */
{
gtm_putmsg(VARLSTCNT(3) ERR_LOADEOF, 1, begin);
mupip_error_occurred = TRUE;
}
if (mupip_error_occurred)
{
util_out_print("Error reading record number: !UL\n", TRUE, iter);
free(rec_buff);
return;
}
}
assert(iter == begin);
util_out_print("Beginning LOAD at record number: !UL\n", TRUE, begin);
max_data_len = 0;
max_subsc_len = 0;
key_count = 0;
for (iter = begin - 1; ; )
{
if (++iter > end)
break;
if (mu_ctrly_occurred)
break;
if (mu_ctrlc_occurred)
{
util_out_print("!AD:!_ Key cnt: !UL max subsc len: !UL max data len: !UL", TRUE,
LEN_AND_LIT(gt_lit), key_count, max_subsc_len, max_data_len);
util_out_print("Last LOAD record number: !UL", TRUE, key_count ? iter : 0);
mu_gvis();
util_out_print(0, TRUE);
mu_ctrlc_occurred = FALSE;
}
if (0 > (len = file_input_get(&ptr)))
break;
if (mupip_error_occurred)
{
mu_gvis();
break;
}
if ('\n' == *ptr)
{
if ('\n' == *(ptr+1))
break;
ptr++;
}
if (0 == len)
continue;
if (MU_FMT_GO != fmt)
{
/* Determine if the ZWR has $extract format */
if ('$' == *ptr)
{
keylength = zwrkeyvallen(ptr, len, &val_off, &val_len, &val_off1, &val_len1);
ptr = ptr + 4; /* Skip first 4 character '$','z','e','(' */
is_setextract = TRUE;
} else
{
/* Determine the ZWR key length. -1 (SIZEOF(=)) is needed since ZWR allows '^x(1,2)='*/
keylength = zwrkeyvallen(ptr, len, &val_off, &val_len, NULL, NULL);
is_setextract = FALSE;
}
ISSUE_TRIGDATAIGNORE_IF_NEEDED(keylength, ptr, hasht_gbl);
if (hasht_gbl)
{
hasht_gbl = FALSE;
continue;
}
go_call_db(GO_PUT_SUB, ptr, keylength, 0, 0);
if (mupip_error_occurred)
{
mu_gvis();
util_out_print("Error loading record number: !UL\n", TRUE, iter);
mupip_error_occurred = FALSE;
continue;
}
assert(keylength < len - 1);
if (max_subsc_len < (gv_currkey->end + 1))
max_subsc_len = gv_currkey->end + 1;
src.len = val_len;
src.addr = val_off;
des.len = 0;
if (src.len > max_rec_size)
{
max_rec_size = src.len;
free(rec_buff);
rec_buff = (unsigned char *)malloc(max_rec_size);
}
des.addr = (char *)rec_buff;
if (FALSE == zwr2format(&src, &des))
{
util_out_print("Format error in record number !8UL: !/!AD", TRUE, iter, src.len, src.addr);
format_error = TRUE;
continue;
}
if (max_data_len < des.len)
max_data_len = des.len;
(is_setextract) ? go_call_db(GO_SET_EXTRACT, des.addr, des.len, val_off1, val_len1)
: go_call_db(GO_PUT_DATA, (char *)rec_buff, des.len, 0, 0);
if (mupip_error_occurred)
{
mu_gvis();
util_out_print("Error loading record number: !UL\n", TRUE, iter);
mupip_error_occurred = FALSE;
continue;
}
key_count++;
} else
{
ISSUE_TRIGDATAIGNORE_IF_NEEDED(len, ptr, hasht_gbl);
if (hasht_gbl)
{
if (0 > (len = file_input_get(&ptr)))
break;
iter++;
hasht_gbl = FALSE;
continue;
}
go_call_db(GO_PUT_SUB, ptr, len, 0, 0);
if (mupip_error_occurred)
{
mu_gvis();
util_out_print("Error loading record number: !UL\n", TRUE, iter);
mupip_error_occurred = FALSE;
continue;
}
if (max_subsc_len < (gv_currkey->end + 1))
max_subsc_len = gv_currkey->end + 1;
if (++iter > end)
{
iter--; /* Decrement as didn't load key */
break;
}
if ((len = file_input_get(&ptr)) < 0)
break;
if (mupip_error_occurred)
{
mu_gvis();
util_out_print("Error loading record number: !UL\n", TRUE, iter);
break;
}
stringpool.free = stringpool.base;
if (max_data_len < len)
max_data_len = len;
go_call_db(GO_PUT_DATA, ptr, len, 0, 0);
if (mupip_error_occurred)
{
mu_gvis();
util_out_print("Error loading record number: !UL\n", TRUE, iter);
mupip_error_occurred = FALSE;
continue;
}
key_count++;
}
}
free(rec_buff);
file_input_close();
if (mu_ctrly_occurred)
{
gtm_putmsg(VARLSTCNT(1) ERR_LOADCTRLY);
mupip_exit(ERR_MUNOFINISH);
}
util_out_print("LOAD TOTAL!_!_Key Cnt: !UL Max Subsc Len: !UL Max Data Len: !UL",TRUE,key_count,max_subsc_len,
max_data_len);
util_out_print("Last LOAD record number: !UL\n", TRUE, key_count ? (iter - 1) : 0);
if (format_error)
mupip_exit(ERR_LOADFILERR);
}
void bin_load(uint4 begin, uint4 end)
{
unsigned char *ptr, *cp1, *cp2, *btop, *gvkey_char_ptr, *tmp_ptr, *tmp_key_ptr;
unsigned char hdr_lvl, src_buff[MAX_KEY_SZ + 1], dest_buff[MAX_ZWR_KEY_SZ],
cmpc_str[MAX_KEY_SZ + 1], dup_key_str[MAX_KEY_SZ + 1];
unsigned char *end_buff;
unsigned short len, rec_len, next_cmpc;
int current, last, length, max_blk_siz, max_key, status;
uint4 iter, max_data_len, max_subsc_len, key_count;
ssize_t rec_count, global_key_count, subsc_len,extr_std_null_coll;
boolean_t need_xlation, new_gvn, utf8_extract;
rec_hdr *rp, *next_rp;
mval v, tmp_mval;
mstr mstr_src, mstr_dest;
collseq *extr_collseq, *db_collseq, *save_gv_target_collseq;
coll_hdr extr_collhdr, db_collhdr;
gv_key *tmp_gvkey;
char std_null_coll[BIN_HEADER_NUMSZ + 1];
error_def(ERR_GVIS);
error_def(ERR_TEXT);
error_def(ERR_LDBINFMT);
error_def(ERR_LOADCTRLY);
error_def(ERR_LOADEOF);
error_def(ERR_MUNOFINISH);
error_def(ERR_COLLTYPVERSION);
error_def(ERR_COLLATIONUNDEF);
error_def(ERR_OLDBINEXTRACT);
error_def(ERR_LOADINVCHSET);
tmp_gvkey = (gv_key *)malloc(sizeof(gv_key) + MAX_KEY_SZ - 1);
assert(4 == sizeof(coll_hdr));
gvinit();
v.mvtype = MV_STR;
len = mu_bin_get((char **)&ptr);
hdr_lvl = EXTR_HEADER_LEVEL(ptr);
if (!((hdr_lvl == '4' && len == BIN_HEADER_SZ) || (hdr_lvl < '4' && len == V3_BIN_HEADER_SZ)))
{
rts_error(VARLSTCNT(1) ERR_LDBINFMT);
mupip_exit(ERR_LDBINFMT);
}
/* assert the assumption that the level can be represented in a single character */
assert(' ' == *(ptr + sizeof(BIN_HEADER_LABEL) - 3));
if (0 != memcmp(ptr, BIN_HEADER_LABEL, sizeof(BIN_HEADER_LABEL) - 2) || hdr_lvl < '2' || *(BIN_HEADER_VERSION) < hdr_lvl)
{ /* ignore the level check */
rts_error(VARLSTCNT(1) ERR_LDBINFMT);
mupip_exit(ERR_LDBINFMT);
}
/* check if extract was generated in UTF-8 mode */
utf8_extract = (0 == MEMCMP_LIT(&ptr[len - BIN_HEADER_LABELSZ], UTF8_NAME)) ? TRUE : FALSE;
if ((utf8_extract && !gtm_utf8_mode) || (!utf8_extract && gtm_utf8_mode))
{ /* extract CHSET doesn't match $ZCHSET */
if (utf8_extract)
rts_error(VARLSTCNT(4) ERR_LOADINVCHSET, 2, LEN_AND_LIT("UTF-8"));
else
rts_error(VARLSTCNT(4) ERR_LOADINVCHSET, 2, LEN_AND_LIT("M"));
mupip_exit(ERR_LDBINFMT);
}
util_out_print("Label = !AD\n", TRUE, len, ptr);
new_gvn = FALSE;
if (hdr_lvl > '3')
{
memcpy(std_null_coll, ptr + BIN_HEADER_NULLCOLLOFFSET, BIN_HEADER_NUMSZ);
std_null_coll[BIN_HEADER_NUMSZ] = '\0';
extr_std_null_coll = STRTOUL(std_null_coll, NULL, 10);
if (0 != extr_std_null_coll && 1!= extr_std_null_coll)
{
rts_error(VARLSTCNT(5) ERR_TEXT, 2, RTS_ERROR_TEXT("Corrupted null collation field in header"),
ERR_LDBINFMT);
mupip_exit(ERR_LDBINFMT);
}
} else
extr_std_null_coll = 0;
if (hdr_lvl > '2')
{
len = mu_bin_get((char **)&ptr);
if (sizeof(coll_hdr) != len)
{
rts_error(VARLSTCNT(5) ERR_TEXT, 2, RTS_ERROR_TEXT("Corrupt collation header"), ERR_LDBINFMT);
mupip_exit(ERR_LDBINFMT);
}
extr_collhdr = *((coll_hdr *)(ptr));
new_gvn = TRUE;
} else
gtm_putmsg(VARLSTCNT(3) ERR_OLDBINEXTRACT, 1, hdr_lvl - '0');
if (begin < 2)
begin = 2;
for (iter = 2; iter < begin; iter++)
{
if (!(len = mu_bin_get((char **)&ptr)))
{
gtm_putmsg(VARLSTCNT(3) ERR_LOADEOF, 1, begin);
util_out_print("Error reading record number: !UL\n", TRUE, iter);
mupip_error_occurred = TRUE;
return;
} else if (len == sizeof(coll_hdr))
{
extr_collhdr = *((coll_hdr *)(ptr));
assert(hdr_lvl > '2');
iter--;
}
}
assert(iter == begin);
util_out_print("Beginning LOAD at record number: !UL\n", TRUE, begin);
max_data_len = 0;
max_subsc_len = 0;
key_count = 0;
rec_count = begin - 1;
extr_collseq = db_collseq = NULL;
need_xlation = FALSE;
for (; !mupip_DB_full ;)
{
if (++rec_count > end)
break;
next_cmpc = 0;
mupip_error_occurred = FALSE;
if (mu_ctrly_occurred)
break;
if (mu_ctrlc_occurred)
{
util_out_print("!AD:!_ Key cnt: !UL max subsc len: !UL max data len: !UL", TRUE,
LEN_AND_LIT(gt_lit), key_count, max_subsc_len, max_data_len);
util_out_print("Last LOAD record number: !UL", TRUE, key_count ? (rec_count - 1) : 0);
mu_gvis();
util_out_print(0, TRUE);
mu_ctrlc_occurred = FALSE;
}
/* reset the stringpool for every record in order to avoid garbage collection */
stringpool.free = stringpool.base;
if (!(len = mu_bin_get((char **)&ptr)) || mupip_error_occurred)
break;
else if (len == sizeof(coll_hdr))
{
extr_collhdr = *((coll_hdr *)(ptr));
assert(hdr_lvl > '2');
new_gvn = TRUE; /* next record will contain a new gvn */
rec_count--; /* Decrement as this record does not count as a record for loading purposes */
continue;
}
global_key_count = 1;
rp = (rec_hdr*)ptr;
btop = ptr + len;
cp1 = (unsigned char*)(rp + 1);
v.str.addr = (char*)cp1;
while (*cp1++) ;
v.str.len =INTCAST((char*)cp1 - v.str.addr - 1);
if (hdr_lvl <= '2' || new_gvn)
{
bin_call_db(BIN_BIND, (INTPTR_T)gd_header, (INTPTR_T)&v.str);
max_key = gv_cur_region->max_key_size;
db_collhdr.act = gv_target->act;
db_collhdr.ver = gv_target->ver;
db_collhdr.nct = gv_target->nct;
}
GET_SHORT(rec_len, &rp->rsiz);
if (rp->cmpc != 0 || v.str.len > rec_len || mupip_error_occurred)
{
bin_call_db(ERR_COR, rec_count, global_key_count);
mu_gvis();
util_out_print(0, TRUE);
continue;
}
if (new_gvn)
{
if ((db_collhdr.act != extr_collhdr.act || db_collhdr.ver != extr_collhdr.ver
|| db_collhdr.nct != extr_collhdr.nct
|| gv_cur_region->std_null_coll != extr_std_null_coll))
{
if (extr_collhdr.act)
{
if (extr_collseq = ready_collseq((int)extr_collhdr.act))
{
if (!do_verify(extr_collseq, extr_collhdr.act, extr_collhdr.ver))
{
gtm_putmsg(VARLSTCNT(8) ERR_COLLTYPVERSION, 2, extr_collhdr.act,
extr_collhdr.ver, ERR_GVIS, 2, gv_altkey->end - 1, gv_altkey->base);
mupip_exit(ERR_COLLTYPVERSION);
}
} else
{
gtm_putmsg(VARLSTCNT(7) ERR_COLLATIONUNDEF, 1, extr_collhdr.act,
ERR_GVIS, 2, gv_altkey->end - 1, gv_altkey->base);
mupip_exit(ERR_COLLATIONUNDEF);
}
}
if (db_collhdr.act)
{
if (db_collseq = ready_collseq((int)db_collhdr.act))
{
if (!do_verify(db_collseq, db_collhdr.act, db_collhdr.ver))
{
gtm_putmsg(VARLSTCNT(8) ERR_COLLTYPVERSION, 2, db_collhdr.act,
db_collhdr.ver, ERR_GVIS, 2, gv_altkey->end - 1, gv_altkey->base);
mupip_exit(ERR_COLLTYPVERSION);
}
} else
{
gtm_putmsg(VARLSTCNT(7) ERR_COLLATIONUNDEF, 1, db_collhdr.act,
ERR_GVIS, 2, gv_altkey->end - 1, gv_altkey->base);
mupip_exit(ERR_COLLATIONUNDEF);
}
}
need_xlation = TRUE;
} else
need_xlation = FALSE;
}
new_gvn = FALSE;
for (; rp < (rec_hdr*)btop; rp = (rec_hdr*)((unsigned char *)rp + rec_len))
{
GET_SHORT(rec_len, &rp->rsiz);
if (rec_len + (unsigned char *)rp > btop)
{
bin_call_db(ERR_COR, rec_count, global_key_count);
mu_gvis();
util_out_print(0, TRUE);
break;
}
cp1 = (unsigned char*)(rp + 1);
cp2 = gv_currkey->base + rp->cmpc;
current = 1;
for (;;)
{
last = current;
current = *cp2++ = *cp1++;
if (0 == last && 0 == current)
break;
if (cp1 > (unsigned char *) rp + rec_len ||
cp2 > (unsigned char *) gv_currkey + gv_currkey->top)
{
bin_call_db(ERR_COR, rec_count, global_key_count);
mu_gvis();
util_out_print(0, TRUE);
break;
}
}
if (mupip_error_occurred)
break;
gv_currkey->end = cp2 - gv_currkey->base - 1;
if (need_xlation)
{
assert(hdr_lvl >= '3');
assert(extr_collhdr.act || db_collhdr.act || extr_collhdr.nct || db_collhdr.nct ||
extr_std_null_coll != gv_cur_region->std_null_coll);
/* gv_currkey would have been modified/translated in the earlier put */
memcpy(gv_currkey->base, cmpc_str, next_cmpc);
next_rp = (rec_hdr *)((unsigned char*)rp + rec_len);
if ((unsigned char*)next_rp < btop)
{
next_cmpc = next_rp->cmpc;
assert(next_cmpc <= gv_currkey->end);
memcpy(cmpc_str, gv_currkey->base, next_cmpc);
} else
next_cmpc = 0;
/* length of the key might change (due to nct variation),
* so get a copy of the original key from the extract */
memcpy(dup_key_str, gv_currkey->base, gv_currkey->end + 1);
gvkey_char_ptr = dup_key_str;
while (*gvkey_char_ptr++) ;
gv_currkey->prev = 0;
gv_currkey->end = gvkey_char_ptr - dup_key_str;
tmp_gvkey->top = gv_keysize;
while (*gvkey_char_ptr)
{
/* get next subscript (in GT.M internal subsc format) */
subsc_len = 0;
tmp_ptr = src_buff;
while (*gvkey_char_ptr)
*tmp_ptr++ = *gvkey_char_ptr++;
subsc_len = tmp_ptr - src_buff;
src_buff[subsc_len] = '\0';
if (extr_collseq)
{
/* undo the extract time collation */
transform = TRUE;
save_gv_target_collseq = gv_target->collseq;
gv_target->collseq = extr_collseq;
} else
transform = FALSE;
/* convert the subscript to string format */
end_buff = gvsub2str(src_buff, dest_buff, FALSE);
/* transform the string to the current subsc format */
transform = TRUE;
tmp_mval.mvtype = MV_STR;
tmp_mval.str.addr = (char *)dest_buff;
tmp_mval.str.len = INTCAST(end_buff - dest_buff);
tmp_gvkey->prev = 0;
tmp_gvkey->end = 0;
if (extr_collseq)
gv_target->collseq = save_gv_target_collseq;
mval2subsc(&tmp_mval, tmp_gvkey);
/* we now have the correctly transformed subscript */
tmp_key_ptr = gv_currkey->base + gv_currkey->end;
memcpy(tmp_key_ptr, tmp_gvkey->base, tmp_gvkey->end + 1);
gv_currkey->prev = gv_currkey->end;
gv_currkey->end += tmp_gvkey->end;
gvkey_char_ptr++;
}
if ( gv_cur_region->std_null_coll != extr_std_null_coll && gv_currkey->prev)
{
if (extr_std_null_coll == 0)
{
GTM2STDNULLCOLL(gv_currkey->base, gv_currkey->end);
} else
{
STD2GTMNULLCOLL(gv_currkey->base, gv_currkey->end);
}
}
}
if (gv_currkey->end >= max_key)
{
bin_call_db(ERR_COR, rec_count, global_key_count);
mu_gvis();
util_out_print(0, TRUE);
continue;
}
if (max_subsc_len < (gv_currkey->end + 1))
max_subsc_len = gv_currkey->end + 1;
v.str.addr = (char*)cp1;
v.str.len =INTCAST(rec_len - (cp1 - (unsigned char *)rp) );
if (max_data_len < v.str.len)
max_data_len = v.str.len;
bin_call_db(BIN_PUT, (INTPTR_T)&v, 0);
if (mupip_error_occurred)
{
if (!mupip_DB_full)
{
bin_call_db(ERR_COR, rec_count, global_key_count);
util_out_print(0, TRUE);
}
break;
}
key_count++;
global_key_count++;
}
}
free(tmp_gvkey);
mu_load_close();
util_out_print("LOAD TOTAL!_!_Key Cnt: !UL Max Subsc Len: !UL Max Data Len: !UL", TRUE, key_count, max_subsc_len,
max_data_len);
util_out_print("Last LOAD record number: !UL\n", TRUE, key_count ? (rec_count - 1) : 0);
if (mu_ctrly_occurred)
{
gtm_putmsg(VARLSTCNT(1) ERR_LOADCTRLY);
mupip_exit(ERR_MUNOFINISH);
}
}
void bin_load(uint4 begin, uint4 end)
{
unsigned char *ptr, *cp1, *cp2, *btop, *gvkey_char_ptr, *tmp_ptr, *tmp_key_ptr, *c, *ctop, *ptr_base;
unsigned char hdr_lvl, src_buff[MAX_KEY_SZ + 1], dest_buff[MAX_ZWR_KEY_SZ],
cmpc_str[MAX_KEY_SZ + 1], dup_key_str[MAX_KEY_SZ + 1], sn_key_str[MAX_KEY_SZ + 1], *sn_key_str_end;
unsigned char *end_buff;
unsigned short rec_len, next_cmpc, numsubs;
int len;
int current, last, length, max_blk_siz, max_key, status;
int tmp_cmpc, sn_chunk_number, expected_sn_chunk_number = 0, sn_hold_buff_pos, sn_hold_buff_size;
uint4 iter, max_data_len, max_subsc_len, key_count, gblsize;
ssize_t rec_count, global_key_count, subsc_len,extr_std_null_coll, last_sn_error_offset=0,
file_offset_base=0, file_offset=0;
boolean_t need_xlation, new_gvn, utf8_extract;
boolean_t is_hidden_subscript, ok_to_put = TRUE, putting_a_sn = FALSE, sn_incmp_gbl_already_killed = FALSE;
rec_hdr *rp, *next_rp;
mval v, tmp_mval;
mstr mstr_src, mstr_dest;
collseq *extr_collseq, *db_collseq, *save_gv_target_collseq;
coll_hdr extr_collhdr, db_collhdr;
gv_key *tmp_gvkey = NULL; /* null-initialize at start, will be malloced later */
gv_key *sn_gvkey = NULL; /* null-initialize at start, will be malloced later */
gv_key *sn_savekey = NULL; /* null-initialize at start, will be malloced later */
char std_null_coll[BIN_HEADER_NUMSZ + 1], *sn_hold_buff = NULL, *sn_hold_buff_temp = NULL;
# ifdef GTM_CRYPT
gtmcrypt_key_t *encr_key_handles;
char *inbuf;
int4 index;
int req_dec_blk_size, init_status, crypt_status;
muext_hash_hdr_ptr_t hash_array = NULL;
# endif
DCL_THREADGBL_ACCESS;
SETUP_THREADGBL_ACCESS;
assert(4 == SIZEOF(coll_hdr));
gvinit();
v.mvtype = MV_STR;
len = file_input_bin_get((char **)&ptr, &file_offset_base, (char **)&ptr_base);
hdr_lvl = EXTR_HEADER_LEVEL(ptr);
if (!(((('4' == hdr_lvl) || ('5' == hdr_lvl)) && (V5_BIN_HEADER_SZ == len)) ||
(('6' == hdr_lvl) && (BIN_HEADER_SZ == len)) ||
(('7' == hdr_lvl) && (BIN_HEADER_SZ == len)) ||
(('4' > hdr_lvl) && (V3_BIN_HEADER_SZ == len))))
{
rts_error(VARLSTCNT(1) ERR_LDBINFMT);
mupip_exit(ERR_LDBINFMT);
}
/* expecting the level in a single character */
assert(' ' == *(ptr + SIZEOF(BIN_HEADER_LABEL) - 3));
if (0 != memcmp(ptr, BIN_HEADER_LABEL, SIZEOF(BIN_HEADER_LABEL) - 2) || ('2' > hdr_lvl) ||
*(BIN_HEADER_VERSION_ENCR) < hdr_lvl)
{ /* ignore the level check */
rts_error(VARLSTCNT(1) ERR_LDBINFMT);
mupip_exit(ERR_LDBINFMT);
}
/* check if extract was generated in UTF-8 mode */
utf8_extract = (0 == MEMCMP_LIT(&ptr[len - BIN_HEADER_LABELSZ], UTF8_NAME)) ? TRUE : FALSE;
if ((utf8_extract && !gtm_utf8_mode) || (!utf8_extract && gtm_utf8_mode))
{ /* extract CHSET doesn't match $ZCHSET */
if (utf8_extract)
rts_error(VARLSTCNT(4) ERR_LOADINVCHSET, 2, LEN_AND_LIT("UTF-8"));
else
rts_error(VARLSTCNT(4) ERR_LOADINVCHSET, 2, LEN_AND_LIT("M"));
mupip_exit(ERR_LDBINFMT);
}
if ('4' >= hdr_lvl)
{ /* Binary extracts in V50000-to-V52000 (label=4) and pre-V50000 (label=3) could have a '\0' byte (NULL byte)
* in the middle of the string. Replace it with ' ' (space) like it would be in V52000 binary extracts and above.
*/
for (c = ptr, ctop = c + len; c < ctop; c++)
{
if ('\0' == *c)
*c = ' ';
}
}
util_out_print("Label = !AD\n", TRUE, len, ptr);
new_gvn = FALSE;
if (hdr_lvl > '3')
{
if (hdr_lvl > '5')
{
memcpy(std_null_coll, ptr + BIN_HEADER_NULLCOLLOFFSET, BIN_HEADER_NUMSZ);
std_null_coll[BIN_HEADER_NUMSZ] = '\0';
}
else
{
memcpy(std_null_coll, ptr + V5_BIN_HEADER_NULLCOLLOFFSET, V5_BIN_HEADER_NUMSZ);
std_null_coll[V5_BIN_HEADER_NUMSZ] = '\0';
}
extr_std_null_coll = STRTOUL(std_null_coll, NULL, 10);
if (0 != extr_std_null_coll && 1!= extr_std_null_coll)
{
rts_error(VARLSTCNT(5) ERR_TEXT, 2, RTS_ERROR_TEXT("Corrupted null collation field in header"),
ERR_LDBINFMT);
mupip_exit(ERR_LDBINFMT);
}
} else
extr_std_null_coll = 0;
# ifdef GTM_CRYPT
if ('7' <= hdr_lvl)
{
int i, num_indexes;
len = file_input_bin_get((char **)&ptr, &file_offset_base, (char **)&ptr_base);
hash_array = (muext_hash_hdr *)malloc(len);
/* store hashes of all the files used during extract into muext_hash_hdr structure */
memcpy((char *)hash_array, ptr, len);
num_indexes = len / GTMCRYPT_HASH_LEN;
encr_key_handles = (gtmcrypt_key_t *)malloc(SIZEOF(gtmcrypt_key_t) * num_indexes);
INIT_PROC_ENCRYPTION(crypt_status);
GC_BIN_LOAD_ERR(crypt_status);
for (index = 0; index < num_indexes; index++)
{
if (0 == memcmp(hash_array[index].gtmcrypt_hash, EMPTY_GTMCRYPT_HASH, GTMCRYPT_HASH_LEN))
continue;
GTMCRYPT_GETKEY(hash_array[index].gtmcrypt_hash, encr_key_handles[index], crypt_status);
GC_BIN_LOAD_ERR(crypt_status);
}
}
# endif
if ('2' < hdr_lvl)
{
len = file_input_bin_get((char **)&ptr, &file_offset_base, (char **)&ptr_base);
if (SIZEOF(coll_hdr) != len)
{
rts_error(VARLSTCNT(5) ERR_TEXT, 2, RTS_ERROR_TEXT("Corrupt collation header"), ERR_LDBINFMT);
mupip_exit(ERR_LDBINFMT);
}
extr_collhdr = *((coll_hdr *)(ptr));
new_gvn = TRUE;
} else
gtm_putmsg(VARLSTCNT(3) ERR_OLDBINEXTRACT, 1, hdr_lvl - '0');
if (begin < 2)
begin = 2;
for (iter = 2; iter < begin; iter++)
{
if (!(len = file_input_bin_get((char **)&ptr, &file_offset_base, (char **)&ptr_base)))
{
gtm_putmsg(VARLSTCNT(3) ERR_LOADEOF, 1, begin);
util_out_print("Error reading record number: !UL\n", TRUE, iter);
mupip_error_occurred = TRUE;
return;
} else if (len == SIZEOF(coll_hdr))
{
extr_collhdr = *((coll_hdr *)(ptr));
assert(hdr_lvl > '2');
iter--;
}
}
assert(iter == begin);
util_out_print("Beginning LOAD at record number: !UL\n", TRUE, begin);
max_data_len = 0;
max_subsc_len = 0;
global_key_count = key_count = 0;
rec_count = begin - 1;
extr_collseq = db_collseq = NULL;
need_xlation = FALSE;
assert(NULL == tmp_gvkey); /* GVKEY_INIT macro relies on this */
GVKEY_INIT(tmp_gvkey, DBKEYSIZE(MAX_KEY_SZ)); /* tmp_gvkey will point to malloced memory after this */
assert(NULL == sn_gvkey); /* GVKEY_INIT macro relies on this */
GVKEY_INIT(sn_gvkey, DBKEYSIZE(MAX_KEY_SZ)); /* sn_gvkey will point to malloced memory after this */
assert(NULL == sn_savekey); /* GVKEY_INIT macro relies on this */
GVKEY_INIT(sn_savekey, DBKEYSIZE(MAX_KEY_SZ)); /* sn_gvkey will point to malloced memory after this */
for (; !mupip_DB_full ;)
{
if (++rec_count > end)
break;
next_cmpc = 0;
mupip_error_occurred = FALSE;
if (mu_ctrly_occurred)
break;
if (mu_ctrlc_occurred)
{
util_out_print("!AD:!_ Key cnt: !UL max subsc len: !UL max data len: !UL", TRUE,
LEN_AND_LIT(gt_lit), key_count, max_subsc_len, max_data_len);
util_out_print("Last LOAD record number: !UL", TRUE, key_count ? (rec_count - 1) : 0);
mu_gvis();
util_out_print(0, TRUE);
mu_ctrlc_occurred = FALSE;
}
if (!(len = file_input_bin_get((char **)&ptr, &file_offset_base, (char **)&ptr_base)) || mupip_error_occurred)
break;
else if (len == SIZEOF(coll_hdr))
{
extr_collhdr = *((coll_hdr *)(ptr));
assert(hdr_lvl > '2');
new_gvn = TRUE; /* next record will contain a new gvn */
rec_count--; /* Decrement as this record does not count as a record for loading purposes */
continue;
}
rp = (rec_hdr*)(ptr);
# ifdef GTM_CRYPT
if ('7' <= hdr_lvl)
{ /* Getting index value from the extracted file. It indicates which database file this record belongs to */
GET_LONG(index, ptr);
if (-1 != index) /* Indicates that the record is encrypted. */
{
req_dec_blk_size = len - SIZEOF(int4);
inbuf = (char *)(ptr + SIZEOF(int4));
GTMCRYPT_DECODE_FAST(encr_key_handles[index], inbuf, req_dec_blk_size, NULL, crypt_status);
GC_BIN_LOAD_ERR(crypt_status);
}
rp = (rec_hdr*)(ptr + SIZEOF(int4));
}
# endif
btop = ptr + len;
cp1 = (unsigned char*)(rp + 1);
v.str.addr = (char*)cp1;
while (*cp1++)
;
v.str.len =INTCAST((char*)cp1 - v.str.addr - 1);
if (('2' >= hdr_lvl) || new_gvn)
{
if ((HASHT_GBLNAME_LEN == v.str.len) && (0 == memcmp(v.str.addr, HASHT_GBLNAME, HASHT_GBLNAME_LEN)))
continue;
bin_call_db(BIN_BIND, (INTPTR_T)gd_header, (INTPTR_T)&v.str);
max_key = gv_cur_region->max_key_size;
db_collhdr.act = gv_target->act;
db_collhdr.ver = gv_target->ver;
db_collhdr.nct = gv_target->nct;
}
GET_USHORT(rec_len, &rp->rsiz);
if (EVAL_CMPC(rp) != 0 || v.str.len > rec_len || mupip_error_occurred)
{
bin_call_db(ERR_COR, (INTPTR_T)rec_count, (INTPTR_T)global_key_count);
mu_gvis();
DISPLAY_FILE_OFFSET_OF_RECORD_AND_REST_OF_BLOCK;
continue;
}
if (new_gvn)
{
global_key_count = 1;
if ((db_collhdr.act != extr_collhdr.act || db_collhdr.ver != extr_collhdr.ver
|| db_collhdr.nct != extr_collhdr.nct
|| gv_cur_region->std_null_coll != extr_std_null_coll))
{
if (extr_collhdr.act)
{
if (extr_collseq = ready_collseq((int)extr_collhdr.act))
{
if (!do_verify(extr_collseq, extr_collhdr.act, extr_collhdr.ver))
{
gtm_putmsg(VARLSTCNT(8) ERR_COLLTYPVERSION, 2, extr_collhdr.act,
extr_collhdr.ver, ERR_GVIS, 2, gv_altkey->end - 1, gv_altkey->base);
mupip_exit(ERR_COLLTYPVERSION);
}
} else
{
gtm_putmsg(VARLSTCNT(7) ERR_COLLATIONUNDEF, 1, extr_collhdr.act,
ERR_GVIS, 2, gv_altkey->end - 1, gv_altkey->base);
mupip_exit(ERR_COLLATIONUNDEF);
}
}
if (db_collhdr.act)
{
if (db_collseq = ready_collseq((int)db_collhdr.act))
{
if (!do_verify(db_collseq, db_collhdr.act, db_collhdr.ver))
{
gtm_putmsg(VARLSTCNT(8) ERR_COLLTYPVERSION, 2, db_collhdr.act,
db_collhdr.ver, ERR_GVIS, 2, gv_altkey->end - 1, gv_altkey->base);
mupip_exit(ERR_COLLTYPVERSION);
}
} else
{
gtm_putmsg(VARLSTCNT(7) ERR_COLLATIONUNDEF, 1, db_collhdr.act,
ERR_GVIS, 2, gv_altkey->end - 1, gv_altkey->base);
mupip_exit(ERR_COLLATIONUNDEF);
}
}
need_xlation = TRUE;
} else
need_xlation = FALSE;
}
new_gvn = FALSE;
for (; rp < (rec_hdr*)btop; rp = (rec_hdr*)((unsigned char *)rp + rec_len))
{
GET_USHORT(rec_len, &rp->rsiz);
if (rec_len + (unsigned char *)rp > btop)
{
bin_call_db(ERR_COR, (INTPTR_T)rec_count, (INTPTR_T)global_key_count);
mu_gvis();
DISPLAY_FILE_OFFSET_OF_RECORD_AND_REST_OF_BLOCK;
break;
}
cp1 = (unsigned char*)(rp + 1);
cp2 = gv_currkey->base + EVAL_CMPC(rp);
current = 1;
for (;;)
{
last = current;
current = *cp2++ = *cp1++;
if (0 == last && 0 == current)
break;
if (cp1 > (unsigned char *)rp + rec_len ||
cp2 > (unsigned char *)gv_currkey + gv_currkey->top)
{
gv_currkey->end = cp2 - gv_currkey->base - 1;
gv_currkey->base[gv_currkey->end] = 0;
gv_currkey->base[gv_currkey->end - 1] = 0;
bin_call_db(ERR_COR, (INTPTR_T)rec_count, (INTPTR_T)global_key_count);
mu_gvis();
DISPLAY_FILE_OFFSET_OF_RECORD_AND_REST_OF_BLOCK;
break;
}
}
if (mupip_error_occurred)
break;
gv_currkey->end = cp2 - gv_currkey->base - 1;
if (need_xlation)
{
assert(hdr_lvl >= '3');
assert(extr_collhdr.act || db_collhdr.act || extr_collhdr.nct || db_collhdr.nct ||
extr_std_null_coll != gv_cur_region->std_null_coll);
/* gv_currkey would have been modified/translated in the earlier put */
memcpy(gv_currkey->base, cmpc_str, next_cmpc);
next_rp = (rec_hdr *)((unsigned char*)rp + rec_len);
if ((unsigned char*)next_rp < btop)
{
next_cmpc = EVAL_CMPC(next_rp);
assert(next_cmpc <= gv_currkey->end);
memcpy(cmpc_str, gv_currkey->base, next_cmpc);
} else
next_cmpc = 0;
/* length of the key might change (due to nct variation),
* so get a copy of the original key from the extract */
memcpy(dup_key_str, gv_currkey->base, gv_currkey->end + 1);
gvkey_char_ptr = dup_key_str;
while (*gvkey_char_ptr++)
;
gv_currkey->prev = 0;
gv_currkey->end = gvkey_char_ptr - dup_key_str;
assert(gv_keysize <= tmp_gvkey->top);
while (*gvkey_char_ptr)
{
/* get next subscript (in GT.M internal subsc format) */
subsc_len = 0;
tmp_ptr = src_buff;
while (*gvkey_char_ptr)
*tmp_ptr++ = *gvkey_char_ptr++;
subsc_len = tmp_ptr - src_buff;
src_buff[subsc_len] = '\0';
if (extr_collseq)
{
/* undo the extract time collation */
TREF(transform) = TRUE;
save_gv_target_collseq = gv_target->collseq;
gv_target->collseq = extr_collseq;
} else
TREF(transform) = FALSE;
/* convert the subscript to string format */
end_buff = gvsub2str(src_buff, dest_buff, FALSE);
/* transform the string to the current subsc format */
TREF(transform) = TRUE;
tmp_mval.mvtype = MV_STR;
tmp_mval.str.addr = (char *)dest_buff;
tmp_mval.str.len = INTCAST(end_buff - dest_buff);
tmp_gvkey->prev = 0;
tmp_gvkey->end = 0;
if (extr_collseq)
gv_target->collseq = save_gv_target_collseq;
mval2subsc(&tmp_mval, tmp_gvkey);
/* we now have the correctly transformed subscript */
tmp_key_ptr = gv_currkey->base + gv_currkey->end;
memcpy(tmp_key_ptr, tmp_gvkey->base, tmp_gvkey->end + 1);
gv_currkey->prev = gv_currkey->end;
gv_currkey->end += tmp_gvkey->end;
gvkey_char_ptr++;
}
if ( gv_cur_region->std_null_coll != extr_std_null_coll && gv_currkey->prev)
{
if (extr_std_null_coll == 0)
{
GTM2STDNULLCOLL(gv_currkey->base, gv_currkey->end);
} else
{
STD2GTMNULLCOLL(gv_currkey->base, gv_currkey->end);
}
}
}
if (gv_currkey->end >= max_key)
{
bin_call_db(ERR_COR, (INTPTR_T)rec_count, (INTPTR_T)global_key_count);
mu_gvis();
DISPLAY_FILE_OFFSET_OF_RECORD_AND_REST_OF_BLOCK;
continue;
}
/*
* Spanning node-related variables and their usage:
*
* expected_sn_chunk_number: 0 - looking for spanning nodes (regular nodes are OK, too)
* !0 - number of the next chunk needed (implies we are building
* a spanning node's value)
*
* While building a spanning node's value:
* numsubs: the number of chunks needed to build the spanning node's value
* gblsize: the expected size of the completed value
* sn_chunk_number: The chunk number of the chunk from the current record from the extract
*
* Managing the value
* sn_hold_buff: buffer used to accumulate the spanning node's value
* sn_hold_buff_size: Allocated size of buffer
* sn_hold_buff_pos: amount of the buffer used; where to place the next chunk
* sn_hold_buff_temp: used when we have to increase the size of the buffer
*
* Controlling the placing of the key,value in the database:
* ok_to_put: means we are ready to place the key,value in the database, i.e., we have the full value
* (either of the spanning node or a regular node).
* putting_a_sn: we are placing a spanning node in the database, i.e, use the key from sn_gvkey and
* the value from sn_hold_buff.
*/
CHECK_HIDDEN_SUBSCRIPT(gv_currkey,is_hidden_subscript);
if (!is_hidden_subscript && (max_subsc_len < (gv_currkey->end + 1)))
max_subsc_len = gv_currkey->end + 1;
v.str.addr = (char*)cp1;
v.str.len =INTCAST(rec_len - (cp1 - (unsigned char *)rp));
if (expected_sn_chunk_number && !is_hidden_subscript)
{ /* we were expecting a chunk of an spanning node and we did not get one */
DISPLAY_INCMP_SN_MSG;
util_out_print("!_!_Expected chunk number : !UL but found a non-spanning node", TRUE,
expected_sn_chunk_number + 1);
if (sn_hold_buff_pos)
DISPLAY_PARTIAL_SN_HOLD_BUFF;
KILL_INCMP_SN_IF_NEEDED;
sn_hold_buff_pos = 0;
expected_sn_chunk_number = 0;
ok_to_put = TRUE;
putting_a_sn = FALSE;
numsubs = 0;
}
if (is_hidden_subscript)
{ /* it's a chunk and we were expecting one */
sn_chunk_number = SPAN_GVSUBS2INT((span_subs *) &(gv_currkey->base[gv_currkey->end - 4]));
if (!expected_sn_chunk_number && is_hidden_subscript && sn_chunk_number)
{ /* we not expecting a payload chunk (as opposed to a control record) but we got one */
DISPLAY_INCMP_SN_MSG;
util_out_print("!_!_Not expecting a spanning node chunk but found chunk : !UL", TRUE,
sn_chunk_number + 1);
if (v.str.len)
DISPLAY_VALUE("!_!_Errant Chunk :");
continue;
}
if (0 == sn_chunk_number)
{ /* first spanning node chunk, get ctrl info */
if (0 != expected_sn_chunk_number)
{
DISPLAY_INCMP_SN_MSG;
util_out_print("!_!_Expected chunk number : !UL but found chunk number : !UL", TRUE,
expected_sn_chunk_number + 1, sn_chunk_number + 1);
if (sn_hold_buff_pos)
DISPLAY_PARTIAL_SN_HOLD_BUFF;
KILL_INCMP_SN_IF_NEEDED;
}
/* start building a new spanning node */
sn_gvkey->end = gv_currkey->end - (SPAN_SUBS_LEN + 1);
memcpy(sn_gvkey->base, gv_currkey->base, sn_gvkey->end);
sn_gvkey->base[sn_gvkey->end] = 0;
sn_gvkey->prev = gv_currkey->prev;
sn_gvkey->top = gv_currkey->top;
GET_NSBCTRL(v.str.addr, numsubs, gblsize);
/* look for first payload chunk */
expected_sn_chunk_number = 1;
sn_hold_buff_pos = 0;
ok_to_put = FALSE;
sn_incmp_gbl_already_killed = FALSE;
} else
{ /* we only need to compare the key before the hidden subscripts */
if ((expected_sn_chunk_number == sn_chunk_number)
&& (sn_gvkey->end == gv_currkey->end - (SPAN_SUBS_LEN + 1))
&& !memcmp(sn_gvkey->base,gv_currkey->base, sn_gvkey->end)
&& ((sn_hold_buff_pos + v.str.len) <= gblsize))
{
if (NULL == sn_hold_buff)
{
sn_hold_buff_size = DEFAULT_SN_HOLD_BUFF_SIZE;
sn_hold_buff = (char *)malloc(DEFAULT_SN_HOLD_BUFF_SIZE);
}
if ((sn_hold_buff_pos + v.str.len) > sn_hold_buff_size)
{
sn_hold_buff_size = sn_hold_buff_size * 2;
sn_hold_buff_temp = (char *)malloc(sn_hold_buff_size);
memcpy(sn_hold_buff_temp, sn_hold_buff, sn_hold_buff_pos);
free (sn_hold_buff);
sn_hold_buff = sn_hold_buff_temp;
}
memcpy(sn_hold_buff + sn_hold_buff_pos, v.str.addr, v.str.len);
sn_hold_buff_pos += v.str.len;
if (expected_sn_chunk_number == numsubs)
{
if (sn_hold_buff_pos != gblsize)
{ /* we don't have the expected size even though */
/* we have all the expected chunks. */
DISPLAY_INCMP_SN_MSG;
util_out_print("!_!_Expected size : !UL actual size : !UL", TRUE,
gblsize, sn_hold_buff_pos);
if (sn_hold_buff_pos)
DISPLAY_PARTIAL_SN_HOLD_BUFF;
KILL_INCMP_SN_IF_NEEDED;
expected_sn_chunk_number = 0;
ok_to_put = FALSE;
sn_hold_buff_pos = 0;
}
else
{
expected_sn_chunk_number = 0;
ok_to_put = TRUE;
putting_a_sn = TRUE;
}
}else
expected_sn_chunk_number++;
}else
{
DISPLAY_INCMP_SN_MSG;
if ((sn_hold_buff_pos + v.str.len) <= gblsize)
util_out_print("!_!_Expected chunk number : !UL but found chunk number : !UL", /*BYPASSOK*/
TRUE, expected_sn_chunk_number + 1, sn_chunk_number + 1);
else
util_out_print("!_!_Global value too large: expected size : !UL actual size : !UL chunk number : !UL", TRUE, /*BYPASSOK*/
gblsize, sn_hold_buff_pos + v.str.len, sn_chunk_number + 1);
if (sn_hold_buff_pos)
DISPLAY_PARTIAL_SN_HOLD_BUFF;
if (v.str.len)
DISPLAY_VALUE("!_!_Errant Chunk :");
KILL_INCMP_SN_IF_NEEDED;
sn_hold_buff_pos = 0;
expected_sn_chunk_number = 0;
}
}
} else
ok_to_put = TRUE;
if (ok_to_put)
{
if (putting_a_sn)
{
gv_currkey->base[gv_currkey->end - (SPAN_SUBS_LEN + 1)] = 0;
gv_currkey->end -= (SPAN_SUBS_LEN + 1);
v.str.addr = sn_hold_buff;
v.str.len = sn_hold_buff_pos;
}
if (max_data_len < v.str.len)
max_data_len = v.str.len;
bin_call_db(BIN_PUT, (INTPTR_T)&v, 0);
if (mupip_error_occurred)
{
if (!mupip_DB_full)
{
bin_call_db(ERR_COR, (INTPTR_T)rec_count, (INTPTR_T)global_key_count);
file_offset = file_offset_base + ((unsigned char *)rp - ptr_base);
util_out_print("!_!_at File offset : [0x!XL]", TRUE, file_offset);
DISPLAY_CURRKEY;
DISPLAY_VALUE("!_!_Value :");
}
break;
}
if (putting_a_sn)
putting_a_sn = FALSE;
else
{
key_count++;
global_key_count++;
}
}
}
}
GTMCRYPT_ONLY(
if (NULL != hash_array)
free(hash_array);
)
void go_load(int begin, int end)
{
char *ptr;
int i, len, fmt, keylength, keystate;
uint4 max_data_len, max_subsc_len, key_count, rec_count, max_rec_size;
mstr src, des;
unsigned char *rec_buff, ch;
boolean_t format_error = FALSE, keepgoing;
error_def(ERR_LOADFILERR);
error_def(ERR_MUNOFINISH);
error_def(ERR_LOADCTRLY);
gvinit();
max_rec_size = DEFAULT_MAX_REC_SIZE;
rec_buff = (unsigned char *)malloc(max_rec_size);
if (!begin)
{
len = mu_load_get(&ptr);
if (mupip_error_occurred)
{
free(rec_buff);
return;
}
if (len >= 0)
util_out_print("!AD", TRUE, len, ptr);
else
mupip_exit(ERR_LOADFILERR);
len = mu_load_get(&ptr);
if (mupip_error_occurred)
{
free(rec_buff);
return;
}
if (len >= 0)
{
util_out_print("!AD", TRUE, len, ptr);
fmt = (0 == memcmp(ptr + len - STR_LIT_LEN("ZWR"), "ZWR", STR_LIT_LEN("ZWR"))) ? MU_FMT_ZWR : MU_FMT_GO;
} else
mupip_exit(ERR_LOADFILERR);
begin = 3;
} else
{
for (i = 1; i < begin; i++)
{
len = mu_load_get(&ptr);
if (mupip_error_occurred)
{
free(rec_buff);
return;
}
if (len < 0)
break;
if (2 == i) /* the format flag appears only in the second record. */
fmt = (0 == memcmp(ptr + len - STR_LIT_LEN("ZWR"), "ZWR", STR_LIT_LEN("ZWR"))) ?
MU_FMT_ZWR : MU_FMT_GO;
}
util_out_print("Beginning LOAD at #!UL\n", TRUE, begin);
}
max_data_len = 0;
max_subsc_len = 0;
key_count = 0;
rec_count = 1;
for (i = begin - 1 ; ;)
{
if (++i > end)
break;
if (mu_ctrly_occurred)
break;
if (mu_ctrlc_occurred)
{
util_out_print("!AD:!_ Key cnt: !UL max subsc len: !UL max data len: !UL", TRUE,
LEN_AND_LIT(gt_lit), key_count, max_subsc_len, max_data_len);
util_out_print("Last LOAD record number: !UL", TRUE, rec_count - 1);
mu_gvis();
util_out_print(0, TRUE);
mu_ctrlc_occurred = FALSE;
}
if ((len = mu_load_get(&ptr)) < 0)
break;
if (mupip_error_occurred)
{
mu_gvis();
break;
}
if ('\n' == *ptr)
{
if ('\n' == *(ptr+1))
break;
ptr++;
}
stringpool.free = stringpool.base;
rec_count++;
if (0 == len)
continue;
if (MU_FMT_GO != fmt)
{
keylength = 0; /* determine length of key */
keystate = 0;
keepgoing = TRUE;
while((keylength < len - 1) && keepgoing) /* 1 == sizeof(=), since ZWR allows '^x(1,2)='*/
{
ch = *(ptr + keylength);
keylength++;
switch (keystate)
{
case 0: /* in global name */
if ('=' == ch) /* end of key */
{
keylength--;
keepgoing = FALSE;
} else if ('(' == ch) /* start of subscripts */
keystate = 1;
break;
case 1: /* in subscript area, but out of "..." or $C(...) */
switch (ch)
{
case ')': /* end of subscripts ==> end of key */
assert('=' == *(ptr + keylength));
keepgoing = FALSE;
break;
case '"': /* step into "..." */
keystate = 2;
break;
case '$': /* step into $C(...) */
assert(('C' == *(ptr + keylength)) || ('c' == *(ptr + keylength)));
assert('(' == *(ptr + keylength + 1));
keylength += 2;
keystate = 3;
break;
}
break;
case 2: /* in "..." */
if ('"' == ch)
{
switch (*(ptr + keylength))
{
case '"': /* "" */
keylength++;
break;
case '_': /* _$C(...) */
assert('$' == *(ptr + keylength + 1));
assert(('c' == *(ptr + keylength + 2)) || ('C' == *(ptr + keylength + 2)));
assert('(' == *(ptr + keylength + 3));
keylength += 4;
keystate = 3;
break;
default: /* step out of "..." */
keystate = 1;
}
}
break;
case 3: /* in $C(...) */
if (')' == ch)
{
if ('_' == *(ptr + keylength)) /* step into "..." */
{
assert('"' == *(ptr + keylength + 1));
keylength += 2;
keystate = 2;
break;
} else
keystate = 1; /* step out of $C(...) */
}
break;
default:
assert(FALSE);
break;
}
}
assert(keylength < len - 1);
go_call_db(GO_PUT_SUB, ptr, keylength);
if (mupip_error_occurred)
{
mu_gvis();
mupip_error_occurred = FALSE;
continue;
}
if (max_subsc_len < gv_currkey->end)
max_subsc_len = gv_currkey->end;
src.len = len - keylength - 1;
src.addr = (char *)(ptr + keylength + 1);
des.len = 0;
if (src.len > max_rec_size)
{
max_rec_size = src.len;
free(rec_buff);
rec_buff = (unsigned char *)malloc(max_rec_size);
}
des.addr = (char *)rec_buff;
if (FALSE == zwr2format(&src, &des))
{
util_out_print("Format error in record !8UL: !/!AD", TRUE, rec_count + 1,
src.len, src.addr);
format_error = TRUE;
continue;
}
if (max_data_len < des.len)
max_data_len = des.len;
stringpool.free = stringpool.base;
go_call_db(GO_PUT_DATA, (char *)rec_buff, des.len);
if (mupip_error_occurred)
{
mu_gvis();
mupip_error_occurred = FALSE;
continue;
}
key_count++;
} else
{
go_call_db(GO_PUT_SUB, ptr, len);
if (mupip_error_occurred)
{
mu_gvis();
mupip_error_occurred = FALSE;
continue;
}
if (max_subsc_len < gv_currkey->end)
max_subsc_len = gv_currkey->end;
if (++i > end)
{
i--; /* Decrement as didn't load key */
break;
}
if ((len = mu_load_get(&ptr)) < 0)
break;
if (mupip_error_occurred)
{
mu_gvis();
break;
}
rec_count++;
stringpool.free = stringpool.base;
if (max_data_len < len)
max_data_len = len;
go_call_db(GO_PUT_DATA, ptr, len);
if (mupip_error_occurred)
{
mu_gvis();
mupip_error_occurred = FALSE;
continue;
}
key_count++;
}
}
free(rec_buff);
mu_load_close();
if(mu_ctrly_occurred)
{
gtm_putmsg(VARLSTCNT(1) ERR_LOADCTRLY);
mupip_exit(ERR_MUNOFINISH);
}
util_out_print("LOAD TOTAL!_!_Key Cnt: !UL Max Subsc Len: !UL Max Data Len: !UL",TRUE,key_count,max_subsc_len,
max_data_len);
util_out_print("Last LOAD record number: !UL\n",TRUE,i - 1);
if (format_error)
mupip_exit(ERR_LOADFILERR);
}