void NumPermuteIntVec(int *arr, int l, real permute_ratio) {
// pairwise permute array l * permute_ratio times
int cnt = permute_ratio * l;
int i, j;
while (cnt-- > 0) {
i = NumRand() * l;
j = NumRand() * (l - 1);
if (j >= i) j++;
SWAP(arr[i], arr[j]);
}
return;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
// NumGenerator - utilizes the Random Number Generator to generate random num x of length SIZE
// * INPUT: Byte x, random (to increase randomness)
// * OUTPUT: SIZE Byte parameters
/////////////////////////////////////////////////////////////////////////////////////////////////
void NumGenerator(Chunk x, int random)
{
int i;
for (i = 0; i < SIZE; i++)
x[i] = NumRand(random++);
}
void DcmeThreadPrintProgBar(int dbg_lvl, int tid, real p, DcmeBookkeeping* b) {
if (sid_dcme_ppb_lock) return;
sid_dcme_ppb_lock = 1;
#ifdef DEBUG
if (NumRand() > 0.01) {
sid_dcme_ppb_lock = 0;
return;
}
char* wdis = WeightDebugInfoStr(weight, C, N, p, tid, start_clock_t,
V_THREAD_NUM, gd_ss);
char* ddis = DcmeDualModelDebugInfoStr(b);
LOGCR(dbg_lvl);
if (V_WEIGHT_DECOR_FILE_PATH)
LOG(dbg_lvl, "[%s]: ", V_WEIGHT_DECOR_FILE_PATH);
LOG(dbg_lvl, "%s", wdis);
LOG(dbg_lvl, " ");
LOG(dbg_lvl, "%s", ddis);
free(wdis);
free(ddis);
#else
char* wdis = WeightDebugInfoStr(weight, C, N, p, tid, start_clock_t,
V_THREAD_NUM, gd_ss);
LOGCLR(dbg_lvl);
if (V_WEIGHT_DECOR_FILE_PATH)
LOG(dbg_lvl, "[%s]: ", V_WEIGHT_DECOR_FILE_PATH);
LOG(dbg_lvl, "%s", wdis);
free(wdis);
#endif
sid_dcme_ppb_lock = 0;
return;
}
PeekSet *sid_peek_build(char *text_file_path, int if_lower,
int if_rm_trail_punc, long int fbeg, long int fend,
real sample_rate, int peek_top_k, Vocabulary *vcb,
long int *fpass, int *size) {
int i;
FILE *fin = fopen(text_file_path, "rb");
if (!fin) {
LOG(0, "Error\n");
exit(1);
}
PeekSet *ps = (PeekSet *)malloc(sizeof(PeekSet));
ps->top_k = peek_top_k;
ps->top_w = (int *)malloc(ps->top_k * sizeof(int));
for (i = 0; i < peek_top_k; i++) ps->top_w[i] = i;
fseek(fin, 0, SEEK_END);
long int fsz = ftell(fin);
fseek(fin, fbeg, SEEK_SET);
if (!fin) {
LOG(0, "Error\n");
exit(1);
}
ps->size = 0;
int cap = 10000;
ps->wids = (int **)malloc(cap * sizeof(int *));
ps->wnum = (int *)malloc(cap * sizeof(int));
int wids[SUP], wnum;
long int sid = 0, fpos1, fpos2;
while (!feof(fin) && (fend < 0 || ftell(fin) < fend)) {
fpos1 = ftell(fin);
wnum = TextReadSent(fin, vcb, wids, if_lower, if_rm_trail_punc, 1);
fpos2 = ftell(fin);
*fpass += fpos2 - fpos1;
if (sid++ > 100 && NumRand() < sample_rate && wnum >= 5) {
ps->wids[ps->size] = NumCloneIntVec(wids, wnum);
ps->wnum[ps->size] = wnum;
ps->size++;
(*size)++;
if (ps->size == cap) {
cap <<= 1; // double resize
ps->wids = (int **)realloc(ps->wids, cap * sizeof(int *));
ps->wnum = (int *)realloc(ps->wnum, cap * sizeof(int));
}
if (!sid_peek_pb_lock) {
sid_peek_pb_lock = 1;
LOGCLR(2);
LOG(2, "[PEEK]: completed %lf%% (%d sentence)",
(double)(*fpass) * 100 / fsz, *size);
sid_peek_pb_lock = 0;
}
}
}
ps->wids = (int **)realloc(ps->wids, ps->size * sizeof(int *)); // shrink
ps->wnum = (int *)realloc(ps->wnum, ps->size * sizeof(int));
return ps;
}
void DcmeThreadPrintProgBar(int dbg_lvl, int tid, real p, DcmeBookkeeping* b) {
if (sid_dcme_ppb_lock) return;
sid_dcme_ppb_lock = 1;
#ifdef DEBUG
if (NumRand() > 1) {
sid_dcme_ppb_lock = 0;
return;
}
char* mdis =
ModelDebugInfoStr(model, p, tid, start_clock_t, V_THREAD_NUM, gd_ss);
#ifdef DEBUGPEEK
char* ddis = DcmeDualModelDebugInfoStr(b);
LOGCR(dbg_lvl);
if (V_MODEL_DECOR_FILE_PATH) LOG(dbg_lvl, "[%s]: ", V_MODEL_DECOR_FILE_PATH);
LOG(dbg_lvl, "%s", mdis);
LOG(dbg_lvl, " ");
LOG(dbg_lvl, "%s", ddis);
LOGCR(dbg_lvl);
free(mdis);
free(ddis);
#else
LOGCR(dbg_lvl);
if (V_MODEL_DECOR_FILE_PATH) LOG(dbg_lvl, "[%s]: ", V_MODEL_DECOR_FILE_PATH);
LOG(dbg_lvl, "%s", mdis);
free(mdis);
#endif
#else
char* mdis =
ModelDebugInfoStr(model, p, tid, start_clock_t, V_THREAD_NUM, gd_ss);
LOGCLR(dbg_lvl);
if (V_MODEL_DECOR_FILE_PATH) LOG(dbg_lvl, "[%s]: ", V_MODEL_DECOR_FILE_PATH);
LOG(dbg_lvl, "%s", mdis);
free(mdis);
#endif
sid_dcme_ppb_lock = 0;
return;
}
void NumRandFillVec(real *arr, int l, real lb, real ub) {
int i;
for (i = 0; i < l; i++) arr[i] = lb + NumRand() * (ub - lb);
return;
}
