void
cache_analysis()
{
//printf("mblk categorization (hit or unknown)\n");
set_cache();
get_mblks();
find_hitloop();
categorize();
}
void Highlight::searchwords_in_area(GtkTextBuffer * textbuffer, vector < GtkTextIter > &start, vector < GtkTextIter > &end, bool area_id, bool area_intro, bool area_heading, bool area_chapter, bool area_study, bool area_notes, bool area_xref, bool area_verse)
/*
Finds out whether the text within the "start" and "end" iterators is inside
one of the given areas. If not, it removes the iterator from the containers.
*/
{
// Categorization data
CategorizeLine categorize("");
// Go through the iterators, starting at the end (to make erasing it easier).
for (int it = start.size() - 1; it >= 0; it--) {
// Get line number of the iterator.
gint linenumber = gtk_text_iter_get_line(&start[it]);
// Get the usfm this line starts with.
ustring usfm;
{
GtkTextIter line1;
gtk_text_buffer_get_iter_at_line(textbuffer, &line1, linenumber);
GtkTextIter line2 = line1;
gtk_text_iter_forward_chars(&line2, 10);
ustring line = gtk_text_iter_get_text(&line1, &line2);
usfm = usfm_extract_marker(line);
}
// See if this usfm is in one of the areas given.
bool in_area = false;
if (area_id)
if (categorize.is_id_marker(usfm))
in_area = true;
if (area_intro)
if (categorize.is_intro_marker(usfm))
in_area = true;
if (area_heading)
if (categorize.is_head_marker(usfm))
in_area = true;
if (area_chapter)
if (categorize.is_chap_marker(usfm))
in_area = true;
if (area_study)
if (categorize.is_study_marker(usfm))
in_area = true;
// The variables "area_notes" and "area_xref" are not relevant.
if (area_verse)
if (categorize.is_verse_marker(usfm))
in_area = true;
// If not in one of the areas, remove this iterator from the container.
if (!in_area) {
vector < GtkTextIter >::iterator startiter = start.begin();
vector < GtkTextIter >::iterator enditer = end.begin();
for (int i = 0; i < it; i++) {
startiter++;
enditer++;
}
start.erase(startiter);
end.erase(enditer);
}
}
}
static void mono_decode(COOKContext *q, float* mlt_buffer) {
int category_index[128];
int quant_index_table[102];
int category[128];
memset(&category, 0, 128*sizeof(int));
memset(&category_index, 0, 128*sizeof(int));
decode_envelope(q, quant_index_table);
q->num_vectors = get_bits(&q->gb,q->log2_numvector_size);
categorize(q, quant_index_table, category, category_index);
expand_category(q, category, category_index);
decode_vectors(q, category, quant_index_table, mlt_buffer);
}
static int mono_decode(COOKContext *q, COOKSubpacket *p, float *mlt_buffer)
{
int category_index[128] = { 0 };
int category[128] = { 0 };
int quant_index_table[102];
int res;
if ((res = decode_envelope(q, p, quant_index_table)) < 0)
return res;
q->num_vectors = get_bits(&q->gb, p->log2_numvector_size);
categorize(q, p, quant_index_table, category, category_index);
expand_category(q, category, category_index);
decode_vectors(q, p, category, quant_index_table, mlt_buffer);
return 0;
}
// Input: the list of prototypes and the name of the unknown file
// Returns: nothing
// Prints the category for each unknown data set
void findNearestNeighbor(std::vector<std::vector<double>> prototypes, char* fileName)
{
// Open file for reading
std::ifstream unknownFile;
unknownFile.open(fileName);
std::string line = "";
std::vector<double> unknown;
double category;
while(getline(unknownFile, line))
{
unknown = convertStrToDoubleVector(line);
category = categorize(prototypes, unknown);
std::cout << category << std::endl;
}
}
/*********************** convertToPostFix **************************************
int convertToPostFix(char *pszInfix, Out out)
Purpose:
Convert a querry line from the infix form to the postfix form.
Parameters:
I char *pszInfix Input querry to be converted.
O Out out Returned output querry in postfix form.
Returns:
Functionally:
Integer representing the success of the operation or error encountered.
0 - success.
WARN_MISSING_LPAREN - input is missing a left parenthesis.
WARN_MISSING_RPAREN - input is missing a right parenthesis.
out parm - the converted querry.
Notes:
Modified from prgrm1 version to add modularity and fix output.
*******************************************************************************/
int convertToPostFix(char *pszInfix, Out out)
{
/***Variables***/
Stack stack = newStack();
Token szToken; //token string to store in Element variables
Element element;
/***conversion***/
while ((pszInfix = getToken(pszInfix, szToken, MAX_TOKEN + 1)) != 0) //while there are tokens in the line of input
{
/***Element set up***/
strcpy(element.szToken, szToken); //store the token in the element
categorize(&element); //initialize the Element variable's values based on the token string entered
/***Actual conversion***/
switch (element.iCategory) //what kind of element is it?
{
case CAT_OPERAND:
convertSubTaskOperand(out, &element);
break;
case CAT_OPERATOR:
convertSubTaskOperator(stack, out, &element);
break;
case CAT_LPAREN:
convertSubTaskLParen(stack, &element);
break;
case CAT_RPAREN:
if (convertSubTaskRParen(stack, out) == WARN_MISSING_LPAREN)
{
return WARN_MISSING_LPAREN;
}
break;
}
}
/***Remaining Stack***/
if (convertSubTaskRemainingStack(stack, out) == WARN_MISSING_RPAREN)
{
return WARN_MISSING_RPAREN;
}
return 0; //return success
}
static int mono_decode(COOKContext *q, COOKSubpacket *p, float *mlt_buffer)
{
int category_index[128] = { 0 };
int category[128] = { 0 };
int quant_index_table[102];
int res, i;
if ((res = decode_envelope(q, p, quant_index_table)) < 0)
return res;
q->num_vectors = get_bits(&q->gb,p->log2_numvector_size);
categorize(q, p, quant_index_table, category, category_index);
expand_category(q, category, category_index);
for (i=0; i<p->total_subbands; i++) {
if (category[i] > 7)
return AVERROR_INVALIDDATA;
}
decode_vectors(q, p, category, quant_index_table, mlt_buffer);
return 0;
}
/*
- cli_regcomp_real - interface for parser and compilation
*/
int /* 0 success, otherwise REG_something */
cli_regcomp_real(regex_t *preg, const char *pattern, int cflags)
{
struct parse pa;
struct re_guts *g;
struct parse *p = &pa;
int i;
size_t len;
#ifdef REDEBUG
# define GOODFLAGS(f) (f)
#else
# define GOODFLAGS(f) ((f)&~REG_DUMP)
#endif
cflags = GOODFLAGS(cflags);
if ((cflags®_EXTENDED) && (cflags®_NOSPEC))
return(REG_INVARG);
if (cflags®_PEND) {
if (preg->re_endp < pattern)
return(REG_INVARG);
len = preg->re_endp - pattern;
} else
len = strlen((const char *)pattern);
/* do the mallocs early so failure handling is easy */
g = (struct re_guts *)cli_malloc(sizeof(struct re_guts) +
(NC-1)*sizeof(cat_t));
if (g == NULL)
return(REG_ESPACE);
p->ssize = len/(size_t)2*(size_t)3 + (size_t)1; /* ugh */
p->strip = (sop *)cli_calloc(p->ssize, sizeof(sop));
p->slen = 0;
if (p->strip == NULL) {
free((char *)g);
return(REG_ESPACE);
}
/* set things up */
p->g = g;
p->next = (char *)pattern; /* convenience; we do not modify it */
p->end = p->next + len;
p->error = 0;
p->ncsalloc = 0;
for (i = 0; i < NPAREN; i++) {
p->pbegin[i] = 0;
p->pend[i] = 0;
}
g->csetsize = NC;
g->sets = NULL;
g->setbits = NULL;
g->ncsets = 0;
g->cflags = cflags;
g->iflags = 0;
g->nbol = 0;
g->neol = 0;
g->must = NULL;
g->mlen = 0;
g->nsub = 0;
g->ncategories = 1; /* category 0 is "everything else" */
g->categories = &g->catspace[-(CHAR_MIN)];
(void) memset((char *)g->catspace, 0, NC*sizeof(cat_t));
g->backrefs = 0;
/* do it */
EMIT(OEND, 0);
g->firststate = THERE();
if (cflags®_EXTENDED)
p_ere(p, OUT);
else if (cflags®_NOSPEC)
p_str(p);
else
p_bre(p, OUT, OUT);
EMIT(OEND, 0);
g->laststate = THERE();
/* tidy up loose ends and fill things in */
categorize(p, g);
stripsnug(p, g);
findmust(p, g);
g->nplus = pluscount(p, g);
g->magic = MAGIC2;
preg->re_nsub = g->nsub;
preg->re_g = g;
preg->re_magic = MAGIC1;
#ifndef REDEBUG
/* not debugging, so can't rely on the assert() in cli_regexec() */
if (g->iflags®EX_BAD)
SETERROR(REG_ASSERT);
#endif
/* win or lose, we're done */
if (p->error != 0) /* lose */
cli_regfree(preg);
return(p->error);
}
void encoder(Word16 number_of_available_bits,
Word16 number_of_regions,
Word16 *mlt_coefs,
Word16 mag_shift,
Word16 *out_words)
{
Word16 num_categorization_control_bits;
Word16 num_categorization_control_possibilities;
Word16 number_of_bits_per_frame;
Word16 number_of_envelope_bits;
Word16 categorization_control;
Word16 region;
Word16 absolute_region_power_index[MAX_NUMBER_OF_REGIONS];
Word16 power_categories[MAX_NUMBER_OF_REGIONS];
Word16 category_balances[MAX_NUM_CATEGORIZATION_CONTROL_POSSIBILITIES-1];
Word16 drp_num_bits[MAX_NUMBER_OF_REGIONS+1];
UWord16 drp_code_bits[MAX_NUMBER_OF_REGIONS+1];
Word16 region_mlt_bit_counts[MAX_NUMBER_OF_REGIONS];
UWord32 region_mlt_bits[4*MAX_NUMBER_OF_REGIONS];
Word16 mag_shift_offset;
Word16 temp;
/* initialize variables */
test();
if (number_of_regions == NUMBER_OF_REGIONS)
{
num_categorization_control_bits = NUM_CATEGORIZATION_CONTROL_BITS;
move16();
num_categorization_control_possibilities = NUM_CATEGORIZATION_CONTROL_POSSIBILITIES;
move16();
}
else
{
num_categorization_control_bits = MAX_NUM_CATEGORIZATION_CONTROL_BITS;
move16();
num_categorization_control_possibilities = MAX_NUM_CATEGORIZATION_CONTROL_POSSIBILITIES;
move16();
}
number_of_bits_per_frame = number_of_available_bits;
move16();
for (region=0; region<number_of_regions; region++)
{
region_mlt_bit_counts[region] = 0;
move16();
}
/* Estimate power envelope. */
number_of_envelope_bits = compute_region_powers(mlt_coefs,
mag_shift,
drp_num_bits,
drp_code_bits,
absolute_region_power_index,
number_of_regions);
/* Adjust number of available bits based on power envelope estimate */
temp = sub(number_of_available_bits,number_of_envelope_bits);
number_of_available_bits = sub(temp,num_categorization_control_bits);
/* get categorizations */
categorize(number_of_available_bits,
number_of_regions,
num_categorization_control_possibilities,
absolute_region_power_index,
power_categories,
category_balances);
/* Adjust absolute_region_category_index[] for mag_shift.
This assumes that REGION_POWER_STEPSIZE_DB is defined
to be exactly 3.010299957 or 20.0 times log base 10
of square root of 2. */
temp = shl_nocheck(mag_shift,1);
mag_shift_offset = add(temp,REGION_POWER_TABLE_NUM_NEGATIVES);
for (region=0; region<number_of_regions; region++)
{
absolute_region_power_index[region] = add(absolute_region_power_index[region],mag_shift_offset);
move16();
}
/* adjust the absolute power region index based on the mlt coefs */
adjust_abs_region_power_index(absolute_region_power_index,mlt_coefs,number_of_regions);
/* quantize and code the mlt coefficients based on categorizations */
vector_quantize_mlts(number_of_available_bits,
number_of_regions,
num_categorization_control_possibilities,
mlt_coefs,
absolute_region_power_index,
power_categories,
category_balances,
&categorization_control,
region_mlt_bit_counts,
region_mlt_bits);
/* stuff bits into words */
bits_to_words(region_mlt_bits,
region_mlt_bit_counts,
drp_num_bits,
drp_code_bits,
out_words,
categorization_control,
number_of_regions,
num_categorization_control_bits,
number_of_bits_per_frame);
}
CheckCapitalization::CheckCapitalization(const ustring & project, const vector < unsigned int >&books, const ustring & punctuation_followed_by_capitals, const ustring & ignore_lowercase_following, const ustring & abbreviations_filename, bool any_prefixes, const ustring & uncapitalized_prefixes_filename, const ustring & capitalized_suffixes_filename, bool gui)
/*
It checks the capitalization in the text.
project: project to check.
books: books to check; if empty it checks them all.
punctuation_followed_by_capitals: give punctuation that may be followed by capitals.
ignore_lowercase_following: ignore the error if a lower case character follows these characters.
abbreviations_filename: filename with the abbreviations.
any_prefixes: allow any prefixes before the capitals.
uncapitalized_prefixes_filename: filename with the list of uncapitalized prefixes.
capitalized_suffixes_filename: filename with the list of the capitalized suffixes.
gui: whether to show graphical progressbar.
*/
{
cancelled = false;
vector < unsigned int >mybooks(books.begin(), books.end());
if (mybooks.empty())
mybooks = project_get_books(project);
for (unsigned int i = 0; i < punctuation_followed_by_capitals.length(); i++) {
punctuation_followed_by_capitals_set.insert(g_utf8_get_char(punctuation_followed_by_capitals.substr(i, 1).c_str()));
}
for (unsigned int i = 0; i < ignore_lowercase_following.length(); i++) {
ignore_lower_case_following_set.insert(g_utf8_get_char(ignore_lowercase_following.substr(i, 1).c_str()));
}
{
ReadText rt(abbreviations_filename, true);
for (unsigned int i = 0; i < rt.lines.size(); i++) {
abbreviations.insert(rt.lines[i]);
}
}
allow_any_uncapitalized_prefixes = any_prefixes;
{
ReadText rt(uncapitalized_prefixes_filename, true);
for (unsigned int i = 0; i < rt.lines.size(); i++) {
uncapitalized_prefixes.insert(rt.lines[i]);
}
}
{
ReadText rt(capitalized_suffixes_filename, true);
for (unsigned int i = 0; i < rt.lines.size(); i++) {
capitalized_suffixes.insert(rt.lines[i]);
}
}
progresswindow = NULL;
if (gui) {
progresswindow = new ProgressWindow(_("Capitalization"), true);
progresswindow->set_iterate(0, 1, mybooks.size());
}
for (unsigned int bk = 0; bk < mybooks.size(); bk++) {
book = mybooks[bk];
if (gui) {
progresswindow->iterate();
if (progresswindow->cancel) {
cancelled = true;
return;
}
}
vector < unsigned int >chapters = project_get_chapters(project, book);
for (unsigned int ch = 0; ch < chapters.size(); ch++) {
chapter = chapters[ch];
vector < ustring > verses = project_get_verses(project, book, chapter);
for (unsigned int vs = 0; vs < verses.size(); vs++) {
verse = verses[vs];
vector < int >mychapters;
vector < ustring > myverses;
vector < size_t > mypointers;
mychapters.push_back(chapters[ch]);
myverses.push_back(verses[vs]);
mypointers.push_back(0);
ustring line = project_retrieve_verse(project, book, chapter, verse);
CategorizeLine categorize(line);
// No checks done on id.
// Introduction text.
check_capitalization(mychapters, myverses, categorize.intro, mypointers, false);
check_suspicious_capitalization(categorize.intro);
// Heading text.
check_capitalization(mychapters, myverses, categorize.head, mypointers, false);
check_suspicious_capitalization(categorize.head);
// No checks done on chapter.
// Study notes.
check_capitalization(mychapters, myverses, categorize.study, mypointers, true);
check_suspicious_capitalization(categorize.study);
// Foot- and endnotes.
check_capitalization(mychapters, myverses, categorize.note, mypointers, true);
check_suspicious_capitalization(categorize.note);
// Crossreferences.
check_capitalization(mychapters, myverses, categorize.ref, mypointers, true);
check_suspicious_capitalization(categorize.ref);
// Store verse text for checking at the end of the book. We cannot check
// per verse or chapter, because sentences could span them.
verse_chapter.push_back(chapter);
verse_verse.push_back(verse);
verse_pointer.push_back(verse_text.length());
if (!verse_text.empty())
verse_text.append(" ");
verse_text.append(categorize.verse);
// Check suspicious capitalization in the versetext.
check_suspicious_capitalization(categorize.verse);
}
}
// We've reached the end of the book. Check all verse text.
check_capitalization(verse_chapter, verse_verse, verse_text, verse_pointer, true);
// Clear containers for verse text.
verse_chapter.clear();
verse_verse.clear();
verse_text.clear();
verse_pointer.clear();
}
}
CheckMatchingPairs::CheckMatchingPairs(const ustring & project, const vector < unsigned int >&books, const ustring & ignore, bool gui)
/*
It checks matching pairs of punctuation, e.g. the ( matches with the ).
project: project to check.
books: books to check; if empty it checks them all.
ignore: punctuation to ignore.
gui: whether to show graphical progressbar.
*/
{
cancelled = false;
vector < unsigned int >mybooks(books.begin(), books.end());
if (mybooks.empty())
mybooks = project_get_books(project);
for (unsigned int i = 0; i < ignore.length(); i++) {
gunichar unichar = g_utf8_get_char(ignore.substr(i, 1).c_str());
ignores.insert(unichar);
gunichar mirror;
if (g_unichar_get_mirror_char(unichar, &mirror)) {
ignores.insert(mirror);
}
}
// Get list of openers and closers.
for (gunichar i = 0; i < 1000000; i++) {
gunichar mirror;
if (g_unichar_get_mirror_char(i, &mirror)) {
if (gclosers.find(i) == gclosers.end()) {
gopeners.insert(i);
gclosers.insert(mirror);
}
}
}
progresswindow = NULL;
if (gui) {
progresswindow = new ProgressWindow(_("Matching pairs"), true);
progresswindow->set_iterate(0, 1, mybooks.size());
}
for (unsigned int bk = 0; bk < mybooks.size(); bk++) {
book = mybooks[bk];
if (gui) {
progresswindow->iterate();
if (progresswindow->cancel) {
cancelled = true;
break;
}
}
vector < unsigned int >chapters = project_get_chapters(project, book);
for (unsigned int ch = 0; ch < chapters.size(); ch++) {
chapter = chapters[ch];
vector < ustring > verses = project_get_verses(project, book, chapter);
for (unsigned int vs = 0; vs < verses.size(); vs++) {
verse = verses[vs];
ustring line = project_retrieve_verse(project, book, chapter, verse);
CategorizeLine categorize(line);
check_matched_pairs(categorize.id);
check_matched_pairs(categorize.intro);
check_matched_pairs(categorize.head);
check_matched_pairs(categorize.chap);
check_matched_pairs(categorize.study);
check_matched_pairs(categorize.note);
check_matched_pairs(categorize.ref);
check_matched_pairs(categorize.verse);
}
// At the end of each chapter, check whether all pairs are "clean"
check_pairs_clean();
}
}
}
