void
fz_analyze_text(fz_context *ctx, fz_text_sheet *sheet, fz_text_page *page)
{
fz_text_line *line;
fz_text_span *span;
line_heights *lh;
region_masks *rms;
int block_num;
/* Simple paragraph analysis; look for the most common 'inter line'
* spacing. This will be assumed to be our line spacing. Anything
* more than 25% wider than this will be assumed to be a paragraph
* space. */
/* Step 1: Gather the line height information */
lh = new_line_heights(ctx);
for (block_num = 0; block_num < page->len; block_num++)
{
fz_text_block *block;
if (page->blocks[block_num].type != FZ_PAGE_BLOCK_TEXT)
continue;
block = page->blocks[block_num].u.text;
for (line = block->lines; line < block->lines + block->len; line++)
{
/* For every style in the line, add lineheight to the
* record for that style. FIXME: This is a nasty n^2
* algorithm at the moment. */
fz_text_style *style = NULL;
if (line->distance == 0)
continue;
for (span = line->first_span; span; span = span->next)
{
int char_num;
if (is_list_entry(line, span, &char_num))
goto list_entry;
for (; char_num < span->len; char_num++)
{
fz_text_char *chr = &span->text[char_num];
/* Ignore any whitespace chars */
if (is_unicode_wspace(chr->c))
continue;
if (chr->style != style)
{
/* Have we had this style before? */
int match = 0;
fz_text_span *span2;
for (span2 = line->first_span; span2 != span; span2 = span2->next)
{
int char_num2;
for (char_num2 = 0; char_num2 < span2->len; char_num2++)
{
fz_text_char *chr2 = &span2->text[char_num2];
if (chr2->style == chr->style)
{
match = 1;
break;
}
}
}
if (char_num > 0 && match == 0)
{
fz_text_span *span2 = span;
int char_num2;
for (char_num2 = 0; char_num2 < char_num; char_num2++)
{
fz_text_char *chr2 = &span2->text[char_num2];
if (chr2->style == chr->style)
{
match = 1;
break;
}
}
}
if (match == 0)
insert_line_height(lh, chr->style, line->distance);
style = chr->style;
}
}
list_entry:
{}
}
}
}
/* Step 2: Find the most popular line height for each style */
cull_line_heights(lh);
/* Step 3: Run through the blocks, breaking each block into two if
* the line height isn't right. */
for (block_num = 0; block_num < page->len; block_num++)
{
int line_num;
fz_text_block *block;
if (page->blocks[block_num].type != FZ_PAGE_BLOCK_TEXT)
continue;
block = page->blocks[block_num].u.text;
for (line_num = 0; line_num < block->len; line_num++)
{
/* For every style in the line, check to see if lineheight
* is correct for that style. FIXME: We check each style
* more than once, currently. */
int ok = 0; /* -1 = early exit, split now. 0 = split. 1 = don't split. */
fz_text_style *style = NULL;
line = &block->lines[line_num];
if (line->distance == 0)
continue;
#ifdef DEBUG_LINE_HEIGHTS
printf("line height=%g\n", line->distance);
#endif
for (span = line->first_span; span; span = span->next)
{
int char_num;
if (is_list_entry(line, span, &char_num))
goto force_paragraph;
/* Now we do the rest of the line */
for (; char_num < span->len; char_num++)
{
fz_text_char *chr = &span->text[char_num];
/* Ignore any whitespace chars */
if (is_unicode_wspace(chr->c))
continue;
if (chr->style != style)
{
float proper_step = line_height_for_style(lh, chr->style);
if (proper_step * 0.95 <= line->distance && line->distance <= proper_step * 1.05)
{
ok = 1;
break;
}
style = chr->style;
}
}
if (ok)
break;
}
if (!ok)
{
force_paragraph:
split_block(ctx, page, block_num, line_num);
break;
}
}
}
free_line_heights(lh);
/* Simple line region analysis:
* For each line:
* form a list of 'start/stop' points (henceforth a 'region mask')
* find the normalised baseline vector for the line.
* Store the region mask and baseline vector.
* Collate lines that have compatible region masks and identical
* baseline vectors.
* If the collated masks are column-like, then split into columns.
* Otherwise split into tables.
*/
rms = new_region_masks(ctx);
/* Step 1: Form the region masks and store them into a list with the
* normalised baseline vectors. */
for (block_num = 0; block_num < page->len; block_num++)
{
fz_text_block *block;
if (page->blocks[block_num].type != FZ_PAGE_BLOCK_TEXT)
continue;
block = page->blocks[block_num].u.text;
for (line = block->lines; line < block->lines + block->len; line++)
{
fz_point blv;
region_mask *rm;
#ifdef DEBUG_MASKS
printf("Line: ");
dump_line(line);
#endif
blv = line->first_span->max;
blv.x -= line->first_span->min.x;
blv.y -= line->first_span->min.y;
fz_normalize_vector(&blv);
rm = new_region_mask(ctx, &blv);
for (span = line->first_span; span; span = span->next)
{
fz_point *region_min = &span->min;
fz_point *region_max = &span->max;
/* Treat adjacent spans as one big region */
while (span->next && span->next->spacing < 1.5)
{
span = span->next;
region_max = &span->max;
}
region_mask_add(rm, region_min, region_max);
}
#ifdef DEBUG_MASKS
dump_region_mask(rm);
#endif
region_masks_add(rms, rm);
}
}
/* Step 2: Sort the region_masks by size of masked region */
region_masks_sort(rms);
#ifdef DEBUG_MASKS
printf("Sorted list of regions:\n");
dump_region_masks(rms);
#endif
/* Step 3: Merge the region masks where possible (large ones first) */
{
int i;
region_masks *rms2;
rms2 = new_region_masks(ctx);
for (i=0; i < rms->len; i++)
{
region_mask *rm = rms->mask[i];
rms->mask[i] = NULL;
region_masks_merge(rms2, rm);
}
free_region_masks(rms);
rms = rms2;
}
#ifdef DEBUG_MASKS
printf("Merged list of regions:\n");
dump_region_masks(rms);
#endif
/* Step 4: Figure out alignment */
region_masks_alignment(rms);
/* Step 5: At this point, we should probably look at the region masks
* to try to guess which ones represent columns on the page. With our
* current code, we could only get blocks of lines that span 2 or more
* columns if the PDF producer wrote text out horizontally across 2
* or more columns, and we've never seen that (yet!). So we skip this
* step for now. */
/* Step 6: Run through the lines again, deciding which ones fit into
* which region mask. */
{
region_mask *prev_match = NULL;
for (block_num = 0; block_num < page->len; block_num++)
{
fz_text_block *block;
if (page->blocks[block_num].type != FZ_PAGE_BLOCK_TEXT)
continue;
block = page->blocks[block_num].u.text;
for (line = block->lines; line < block->lines + block->len; line++)
{
fz_point blv;
region_mask *rm;
region_mask *match;
blv = line->first_span->max;
blv.x -= line->first_span->min.x;
blv.y -= line->first_span->min.y;
fz_normalize_vector(&blv);
#ifdef DEBUG_MASKS
dump_line(line);
#endif
rm = new_region_mask(ctx, &blv);
for (span = line->first_span; span; span = span->next)
{
fz_point *region_min = &span->min;
fz_point *region_max = &span->max;
/* Treat adjacent spans as one big region */
while (span->next && span->next->spacing < 1.5)
{
span = span->next;
region_max = &span->max;
}
region_mask_add(rm, region_min, region_max);
}
#ifdef DEBUG_MASKS
printf("Mask: ");
dump_region_mask(rm);
#endif
match = region_masks_match(rms, rm, line, prev_match);
prev_match = match;
#ifdef DEBUG_MASKS
printf("Matches: ");
dump_region_mask(match);
#endif
free_region_mask(rm);
span = line->first_span;
while (span)
{
fz_point *region_min = &span->min;
fz_point *region_max = &span->max;
fz_text_span *sn;
int col, align;
float colw, left;
/* Treat adjacent spans as one big region */
#ifdef DEBUG_ALIGN
dump_span(span);
#endif
for (sn = span->next; sn && sn->spacing < 1.5; sn = sn->next)
{
region_max = &sn->max;
#ifdef DEBUG_ALIGN
dump_span(sn);
#endif
}
col = region_mask_column(match, region_min, region_max, &align, &colw, &left);
#ifdef DEBUG_ALIGN
printf(" = col%d colw=%g align=%d\n", col, colw, align);
#endif
do
{
span->column = col;
span->align = align;
span->indent = left;
span->column_width = colw;
span = span->next;
}
while (span != sn);
if (span)
span = span->next;
}
line->region = match;
}
}
free_region_masks(rms);
}
/* Step 7: Collate lines within a block that share the same region
* mask. */
for (block_num = 0; block_num < page->len; block_num++)
{
int line_num;
int prev_line_num;
fz_text_block *block;
if (page->blocks[block_num].type != FZ_PAGE_BLOCK_TEXT)
continue;
block = page->blocks[block_num].u.text;
/* First merge lines. This may leave empty lines behind. */
for (prev_line_num = 0, line_num = 1; line_num < block->len; line_num++)
{
fz_text_line *prev_line;
line = &block->lines[line_num];
if (!line->first_span)
continue;
prev_line = &block->lines[prev_line_num];
if (prev_line->region == line->region)
{
/* We only merge lines if the second line
* only uses 1 of the columns. */
int col = line->first_span->column;
/* Copy the left value for the first span
* in the first column in this line forward
* for all the rest of the spans in the same
* column. */
float indent = line->first_span->indent;
for (span = line->first_span->next; span; span = span->next)
{
if (col != span->column)
break;
span->indent = indent;
}
if (span)
{
prev_line_num = line_num;
continue;
}
/* Merge line into prev_line */
{
fz_text_span **prev_line_span = &prev_line->first_span;
int try_dehyphen = -1;
fz_text_span *prev_span = NULL;
span = line->first_span;
while (span && *prev_line_span)
{
/* Skip forwards through the original
* line, until we find a place where
* span should go. */
if ((*prev_line_span)->column <= span->column)
{
/* The current span we are considering
* in prev_line is earlier than span.
* Just skip forwards in prev_line. */
prev_span = (*prev_line_span);
prev_line_span = &prev_span->next;
try_dehyphen = span->column;
}
else
{
/* We want to copy span into prev_line. */
fz_text_span *next = (*prev_line_span)->next;
if (prev_line_span == &prev_line->first_span)
prev_line->first_span = span;
if (next == NULL)
prev_line->last_span = span;
if (try_dehyphen == span->column)
dehyphenate(prev_span, span);
try_dehyphen = -1;
prev_span = *prev_line_span = span;
span = span->next;
(*prev_line_span)->next = next;
prev_line_span = &(*prev_line_span)->next;
}
}
if (span)
{
*prev_line_span = span;
prev_line->last_span = line->last_span;
}
line->first_span = NULL;
line->last_span = NULL;
}
}
else
prev_line_num = line_num;
}
/* Now get rid of the empty lines */
for (prev_line_num = 0, line_num = 0; line_num < block->len; line_num++)
{
line = &block->lines[line_num];
if (line->first_span)
block->lines[prev_line_num++] = *line;
}
block->len = prev_line_num;
/* Now try to spot indents */
for (line_num = 0; line_num < block->len; line_num++)
{
fz_text_span *span_num, *sn;
int col, count;
line = &block->lines[line_num];
/* Run through the spans... */
span_num = line->first_span;
{
float indent = 0;
/* For each set of spans that share the same
* column... */
col = span_num->column;
#ifdef DEBUG_INDENTS
printf("Indent %g: ", span_num->indent);
dump_span(span_num);
printf("\n");
#endif
/* find the average indent of all but the first.. */
for (sn = span_num->next, count = 0; sn && sn->column == col; sn = sn->next, count++)
{
#ifdef DEBUG_INDENTS
printf("Indent %g: ", sn->indent);
dump_span(sn);
printf("\n");
#endif
indent += sn->indent;
sn->indent = 0;
}
if (sn != span_num->next)
indent /= count;
/* And compare this indent with the first one... */
#ifdef DEBUG_INDENTS
printf("Average indent %g ", indent);
#endif
indent -= span_num->indent;
#ifdef DEBUG_INDENTS
printf("delta %g ", indent);
#endif
if (fabsf(indent) < 1)
{
/* No indent worth speaking of */
indent = 0;
}
#ifdef DEBUG_INDENTS
printf("recorded %g\n", indent);
#endif
span_num->indent = indent;
span_num = sn;
}
for (; span_num; span_num = span_num->next)
{
span_num->indent = 0;
}
}
}
}
void
fz_text_analysis(fz_context *ctx, fz_text_sheet *sheet, fz_text_page *page)
{
fz_text_block *block;
fz_text_line *line;
fz_text_span *span;
fz_text_line *prev_line;
line_heights *lh;
int blocknum;
/* Simple paragraph analysis; look for the most common 'inter line'
* spacing. This will be assumed to be our line spacing. Anything
* more than 25% wider than this will be assumed to be a paragraph
* space. */
/* Step 1: Gather the line height information */
lh = new_line_heights(ctx);
prev_line = NULL;
for (block = page->blocks; block < page->blocks + page->len; block++)
{
for (line = block->lines; line < block->lines + block->len; line++)
{
/* In a line made up of several spans, find the tallest
* span. This line difference will count as being a
* difference in a line of that style. */
fz_text_span *tallest_span = NULL;
float tallest = 0;
float span_height;
for (span = line->spans; span < line->spans + line->len; span++)
{
span_height = span->bbox.y1 - span->bbox.y0;
if (tallest_span == NULL || span_height > tallest)
{
tallest_span = span;
tallest = span_height;
}
}
if (prev_line)
{
/* Should really work on the baseline positions,
* but we don't have that at this stage. */
float line_step = line->bbox.y1 - prev_line->bbox.y1;
if (line_step > 0)
{
insert_line_height(lh, tallest_span->style, line_step);
}
}
prev_line = line;
}
}
/* Step 2: Find the most popular line height for each style */
cull_line_heights(lh);
/* Step 3: Run through the blocks, breaking each block into two if
* the line height isn't right. */
prev_line = NULL;
for (blocknum = 0; blocknum < page->len; blocknum++)
{
block = &page->blocks[blocknum];
for (line = block->lines; line < block->lines + block->len; line++)
{
/* In a line made up of several spans, find the tallest
* span. This line difference will count as being a
* difference in a line of that style. */
fz_text_span *tallest_span = NULL;
float tallest = 0;
float span_height;
for (span = line->spans; span < line->spans + line->len; span++)
{
span_height = span->bbox.y1 - span->bbox.y0;
if (tallest_span == NULL || span_height > tallest)
{
tallest_span = span;
tallest = span_height;
}
}
if (prev_line)
{
float proper_step = line_height_for_style(lh, tallest_span->style);
float line_step = line->bbox.y1 - prev_line->bbox.y1;
if (proper_step * 0.95 > line_step || line_step > proper_step * 1.05)
{
split_block(ctx, page, block - page->blocks, line - block->lines);
prev_line = NULL;
break;
}
}
prev_line = line;
}
}
}
