void fz_analyze_text(fz_context *ctx, fz_text_sheet *sheet, fz_text_page *page) { fz_text_line *line; fz_text_span *span; line_heights *lh; region_masks *rms; int block_num; /* Simple paragraph analysis; look for the most common 'inter line' * spacing. This will be assumed to be our line spacing. Anything * more than 25% wider than this will be assumed to be a paragraph * space. */ /* Step 1: Gather the line height information */ lh = new_line_heights(ctx); for (block_num = 0; block_num < page->len; block_num++) { fz_text_block *block; if (page->blocks[block_num].type != FZ_PAGE_BLOCK_TEXT) continue; block = page->blocks[block_num].u.text; for (line = block->lines; line < block->lines + block->len; line++) { /* For every style in the line, add lineheight to the * record for that style. FIXME: This is a nasty n^2 * algorithm at the moment. */ fz_text_style *style = NULL; if (line->distance == 0) continue; for (span = line->first_span; span; span = span->next) { int char_num; if (is_list_entry(line, span, &char_num)) goto list_entry; for (; char_num < span->len; char_num++) { fz_text_char *chr = &span->text[char_num]; /* Ignore any whitespace chars */ if (is_unicode_wspace(chr->c)) continue; if (chr->style != style) { /* Have we had this style before? */ int match = 0; fz_text_span *span2; for (span2 = line->first_span; span2 != span; span2 = span2->next) { int char_num2; for (char_num2 = 0; char_num2 < span2->len; char_num2++) { fz_text_char *chr2 = &span2->text[char_num2]; if (chr2->style == chr->style) { match = 1; break; } } } if (char_num > 0 && match == 0) { fz_text_span *span2 = span; int char_num2; for (char_num2 = 0; char_num2 < char_num; char_num2++) { fz_text_char *chr2 = &span2->text[char_num2]; if (chr2->style == chr->style) { match = 1; break; } } } if (match == 0) insert_line_height(lh, chr->style, line->distance); style = chr->style; } } list_entry: {} } } } /* Step 2: Find the most popular line height for each style */ cull_line_heights(lh); /* Step 3: Run through the blocks, breaking each block into two if * the line height isn't right. */ for (block_num = 0; block_num < page->len; block_num++) { int line_num; fz_text_block *block; if (page->blocks[block_num].type != FZ_PAGE_BLOCK_TEXT) continue; block = page->blocks[block_num].u.text; for (line_num = 0; line_num < block->len; line_num++) { /* For every style in the line, check to see if lineheight * is correct for that style. FIXME: We check each style * more than once, currently. */ int ok = 0; /* -1 = early exit, split now. 0 = split. 1 = don't split. */ fz_text_style *style = NULL; line = &block->lines[line_num]; if (line->distance == 0) continue; #ifdef DEBUG_LINE_HEIGHTS printf("line height=%g\n", line->distance); #endif for (span = line->first_span; span; span = span->next) { int char_num; if (is_list_entry(line, span, &char_num)) goto force_paragraph; /* Now we do the rest of the line */ for (; char_num < span->len; char_num++) { fz_text_char *chr = &span->text[char_num]; /* Ignore any whitespace chars */ if (is_unicode_wspace(chr->c)) continue; if (chr->style != style) { float proper_step = line_height_for_style(lh, chr->style); if (proper_step * 0.95 <= line->distance && line->distance <= proper_step * 1.05) { ok = 1; break; } style = chr->style; } } if (ok) break; } if (!ok) { force_paragraph: split_block(ctx, page, block_num, line_num); break; } } } free_line_heights(lh); /* Simple line region analysis: * For each line: * form a list of 'start/stop' points (henceforth a 'region mask') * find the normalised baseline vector for the line. * Store the region mask and baseline vector. * Collate lines that have compatible region masks and identical * baseline vectors. * If the collated masks are column-like, then split into columns. * Otherwise split into tables. */ rms = new_region_masks(ctx); /* Step 1: Form the region masks and store them into a list with the * normalised baseline vectors. */ for (block_num = 0; block_num < page->len; block_num++) { fz_text_block *block; if (page->blocks[block_num].type != FZ_PAGE_BLOCK_TEXT) continue; block = page->blocks[block_num].u.text; for (line = block->lines; line < block->lines + block->len; line++) { fz_point blv; region_mask *rm; #ifdef DEBUG_MASKS printf("Line: "); dump_line(line); #endif blv = line->first_span->max; blv.x -= line->first_span->min.x; blv.y -= line->first_span->min.y; fz_normalize_vector(&blv); rm = new_region_mask(ctx, &blv); for (span = line->first_span; span; span = span->next) { fz_point *region_min = &span->min; fz_point *region_max = &span->max; /* Treat adjacent spans as one big region */ while (span->next && span->next->spacing < 1.5) { span = span->next; region_max = &span->max; } region_mask_add(rm, region_min, region_max); } #ifdef DEBUG_MASKS dump_region_mask(rm); #endif region_masks_add(rms, rm); } } /* Step 2: Sort the region_masks by size of masked region */ region_masks_sort(rms); #ifdef DEBUG_MASKS printf("Sorted list of regions:\n"); dump_region_masks(rms); #endif /* Step 3: Merge the region masks where possible (large ones first) */ { int i; region_masks *rms2; rms2 = new_region_masks(ctx); for (i=0; i < rms->len; i++) { region_mask *rm = rms->mask[i]; rms->mask[i] = NULL; region_masks_merge(rms2, rm); } free_region_masks(rms); rms = rms2; } #ifdef DEBUG_MASKS printf("Merged list of regions:\n"); dump_region_masks(rms); #endif /* Step 4: Figure out alignment */ region_masks_alignment(rms); /* Step 5: At this point, we should probably look at the region masks * to try to guess which ones represent columns on the page. With our * current code, we could only get blocks of lines that span 2 or more * columns if the PDF producer wrote text out horizontally across 2 * or more columns, and we've never seen that (yet!). So we skip this * step for now. */ /* Step 6: Run through the lines again, deciding which ones fit into * which region mask. */ { region_mask *prev_match = NULL; for (block_num = 0; block_num < page->len; block_num++) { fz_text_block *block; if (page->blocks[block_num].type != FZ_PAGE_BLOCK_TEXT) continue; block = page->blocks[block_num].u.text; for (line = block->lines; line < block->lines + block->len; line++) { fz_point blv; region_mask *rm; region_mask *match; blv = line->first_span->max; blv.x -= line->first_span->min.x; blv.y -= line->first_span->min.y; fz_normalize_vector(&blv); #ifdef DEBUG_MASKS dump_line(line); #endif rm = new_region_mask(ctx, &blv); for (span = line->first_span; span; span = span->next) { fz_point *region_min = &span->min; fz_point *region_max = &span->max; /* Treat adjacent spans as one big region */ while (span->next && span->next->spacing < 1.5) { span = span->next; region_max = &span->max; } region_mask_add(rm, region_min, region_max); } #ifdef DEBUG_MASKS printf("Mask: "); dump_region_mask(rm); #endif match = region_masks_match(rms, rm, line, prev_match); prev_match = match; #ifdef DEBUG_MASKS printf("Matches: "); dump_region_mask(match); #endif free_region_mask(rm); span = line->first_span; while (span) { fz_point *region_min = &span->min; fz_point *region_max = &span->max; fz_text_span *sn; int col, align; float colw, left; /* Treat adjacent spans as one big region */ #ifdef DEBUG_ALIGN dump_span(span); #endif for (sn = span->next; sn && sn->spacing < 1.5; sn = sn->next) { region_max = &sn->max; #ifdef DEBUG_ALIGN dump_span(sn); #endif } col = region_mask_column(match, region_min, region_max, &align, &colw, &left); #ifdef DEBUG_ALIGN printf(" = col%d colw=%g align=%d\n", col, colw, align); #endif do { span->column = col; span->align = align; span->indent = left; span->column_width = colw; span = span->next; } while (span != sn); if (span) span = span->next; } line->region = match; } } free_region_masks(rms); } /* Step 7: Collate lines within a block that share the same region * mask. */ for (block_num = 0; block_num < page->len; block_num++) { int line_num; int prev_line_num; fz_text_block *block; if (page->blocks[block_num].type != FZ_PAGE_BLOCK_TEXT) continue; block = page->blocks[block_num].u.text; /* First merge lines. This may leave empty lines behind. */ for (prev_line_num = 0, line_num = 1; line_num < block->len; line_num++) { fz_text_line *prev_line; line = &block->lines[line_num]; if (!line->first_span) continue; prev_line = &block->lines[prev_line_num]; if (prev_line->region == line->region) { /* We only merge lines if the second line * only uses 1 of the columns. */ int col = line->first_span->column; /* Copy the left value for the first span * in the first column in this line forward * for all the rest of the spans in the same * column. */ float indent = line->first_span->indent; for (span = line->first_span->next; span; span = span->next) { if (col != span->column) break; span->indent = indent; } if (span) { prev_line_num = line_num; continue; } /* Merge line into prev_line */ { fz_text_span **prev_line_span = &prev_line->first_span; int try_dehyphen = -1; fz_text_span *prev_span = NULL; span = line->first_span; while (span && *prev_line_span) { /* Skip forwards through the original * line, until we find a place where * span should go. */ if ((*prev_line_span)->column <= span->column) { /* The current span we are considering * in prev_line is earlier than span. * Just skip forwards in prev_line. */ prev_span = (*prev_line_span); prev_line_span = &prev_span->next; try_dehyphen = span->column; } else { /* We want to copy span into prev_line. */ fz_text_span *next = (*prev_line_span)->next; if (prev_line_span == &prev_line->first_span) prev_line->first_span = span; if (next == NULL) prev_line->last_span = span; if (try_dehyphen == span->column) dehyphenate(prev_span, span); try_dehyphen = -1; prev_span = *prev_line_span = span; span = span->next; (*prev_line_span)->next = next; prev_line_span = &(*prev_line_span)->next; } } if (span) { *prev_line_span = span; prev_line->last_span = line->last_span; } line->first_span = NULL; line->last_span = NULL; } } else prev_line_num = line_num; } /* Now get rid of the empty lines */ for (prev_line_num = 0, line_num = 0; line_num < block->len; line_num++) { line = &block->lines[line_num]; if (line->first_span) block->lines[prev_line_num++] = *line; } block->len = prev_line_num; /* Now try to spot indents */ for (line_num = 0; line_num < block->len; line_num++) { fz_text_span *span_num, *sn; int col, count; line = &block->lines[line_num]; /* Run through the spans... */ span_num = line->first_span; { float indent = 0; /* For each set of spans that share the same * column... */ col = span_num->column; #ifdef DEBUG_INDENTS printf("Indent %g: ", span_num->indent); dump_span(span_num); printf("\n"); #endif /* find the average indent of all but the first.. */ for (sn = span_num->next, count = 0; sn && sn->column == col; sn = sn->next, count++) { #ifdef DEBUG_INDENTS printf("Indent %g: ", sn->indent); dump_span(sn); printf("\n"); #endif indent += sn->indent; sn->indent = 0; } if (sn != span_num->next) indent /= count; /* And compare this indent with the first one... */ #ifdef DEBUG_INDENTS printf("Average indent %g ", indent); #endif indent -= span_num->indent; #ifdef DEBUG_INDENTS printf("delta %g ", indent); #endif if (fabsf(indent) < 1) { /* No indent worth speaking of */ indent = 0; } #ifdef DEBUG_INDENTS printf("recorded %g\n", indent); #endif span_num->indent = indent; span_num = sn; } for (; span_num; span_num = span_num->next) { span_num->indent = 0; } } } }
void fz_text_analysis(fz_context *ctx, fz_text_sheet *sheet, fz_text_page *page) { fz_text_block *block; fz_text_line *line; fz_text_span *span; fz_text_line *prev_line; line_heights *lh; int blocknum; /* Simple paragraph analysis; look for the most common 'inter line' * spacing. This will be assumed to be our line spacing. Anything * more than 25% wider than this will be assumed to be a paragraph * space. */ /* Step 1: Gather the line height information */ lh = new_line_heights(ctx); prev_line = NULL; for (block = page->blocks; block < page->blocks + page->len; block++) { for (line = block->lines; line < block->lines + block->len; line++) { /* In a line made up of several spans, find the tallest * span. This line difference will count as being a * difference in a line of that style. */ fz_text_span *tallest_span = NULL; float tallest = 0; float span_height; for (span = line->spans; span < line->spans + line->len; span++) { span_height = span->bbox.y1 - span->bbox.y0; if (tallest_span == NULL || span_height > tallest) { tallest_span = span; tallest = span_height; } } if (prev_line) { /* Should really work on the baseline positions, * but we don't have that at this stage. */ float line_step = line->bbox.y1 - prev_line->bbox.y1; if (line_step > 0) { insert_line_height(lh, tallest_span->style, line_step); } } prev_line = line; } } /* Step 2: Find the most popular line height for each style */ cull_line_heights(lh); /* Step 3: Run through the blocks, breaking each block into two if * the line height isn't right. */ prev_line = NULL; for (blocknum = 0; blocknum < page->len; blocknum++) { block = &page->blocks[blocknum]; for (line = block->lines; line < block->lines + block->len; line++) { /* In a line made up of several spans, find the tallest * span. This line difference will count as being a * difference in a line of that style. */ fz_text_span *tallest_span = NULL; float tallest = 0; float span_height; for (span = line->spans; span < line->spans + line->len; span++) { span_height = span->bbox.y1 - span->bbox.y0; if (tallest_span == NULL || span_height > tallest) { tallest_span = span; tallest = span_height; } } if (prev_line) { float proper_step = line_height_for_style(lh, tallest_span->style); float line_step = line->bbox.y1 - prev_line->bbox.y1; if (proper_step * 0.95 > line_step || line_step > proper_step * 1.05) { split_block(ctx, page, block - page->blocks, line - block->lines); prev_line = NULL; break; } } prev_line = line; } } }