41#include <allheaders.h>
54# define putenv(s) _putenv(s)
70 "Degrade rendered image with speckle noise, dilation/erosion "
74static BOOL_PARAM_FLAG(rotate_image,
true,
"Rotate the image in a random way.");
77static INT_PARAM_FLAG(exposure, 0,
"Exposure level in photocopier");
80static BOOL_PARAM_FLAG(distort_image,
false,
"Degrade rendered image with noise, blur, invert.");
97static BOOL_PARAM_FLAG(perspective,
false,
"Generate Perspective Distortion");
100static INT_PARAM_FLAG(box_reduction, 0,
"Integer reduction factor box_scale");
114static INT_PARAM_FLAG(max_pages, 0,
"Maximum number of pages to output (0=unlimited)");
128 "Fraction of words to underline (value in [0,1])");
132 "Fraction of words to underline (value in [0,1])");
135static INT_PARAM_FLAG(leading, 12,
"Inter-line space (in pixels)");
139 "Specify one of the following writing"
141 "'horizontal' : Render regular horizontal text. (default)\n"
142 "'vertical' : Render vertical text. Glyph orientation is"
143 " selected by Pango.\n"
144 "'vertical-upright' : Render vertical text. Glyph "
145 " orientation is set to be upright.");
147static INT_PARAM_FLAG(box_padding, 0,
"Padding around produced bounding boxes");
150 "Remove unrenderable words from source text");
155static BOOL_PARAM_FLAG(ligatures,
false,
"Rebuild and render ligatures");
157static BOOL_PARAM_FLAG(find_fonts,
false,
"Search for all fonts that can render the text");
159 "If find_fonts==true, render each font to its own image. "
160 "Image filenames are of the form output_name.font_name.tif");
162 "If find_fonts==true, the minimum coverage the font has of "
163 "the characters in the text file to include it, between "
166static BOOL_PARAM_FLAG(list_available_fonts,
false,
"List available fonts and quit.");
169 "Put each space-separated entity from the"
170 " input file into one bounding box. The ngrams in the input"
171 " file will be randomly permuted before rendering (so that"
172 " there is sufficient variety of characters on each line).");
175 "Output word bounding boxes instead of character boxes. "
176 "This is used for Cube training, and implied by "
180 "File with characters in the unicharset. If --render_ngrams"
181 " is true and --unicharset_file is specified, ngrams with"
182 " characters that are not in unicharset will be omitted");
184static BOOL_PARAM_FLAG(bidirectional_rotation,
false,
"Rotate the generated characters both ways.");
187 "Assumes that the input file contains a list of ngrams. Renders"
188 " each ngram, extracts spacing properties and records them in"
189 " output_base/[font_name].fontinfo file.");
193 "If true also outputs individual character images");
195 "Each glyph is square with this side length in pixels");
197 "Final_size=glyph_resized_size+2*glyph_num_border_pixels_to_pad");
211static bool IsWhitespaceBox(
const BoxChar *boxchar) {
215static std::string StringReplace(
const std::string &in,
const std::string &oldsub,
216 const std::string &newsub) {
218 size_t start_pos = 0, pos;
219 while ((pos = in.find(oldsub, start_pos)) != std::string::npos) {
220 out.append(in.data() + start_pos, pos - start_pos);
221 out.append(newsub.data(), newsub.length());
222 start_pos = pos + oldsub.length();
224 out.append(in.data() + start_pos, in.length() - start_pos);
238static void ExtractFontProperties(
const std::string &utf8_text,
StringRenderer *render,
239 const std::string &output_base) {
240 std::map<std::string, SpacingProperties> spacing_map;
241 std::map<std::string, SpacingProperties>::iterator spacing_map_it0;
242 std::map<std::string, SpacingProperties>::iterator spacing_map_it1;
243 int x_bearing, x_advance;
244 int len = utf8_text.length();
246 const char *text = utf8_text.c_str();
247 while (offset < len) {
248 offset += render->
RenderToImage(text + offset, strlen(text + offset),
nullptr);
249 const std::vector<BoxChar *> &boxes = render->
GetBoxes();
253 if (boxes.size() > 2 && !IsWhitespaceBox(boxes[boxes.size() - 1]) &&
254 IsWhitespaceBox(boxes[boxes.size() - 2])) {
255 if (boxes.size() > 3) {
256 tprintf(
"WARNING: Adjusting to bad page break after '%s%s'\n",
257 boxes[boxes.size() - 4]->ch().c_str(), boxes[boxes.size() - 3]->ch().c_str());
259 offset -= boxes[boxes.size() - 1]->ch().size();
262 for (
size_t b = 0; b < boxes.size(); b += 2) {
263 while (b < boxes.size() && IsWhitespaceBox(boxes[b])) {
266 if (b + 1 >= boxes.size()) {
269 const std::string &ch0 = boxes[b]->ch();
282 if (IsWhitespaceBox(boxes[b + 1])) {
285 int xgap = (boxes[b + 1]->box()->x - (boxes[b]->box()->x + boxes[b]->box()->w));
286 spacing_map_it0 = spacing_map.find(ch0);
288 if (spacing_map_it0 == spacing_map.end() &&
290 spacing_map[ch0] =
SpacingProperties(x_bearing, x_advance - x_bearing - boxes[b]->box()->w);
291 spacing_map_it0 = spacing_map.find(ch0);
294 const std::string &ch1 = boxes[b + 1]->ch();
295 tlog(3,
"%s%s\n", ch0.c_str(), ch1.c_str());
296 spacing_map_it1 = spacing_map.find(ch1);
297 if (spacing_map_it1 == spacing_map.end() &&
301 spacing_map_it1 = spacing_map.find(ch1);
305 xgap != (spacing_map_it0->second.x_gap_after + spacing_map_it1->second.x_gap_before)) {
306 spacing_map_it0->second.kerned_x_gaps[ch1] = xgap;
311 std::string output_string;
312 const int kBufSize = 1024;
314 snprintf(buf, kBufSize,
"%d\n",
static_cast<int>(spacing_map.size()));
315 output_string.append(buf);
316 std::map<std::string, SpacingProperties>::const_iterator spacing_map_it;
317 for (spacing_map_it = spacing_map.begin(); spacing_map_it != spacing_map.end();
319 snprintf(buf, kBufSize,
"%s %d %d %d", spacing_map_it->first.c_str(),
320 spacing_map_it->second.x_gap_before, spacing_map_it->second.x_gap_after,
321 static_cast<int>(spacing_map_it->second.kerned_x_gaps.size()));
322 output_string.append(buf);
323 std::map<std::string, int>::const_iterator kern_it;
324 for (kern_it = spacing_map_it->second.kerned_x_gaps.begin();
325 kern_it != spacing_map_it->second.kerned_x_gaps.end(); ++kern_it) {
326 snprintf(buf, kBufSize,
" %s %d", kern_it->first.c_str(), kern_it->second);
327 output_string.append(buf);
329 output_string.append(
"\n");
334static bool MakeIndividualGlyphs(
Image pix,
const std::vector<BoxChar *> &vbox,
335 const int input_tiff_page) {
338 tprintf(
"ERROR: MakeIndividualGlyphs(): Input Pix* is nullptr\n");
340 }
else if (FLAGS_glyph_resized_size <= 0) {
341 tprintf(
"ERROR: --glyph_resized_size must be positive\n");
343 }
else if (FLAGS_glyph_num_border_pixels_to_pad < 0) {
344 tprintf(
"ERROR: --glyph_num_border_pixels_to_pad must be 0 or positive\n");
348 const int n_boxes = vbox.size();
349 int n_boxes_saved = 0;
350 int current_tiff_page = 0;
352 static int glyph_count = 0;
353 for (
int i = 0;
i < n_boxes;
i++) {
355 Box *b = vbox[
i]->mutable_box();
364 if (
y < y_previous - pixGetHeight(pix) / 10) {
365 tprintf(
"ERROR: Wrap-around encountered, at i=%d\n",
i);
368 if (current_tiff_page < input_tiff_page) {
370 }
else if (current_tiff_page > input_tiff_page) {
374 if (
x < 0 ||
y < 0 || (
x + w - 1) >= pixGetWidth(pix) || (
y + h - 1) >= pixGetHeight(pix)) {
376 "ERROR: MakeIndividualGlyphs(): Index out of range, at i=%d"
377 " (x=%d, y=%d, w=%d, h=%d\n)",
380 }
else if (w < FLAGS_glyph_num_border_pixels_to_pad &&
381 h < FLAGS_glyph_num_border_pixels_to_pad) {
382 tprintf(
"ERROR: Input image too small to be a character, at i=%d\n",
i);
386 Image pix_glyph = pixClipRectangle(pix, b,
nullptr);
388 tprintf(
"ERROR: MakeIndividualGlyphs(): Failed to clip, at i=%d\n",
i);
393 pixScaleToSize(pix_glyph, FLAGS_glyph_resized_size, FLAGS_glyph_resized_size);
395 tprintf(
"ERROR: MakeIndividualGlyphs(): Failed to resize, at i=%d\n",
i);
399 Image pix_glyph_sq_pad = pixAddBorder(pix_glyph_sq, FLAGS_glyph_num_border_pixels_to_pad, 0);
400 if (!pix_glyph_sq_pad) {
401 tprintf(
"ERROR: MakeIndividualGlyphs(): Failed to zero-pad, at i=%d\n",
i);
405 Image pix_glyph_sq_pad_8 = pixConvertTo8(pix_glyph_sq_pad,
false);
407 snprintf(filename, 1024,
"%s_%d.jpg", FLAGS_outputbase.c_str(), glyph_count++);
408 if (pixWriteJpeg(filename, pix_glyph_sq_pad_8, 100, 0)) {
410 "ERROR: MakeIndividualGlyphs(): Failed to write JPEG to %s,"
423 if (n_boxes_saved == 0) {
426 tprintf(
"Total number of characters saved = %d\n", n_boxes_saved);
433using tesseract::ExtractFontProperties;
441 if (FLAGS_list_available_fonts) {
442 const std::vector<std::string> &all_fonts = FontUtils::ListAvailableFonts();
443 for (
unsigned int i = 0;
i < all_fonts.size(); ++
i) {
447 std::string font_name(all_fonts[
i].c_str());
448 if (font_name.back() ==
',') {
449 font_name.pop_back();
451 printf(
"%3u: %s\n",
i, font_name.c_str());
453 "Font %s is unrecognized.\n", all_fonts[
i].c_str());
459 if (FLAGS_text.empty()) {
460 tprintf(
"'--text' option is missing!\n");
463 if (FLAGS_outputbase.empty()) {
464 tprintf(
"'--outputbase' option is missing!\n");
467 if (!FLAGS_unicharset_file.empty() && FLAGS_render_ngrams) {
468 tprintf(
"Use '--unicharset_file' only if '--render_ngrams' is set.\n");
472 std::string font_name = FLAGS_font.c_str();
473 if (!FLAGS_find_fonts && !FontUtils::IsAvailableFont(font_name.c_str())) {
475 std::string pango_name;
476 if (!FontUtils::IsAvailableFont(font_name.c_str(), &pango_name)) {
477 tprintf(
"Could not find font named '%s'.\n", FLAGS_font.c_str());
478 if (!pango_name.empty()) {
479 tprintf(
"Pango suggested font '%s'.\n", pango_name.c_str());
481 tprintf(
"Please correct --font arg.\n");
486 if (FLAGS_render_ngrams) {
487 FLAGS_output_word_boxes =
true;
490 char font_desc_name[1024];
491 snprintf(font_desc_name, 1024,
"%s %d", font_name.c_str(),
static_cast<int>(FLAGS_ptsize));
507 if (FLAGS_writing_mode ==
"horizontal") {
512 }
else if (FLAGS_writing_mode ==
"vertical") {
517 }
else if (FLAGS_writing_mode ==
"vertical-upright") {
527 tprintf(
"Invalid writing mode: %s\n", FLAGS_writing_mode.c_str());
531 std::string src_utf8;
533 if (!File::ReadFileToString(FLAGS_text.c_str(), &src_utf8)) {
534 tprintf(
"Failed to read file: %s\n", FLAGS_text.c_str());
539 if (strncmp(src_utf8.c_str(),
"\xef\xbb\xbf", 3) == 0) {
540 src_utf8.erase(0, 3);
542 tlog(1,
"Render string of size %zu\n", src_utf8.length());
544 if (FLAGS_render_ngrams || FLAGS_only_extract_font_properties) {
547 const std::string kSeparator = FLAGS_render_ngrams ?
" " :
" ";
551 const unsigned int kCharsPerLine = (FLAGS_ptsize > 20) ? 50 : 100;
552 std::string rand_utf8;
554 if (FLAGS_render_ngrams && !FLAGS_unicharset_file.empty() &&
556 tprintf(
"Failed to load unicharset from file %s\n", FLAGS_unicharset_file.c_str());
563 const char *str8 = src_utf8.c_str();
564 int len = src_utf8.length();
566 std::vector<std::pair<int, int>> offsets;
568 while (offset < len) {
570 offsets.emplace_back(offset, step);
574 if (FLAGS_render_ngrams) {
576 std::mt19937 random_gen(seed);
577 std::shuffle(offsets.begin(), offsets.end(), random_gen);
580 for (
size_t i = 0, line = 1;
i < offsets.size(); ++
i) {
581 const char *curr_pos = str8 + offsets[
i].first;
582 int ngram_len = offsets[
i].second;
584 std::string cleaned = UNICHARSET::CleanupString(curr_pos, ngram_len);
585 if (!FLAGS_unicharset_file.empty() &&
589 rand_utf8.append(curr_pos, ngram_len);
590 if (rand_utf8.length() > line * kCharsPerLine) {
591 rand_utf8.append(
" \n");
594 rand_utf8.append(kSeparator);
597 rand_utf8.append(kSeparator);
600 tlog(1,
"Rendered ngram string of size %zu\n", rand_utf8.length());
601 src_utf8.swap(rand_utf8);
603 if (FLAGS_only_extract_font_properties) {
604 tprintf(
"Extracting font properties only\n");
605 ExtractFontProperties(src_utf8, &render, FLAGS_outputbase.c_str());
611 std::vector<float> page_rotation;
612 const char *to_render_utf8 = src_utf8.c_str();
616 std::vector<std::string> font_names;
620 int num_pass = FLAGS_bidirectional_rotation ? 2 : 1;
621 for (
int pass = 0; pass < num_pass; ++pass) {
623 std::string font_used;
624 for (
size_t offset = 0;
625 offset < strlen(to_render_utf8) && (FLAGS_max_pages == 0 || page_num < FLAGS_max_pages);
627 tlog(1,
"Starting page %d\n", im);
629 if (FLAGS_find_fonts) {
631 strlen(to_render_utf8 + offset), &font_used, &pix);
634 render.
RenderToImage(to_render_utf8 + offset, strlen(to_render_utf8 + offset), &pix);
636 if (pix !=
nullptr) {
640 rotation = -1 * page_rotation[page_num];
642 if (FLAGS_degrade_image) {
644 FLAGS_rotate_image ? &rotation :
nullptr);
646 if (FLAGS_distort_image) {
649 FLAGS_blur, 1, &randomizer,
nullptr);
655 page_rotation.push_back(rotation);
658 Image gray_pix = pixConvertTo8(pix,
false);
660 Image binary = pixThresholdToBinary(gray_pix, 128);
662 char tiff_name[1024];
663 if (FLAGS_find_fonts) {
664 if (FLAGS_render_per_font) {
665 std::string fontname_for_file = tesseract::StringReplace(font_used,
" ",
"_");
666 snprintf(tiff_name, 1024,
"%s.%s.tif", FLAGS_outputbase.c_str(),
667 fontname_for_file.c_str());
668 pixWriteTiff(tiff_name, binary, IFF_TIFF_G4,
"w");
669 tprintf(
"Rendered page %d to file %s\n", im, tiff_name);
671 font_names.push_back(font_used);
674 snprintf(tiff_name, 1024,
"%s.tif", FLAGS_outputbase.c_str());
675 pixWriteTiff(tiff_name, binary, IFF_TIFF_G4, im == 0 ?
"w" :
"a");
676 tprintf(
"Rendered page %d to file %s\n", im, tiff_name);
679 if (FLAGS_output_individual_glyph_images) {
680 if (!MakeIndividualGlyphs(binary, render.
GetBoxes(), im)) {
681 tprintf(
"ERROR: Individual glyphs not saved\n");
686 if (FLAGS_find_fonts && offset != 0) {
693 if (!FLAGS_find_fonts) {
694 std::string box_name = FLAGS_outputbase.c_str();
697 }
else if (!FLAGS_render_per_font && !font_names.empty()) {
698 std::string filename = FLAGS_outputbase.c_str();
699 filename +=
".fontlist.txt";
700 FILE *fp = fopen(filename.c_str(),
"wb");
702 tprintf(
"Failed to create output font list %s\n", filename.c_str());
704 for (
auto &font_name : font_names) {
705 fprintf(fp,
"%s\n", font_name.c_str());
714int main(
int argc,
char **argv) {
720 backend = getenv(
"PANGOCAIRO_BACKEND");
721 if (backend ==
nullptr) {
722 static char envstring[] =
"PANGOCAIRO_BACKEND=fc";
726 "Using '%s' as pango cairo backend based on environment "
730 tesseract::CheckSharedLibraryVersion();
732 if ((strcmp(argv[1],
"-v") == 0) || (strcmp(argv[1],
"--version") == 0)) {
733 FontUtils::PangoFontTypeInfo();
734 printf(
"Pango version: %s\n", pango_version_string());
#define ASSERT_HOST_MSG(x,...)
int main(int argc, char **argv)
#define DOUBLE_PARAM_FLAG(name, val, comment)
#define BOOL_PARAM_FLAG(name, val, comment)
#define INT_PARAM_FLAG(name, val, comment)
#define STRING_PARAM_FLAG(name, val, comment)
void ParseCommandLineFlags(const char *usage, int *argc, char ***argv, const bool remove_flags)
unsigned int SpanUTF8Whitespace(const char *text)
void tprintf(const char *format,...)
Image PrepareDistortedPix(const Image pix, bool perspective, bool invert, bool white_noise, bool smooth_noise, bool blur, int box_reduction, TRand *randomizer, std::vector< TBOX > *boxes)
Image DegradeImage(Image input, int exposure, TRand *randomizer, float *rotation)
unsigned int SpanUTF8NotWhitespace(const char *text)
void set_seed(uint64_t seed)
bool encodable_string(const char *str, unsigned *first_bad_position) const
bool load_from_file(const char *const filename, bool skip_fragments)
const std::string & ch() const
bool GetSpacingProperties(const std::string &utf8_char, int *x_bearing, int *x_advance) const
int RenderToImage(const char *text, int text_length, Image *pix)
void set_underline_start_prob(const double frac)
void set_vertical_text(bool vertical_text)
int RenderAllFontsToImage(double min_coverage, const char *text, int text_length, std::string *font_used, Image *pix)
void set_render_fullwidth_latin(bool render_fullwidth_latin)
const std::vector< BoxChar * > & GetBoxes() const
void set_gravity_hint_strong(bool gravity_hint_strong)
void set_resolution(const int resolution)
const PangoFontInfo & font() const
void set_underline_continuation_prob(const double frac)
void set_strip_unrenderable_words(bool val)
void set_add_ligatures(bool add_ligatures)
void set_h_margin(const int h_margin)
void set_char_spacing(int char_spacing)
void set_output_word_boxes(bool val)
void set_v_margin(const int v_margin)
void WriteAllBoxes(const std::string &filename)
void RotatePageBoxes(float rotation)
void set_leading(int leading)
void set_box_padding(int val)
SpacingProperties(int b, int a)
std::map< std::string, int > kerned_x_gaps
static void WriteStringToFileOrDie(const std::string &str, const std::string &filename)