#include <validator.h>

Inheritance diagram for tesseract::Validator:

Public Member Functions
virtual	~Validator ()

Static Public Member Functions
static bool	ValidateCleanAndSegment (GraphemeNormMode g_mode, bool report_errors, const std::vector< char32 > &src, std::vector< std::vector< char32 >> *dest)

static bool	IsZeroWidthMark (char32 ch)

Static Public Attributes
static const char32	kZeroWidthSpace = 0x200B

static const char32	kZeroWidthNonJoiner = 0x200C

static const char32	kZeroWidthJoiner = 0x200D

static const char32	kLeftToRightMark = 0x200E

static const char32	kRightToLeftMark = 0x200F

static const char32	kInvalid = 0xfffd

Protected Types
enum	CharClass { CharClass::kConsonant = 'C', CharClass::kVowel = 'V', CharClass::kVirama = 'H', CharClass::kMatra = 'M', CharClass::kMatraPiece = 'P', CharClass::kVowelModifier = 'D', CharClass::kZeroWidthNonJoiner = 'z', CharClass::kZeroWidthJoiner = 'Z', CharClass::kVedicMark = 'v', CharClass::kNukta = 'N', CharClass::kRobat = 'R', CharClass::kOther = 'O', CharClass::kWhitespace = ' ', CharClass::kCombiner = 'c' }

typedef std::pair< CharClass, char32 >	IndicPair

Protected Member Functions
	Validator (ViramaScript script, bool report_errors)

bool	ValidateCleanAndSegmentInternal (GraphemeNormMode g_mode, const std::vector< char32 > &src, std::vector< std::vector< char32 >> *dest)

void	MoveResultsToDest (GraphemeNormMode g_mode, std::vector< std::vector< char32 >> *dest)

bool	IsSubscriptScript () const

bool	CodeOnlyToOutput ()

void	MultiCodePart (int length)

bool	UseMultiCode (int length)

virtual bool	ConsumeGraphemeIfValid ()=0

void	ComputeClassCodes (const std::vector< char32 > &text)

virtual CharClass	UnicodeToCharClass (char32 ch) const =0

void	Clear ()

Static Protected Member Functions
static std::unique_ptr< Validator >	ScriptValidator (ViramaScript script, bool report_errors)

static ViramaScript	MostFrequentViramaScript (const std::vector< char32 > &utf32)

static bool	IsVirama (char32 unicode)

static bool	IsVedicAccent (char32 unicode)

Protected Attributes
ViramaScript	script_

std::vector< IndicPair >	codes_

std::vector< std::vector< char32 > >	parts_

std::vector< char32 >	output_

int	codes_used_

int	output_used_

bool	report_errors_

Static Protected Attributes
static const int	kIndicCodePageSize = 128

static const char32	kMinIndicUnicode = 0x900

static const char32	kMaxSinhalaUnicode = 0xdff

static const char32	kMaxViramaScriptUnicode = 0x17ff

static const char32	kSinhalaVirama = 0xdca

static const char32	kMyanmarVirama = 0x1039

static const char32	kKhmerVirama = 0x17d2

Detailed Description

Definition at line 71 of file validator.h.

Member Typedef Documentation

◆ IndicPair

typedef std::pair<CharClass, char32> tesseract::Validator::IndicPair

protected

Definition at line 133 of file validator.h.

Member Enumeration Documentation

◆ CharClass

enum tesseract::Validator::CharClass

strongprotected

Enumerator
kConsonant
kVowel
kVirama
kMatra
kMatraPiece
kVowelModifier
kZeroWidthNonJoiner
kZeroWidthJoiner
kVedicMark
kNukta
kRobat
kOther
kWhitespace
kCombiner

Definition at line 111 of file validator.h.

                        {
     // NOTE: The values of the enum members are meaningless and arbitrary, ie
     // they are not used for sorting, or any other risky application.
     // The reason they are what they are is they are a single character
     // abbreviation that can be used in a regexp/BNF definition of a grammar,
     // IN A COMMENT, and still not relied upon in the code.
     kConsonant = 'C',
     kVowel = 'V',
     kVirama = 'H',              // (aka Halant)
     kMatra = 'M',               // (aka Dependent Vowel)
     kMatraPiece = 'P',          // unicode provides pieces of Matras.
     kVowelModifier = 'D',       // (candrabindu, anusvara, visarga, other marks)
     kZeroWidthNonJoiner = 'z',  // Unicode Zero Width Non-Joiner U+200C
     kZeroWidthJoiner = 'Z',     // Unicode Zero Width Joiner U+200D
     kVedicMark = 'v',           // Modifiers can come modify any indic syllable.
     kNukta = 'N',               // Occurs only immediately after consonants.
     kRobat = 'R',               // Khmer only.
     kOther = 'O',               // (digits, measures, non-Indic, etc)
     // Additional classes used only by ValidateGrapheme.
     kWhitespace = ' ',
     kCombiner = 'c',  // Combiners other than virama.
   };

Constructor & Destructor Documentation

◆ ~Validator()

virtual tesseract::Validator::~Validator ( )

inlinevirtual

Definition at line 91 of file validator.h.

91 {}

◆ Validator()

tesseract::Validator::Validator	(	ViramaScript	script,
		bool	report_errors
	)

inlineprotected

Definition at line 135 of file validator.h.

       : script_(script),
         codes_used_(0),
         output_used_(0),
         report_errors_(report_errors) {}

Member Function Documentation

◆ Clear()

void tesseract::Validator::Clear ( )

protected

Definition at line 198 of file validator.cpp.

                       {
   codes_.clear();
   parts_.clear();
   output_.clear();
   codes_used_ = 0;
   output_used_ = 0;
 }

◆ CodeOnlyToOutput()

bool tesseract::Validator::CodeOnlyToOutput ( )

inlineprotected

Definition at line 171 of file validator.h.

                           {
     output_.push_back(codes_[codes_used_].second);
     return ++codes_used_ == codes_.size();
   }

◆ ComputeClassCodes()

void tesseract::Validator::ComputeClassCodes ( const std::vector< char32 > & text )

protected

Definition at line 190 of file validator.cpp.

                                                                {
   codes_.reserve(text.size());
   for (char32 c : text) {
     codes_.push_back(std::make_pair(UnicodeToCharClass(c), c));
   }
 }

◆ ConsumeGraphemeIfValid()

virtual bool tesseract::Validator::ConsumeGraphemeIfValid ( )

protectedpure virtual

Implemented in tesseract::ValidateIndic, tesseract::ValidateGrapheme, tesseract::ValidateKhmer, and tesseract::ValidateMyanmar.

◆ IsSubscriptScript()

bool tesseract::Validator::IsSubscriptScript ( ) const

protected

Definition at line 184 of file validator.cpp.

                                         {
   return script_ == ViramaScript::kTelugu ||
          script_ == ViramaScript::kKannada ||
          script_ == ViramaScript::kMyanmar || script_ == ViramaScript::kKhmer;
 }

◆ IsVedicAccent()

bool tesseract::Validator::IsVedicAccent ( char32 unicode )

staticprotected

Definition at line 179 of file validator.cpp.

                                             {
   return 0x1cd0 <= unicode && unicode < 0x1d00;
 }

◆ IsVirama()

bool tesseract::Validator::IsVirama ( char32 unicode )

staticprotected

Definition at line 170 of file validator.cpp.

                                        {
   return (kMinIndicUnicode <= unicode && unicode <= kMaxSinhalaUnicode &&
           (unicode & 0x7f) == 0x4d) ||
          unicode == kSinhalaVirama || unicode == kMyanmarVirama ||
          unicode == kKhmerVirama;
 }

◆ IsZeroWidthMark()

static bool tesseract::Validator::IsZeroWidthMark ( char32 ch )

inlinestatic

Definition at line 87 of file validator.h.

                                          {
     return ch == kZeroWidthSpace || ch == kLeftToRightMark ||
            ch == kRightToLeftMark || ch == kInvalid;
   }

◆ MostFrequentViramaScript()

ViramaScript tesseract::Validator::MostFrequentViramaScript ( const std::vector< char32 > & utf32 )

staticprotected

Definition at line 134 of file validator.cpp.

                                     {
   std::unordered_map<int, int> histogram;
   for (char32 ch : utf32) {
     // Determine the codepage base. For the Indic scripts, and Khmer, it is
     // sufficient to divide by kIndicCodePageSize but Myanmar is all over the
     // unicode code space, so use its script id.
     int base = ch / kIndicCodePageSize;
     IcuErrorCode err;
     UScriptCode script_code = uscript_getScript(ch, err);
     if ((kMinIndicUnicode <= ch && ch <= kMaxViramaScriptUnicode &&
          script_code != USCRIPT_COMMON) ||
         script_code == USCRIPT_MYANMAR) {
       if (script_code == USCRIPT_MYANMAR)
         base = static_cast<char32>(ViramaScript::kMyanmar) / kIndicCodePageSize;
       ++histogram[base];
     }
   }
   if (!histogram.empty()) {
     int base =
         std::max_element(histogram.begin(), histogram.end(), CmpPairSecond)
             ->first;
     char32 codebase = static_cast<char32>(base * kIndicCodePageSize);
     // Check for validity.
     if (codebase == static_cast<char32>(ViramaScript::kMyanmar) ||
         codebase == static_cast<char32>(ViramaScript::kKhmer) ||
         (static_cast<char32>(ViramaScript::kDevanagari) <= codebase &&
          codebase <= static_cast<char32>(ViramaScript::kSinhala))) {
       return static_cast<ViramaScript>(codebase);
     }
   }
   return ViramaScript::kNonVirama;
 }

◆ MoveResultsToDest()

void tesseract::Validator::MoveResultsToDest	(	GraphemeNormMode	g_mode,
		std::vector< std::vector< char32 >> *	dest
	)

protected

Definition at line 105 of file validator.cpp.

                                                                       {
   if (g_mode == GraphemeNormMode::kIndividualUnicodes) {
     // Append each element of the combined output_ that we made as a new vector
     // in dest.
     dest->reserve(dest->size() + output_.size());
     for (char32 ch : output_) dest->push_back({ch});
   } else if (g_mode == GraphemeNormMode::kGlyphSplit) {
     // Append all the parts_ that we made onto dest.
     std::move(parts_.begin(), parts_.end(), std::back_inserter(*dest));
   } else if (g_mode == GraphemeNormMode::kCombined || dest->empty()) {
     // Append the combined output_ that we made onto dest as one new vector.
     dest->push_back(std::vector<char32>());
     output_.swap(dest->back());
   } else {  // kNone.
     // Append the combined output_ that we made onto the last existing element
     // of dest.
     dest->back().insert(dest->back().end(), output_.begin(), output_.end());
   }
 }

◆ MultiCodePart()

void tesseract::Validator::MultiCodePart ( int length )

inlineprotected

Definition at line 181 of file validator.h.

                                  {
     while (output_used_ + length < output_.size()) {
       parts_.emplace_back(
           std::initializer_list<char32>{output_[output_used_++]});
     }
     parts_.emplace_back(std::initializer_list<char32>{output_[output_used_]});
     while (++output_used_ < output_.size()) {
       parts_.back().push_back(output_[output_used_]);
     }
   }

◆ ScriptValidator()

std::unique_ptr< Validator > tesseract::Validator::ScriptValidator	(	ViramaScript	script,
		bool	report_errors
	)

staticprotected

Definition at line 65 of file validator.cpp.

                                                                           {
   switch (script) {
     case ViramaScript::kNonVirama:
       return std::unique_ptr<Validator>(
           new ValidateGrapheme(script, report_errors));
     case ViramaScript::kMyanmar:
       return std::unique_ptr<Validator>(
           new ValidateMyanmar(script, report_errors));
     case ViramaScript::kKhmer:
       return std::unique_ptr<Validator>(
           new ValidateKhmer(script, report_errors));
     default:
       return std::unique_ptr<Validator>(
           new ValidateIndic(script, report_errors));
   }
 }

◆ UnicodeToCharClass()

virtual CharClass tesseract::Validator::UnicodeToCharClass ( char32 ch ) const

protectedpure virtual

Implemented in tesseract::ValidateIndic, tesseract::ValidateGrapheme, tesseract::ValidateKhmer, and tesseract::ValidateMyanmar.

◆ UseMultiCode()

bool tesseract::Validator::UseMultiCode ( int length )

inlineprotected

Definition at line 195 of file validator.h.

                                 {
     output_.push_back(codes_[codes_used_].second);
     MultiCodePart(length);
     return ++codes_used_ == codes_.size();
   }

◆ ValidateCleanAndSegment()

bool tesseract::Validator::ValidateCleanAndSegment	(	GraphemeNormMode	g_mode,
		bool	report_errors,
		const std::vector< char32 > &	src,
		std::vector< std::vector< char32 >> *	dest
	)

static

Definition at line 34 of file validator.cpp.

                                         {
   ValidateGrapheme g_validator(ViramaScript::kNonVirama, report_errors);
   std::vector<std::vector<char32>> graphemes;
   ViramaScript script = MostFrequentViramaScript(src);
   bool success = true;
   if (script == ViramaScript::kNonVirama) {
     // The grapheme segmenter's maximum segmentation is the grapheme unit, so
     // up the mode by 1 to get the desired effect.
     if (g_mode == GraphemeNormMode::kCombined)
       g_mode = GraphemeNormMode::kGlyphSplit;
     else if (g_mode == GraphemeNormMode::kGlyphSplit)
       g_mode = GraphemeNormMode::kIndividualUnicodes;
     // Just do grapheme segmentation.
     success = g_validator.ValidateCleanAndSegmentInternal(g_mode, src, dest);
   } else {
     success = g_validator.ValidateCleanAndSegmentInternal(
         GraphemeNormMode::kGlyphSplit, src, &graphemes);
     std::unique_ptr<Validator> validator(
         ScriptValidator(script, report_errors));
     for (const auto& grapheme : graphemes) {
       if (!validator->ValidateCleanAndSegmentInternal(g_mode, grapheme, dest)) {
         success = false;
       }
     }
   }
   return success;
 }

◆ ValidateCleanAndSegmentInternal()

bool tesseract::Validator::ValidateCleanAndSegmentInternal	(	GraphemeNormMode	g_mode,
		const std::vector< char32 > &	src,
		std::vector< std::vector< char32 >> *	dest
	)

protected

Definition at line 88 of file validator.cpp.

                                         {
   Clear();
   ComputeClassCodes(src);
   bool success = true;
   for (codes_used_ = 0; codes_used_ < codes_.size();) {
     if (!ConsumeGraphemeIfValid()) {
       success = false;
       ++codes_used_;
     }
   }
   MoveResultsToDest(g_mode, dest);
   return success;
 }

Member Data Documentation

◆ codes_

std::vector<IndicPair> tesseract::Validator::codes_

protected

Definition at line 228 of file validator.h.

◆ codes_used_

int tesseract::Validator::codes_used_

protected

Definition at line 234 of file validator.h.

◆ kIndicCodePageSize

const int tesseract::Validator::kIndicCodePageSize = 128

staticprotected

Definition at line 213 of file validator.h.

◆ kInvalid

const char32 tesseract::Validator::kInvalid = 0xfffd

static

Definition at line 99 of file validator.h.

◆ kKhmerVirama

const char32 tesseract::Validator::kKhmerVirama = 0x17d2

staticprotected

Definition at line 223 of file validator.h.

◆ kLeftToRightMark

const char32 tesseract::Validator::kLeftToRightMark = 0x200E

static

Definition at line 97 of file validator.h.

◆ kMaxSinhalaUnicode

const char32 tesseract::Validator::kMaxSinhalaUnicode = 0xdff

staticprotected

Definition at line 217 of file validator.h.

◆ kMaxViramaScriptUnicode

const char32 tesseract::Validator::kMaxViramaScriptUnicode = 0x17ff

staticprotected

Definition at line 219 of file validator.h.

◆ kMinIndicUnicode

const char32 tesseract::Validator::kMinIndicUnicode = 0x900

staticprotected

Definition at line 215 of file validator.h.

◆ kMyanmarVirama

const char32 tesseract::Validator::kMyanmarVirama = 0x1039

staticprotected

Definition at line 222 of file validator.h.

◆ kRightToLeftMark

const char32 tesseract::Validator::kRightToLeftMark = 0x200F

static

Definition at line 98 of file validator.h.

◆ kSinhalaVirama

const char32 tesseract::Validator::kSinhalaVirama = 0xdca

staticprotected

Definition at line 221 of file validator.h.

◆ kZeroWidthJoiner

const char32 tesseract::Validator::kZeroWidthJoiner = 0x200D

static

Definition at line 96 of file validator.h.

◆ kZeroWidthNonJoiner

const char32 tesseract::Validator::kZeroWidthNonJoiner = 0x200C

static

Definition at line 95 of file validator.h.

◆ kZeroWidthSpace

const char32 tesseract::Validator::kZeroWidthSpace = 0x200B

static

Definition at line 94 of file validator.h.

◆ output_

std::vector<char32> tesseract::Validator::output_

protected

Definition at line 232 of file validator.h.

◆ output_used_

int tesseract::Validator::output_used_

protected

Definition at line 236 of file validator.h.

◆ parts_

std::vector<std::vector<char32> > tesseract::Validator::parts_

protected

Definition at line 230 of file validator.h.

◆ report_errors_

bool tesseract::Validator::report_errors_

protected

Definition at line 238 of file validator.h.

◆ script_

ViramaScript tesseract::Validator::script_

protected

Definition at line 226 of file validator.h.

The documentation for this class was generated from the following files:

/home/stweil/src/github/tesseract-ocr/tesseract/training/validator.h
/home/stweil/src/github/tesseract-ocr/tesseract/training/validator.cpp

Public Member Functions

Static Public Member Functions

Static Public Attributes

Protected Types

Protected Member Functions

Static Protected Member Functions

Protected Attributes

Static Protected Attributes

Detailed Description

Member Typedef Documentation

◆ IndicPair

Member Enumeration Documentation

◆ CharClass

Constructor & Destructor Documentation

◆ ~Validator()

◆ Validator()

Member Function Documentation

◆ Clear()

◆ CodeOnlyToOutput()

◆ ComputeClassCodes()

◆ ConsumeGraphemeIfValid()

◆ IsSubscriptScript()

◆ IsVedicAccent()

◆ IsVirama()

◆ IsZeroWidthMark()

◆ MostFrequentViramaScript()

◆ MoveResultsToDest()

◆ MultiCodePart()

◆ ScriptValidator()

◆ UnicodeToCharClass()

◆ UseMultiCode()

◆ ValidateCleanAndSegment()

◆ ValidateCleanAndSegmentInternal()

Member Data Documentation

◆ codes_

◆ codes_used_

◆ kIndicCodePageSize

◆ kInvalid

◆ kKhmerVirama

◆ kLeftToRightMark

◆ kMaxSinhalaUnicode

◆ kMaxViramaScriptUnicode

◆ kMinIndicUnicode

◆ kMyanmarVirama

◆ kRightToLeftMark

◆ kSinhalaVirama

◆ kZeroWidthJoiner

◆ kZeroWidthNonJoiner

◆ kZeroWidthSpace

◆ output_

◆ output_used_

◆ parts_

◆ report_errors_

◆ script_