#include <validator.h>

Inheritance diagram for tesseract::Validator:

Public Member Functions
virtual	~Validator ()

Static Public Member Functions
static bool	ValidateCleanAndSegment (GraphemeNormMode g_mode, bool report_errors, const std::vector< char32 > &src, std::vector< std::vector< char32 > > *dest)

static bool	IsZeroWidthMark (char32 ch)

Static Public Attributes
static const char32	kZeroWidthSpace = 0x200B

static const char32	kZeroWidthNonJoiner = 0x200C

static const char32	kZeroWidthJoiner = 0x200D

static const char32	kLeftToRightMark = 0x200E

static const char32	kRightToLeftMark = 0x200F

static const char32	kInvalid = 0xfffd

Protected Types
enum class	CharClass { kConsonant = 'C' , kVowel = 'V' , kVirama = 'H' , kMatra = 'M' , kMatraPiece = 'P' , kVowelModifier = 'D' , kZeroWidthNonJoiner = 'z' , kZeroWidthJoiner = 'Z' , kVedicMark = 'v' , kNukta = 'N' , kRobat = 'R' , kOther = 'O' , kWhitespace = ' ' , kCombiner = 'c' }

using	IndicPair = std::pair< CharClass, char32 >

Protected Member Functions
	Validator (ViramaScript script, bool report_errors)

bool	ValidateCleanAndSegmentInternal (GraphemeNormMode g_mode, const std::vector< char32 > &src, std::vector< std::vector< char32 > > *dest)

void	MoveResultsToDest (GraphemeNormMode g_mode, std::vector< std::vector< char32 > > *dest)

bool	IsSubscriptScript () const

bool	CodeOnlyToOutput ()

void	MultiCodePart (unsigned length)

bool	UseMultiCode (unsigned length)

virtual bool	ConsumeGraphemeIfValid ()=0

void	ComputeClassCodes (const std::vector< char32 > &text)

virtual CharClass	UnicodeToCharClass (char32 ch) const =0

void	Clear ()

Static Protected Member Functions
static std::unique_ptr< Validator >	ScriptValidator (ViramaScript script, bool report_errors)

static ViramaScript	MostFrequentViramaScript (const std::vector< char32 > &utf32)

static bool	IsVirama (char32 unicode)

static bool	IsVedicAccent (char32 unicode)

Protected Attributes
ViramaScript	script_

std::vector< IndicPair >	codes_

std::vector< std::vector< char32 > >	parts_

std::vector< char32 >	output_

unsigned	codes_used_

unsigned	output_used_

bool	report_errors_

Static Protected Attributes
static const int	kIndicCodePageSize = 128

static const char32	kMinIndicUnicode = 0x900

static const char32	kMaxSinhalaUnicode = 0xdff

static const char32	kMaxViramaScriptUnicode = 0x17ff

static const char32	kSinhalaVirama = 0xdca

static const char32	kMyanmarVirama = 0x1039

static const char32	kKhmerVirama = 0x17d2

static const char32	kJavaneseVirama = 0xa9c0

static const char32	kMaxJavaneseUnicode = 0xa9df

Detailed Description

Definition at line 74 of file validator.h.

Member Typedef Documentation

◆ IndicPair

using tesseract::Validator::IndicPair = std::pair<CharClass, char32>

protected

Definition at line 135 of file validator.h.

Member Enumeration Documentation

◆ CharClass

enum class tesseract::Validator::CharClass

strongprotected

Enumerator
kConsonant
kVowel
kVirama
kMatra
kMatraPiece
kVowelModifier
kZeroWidthNonJoiner
kZeroWidthJoiner
kVedicMark
kNukta
kRobat
kOther
kWhitespace
kCombiner

Definition at line 113 of file validator.h.

                       {
    // NOTE: The values of the enum members are meaningless and arbitrary, ie
    // they are not used for sorting, or any other risky application.
    // The reason they are what they are is they are a single character
    // abbreviation that can be used in a regexp/BNF definition of a grammar,
    // IN A COMMENT, and still not relied upon in the code.
    kConsonant = 'C',
    kVowel = 'V',
    kVirama = 'H',             // (aka Halant)
    kMatra = 'M',              // (aka Dependent Vowel)
    kMatraPiece = 'P',         // unicode provides pieces of Matras.
    kVowelModifier = 'D',      // (candrabindu, anusvara, visarga, other marks)
    kZeroWidthNonJoiner = 'z', // Unicode Zero Width Non-Joiner U+200C
    kZeroWidthJoiner = 'Z',    // Unicode Zero Width Joiner U+200D
    kVedicMark = 'v',          // Modifiers can come modify any indic syllable.
    kNukta = 'N',              // Occurs only immediately after consonants.
    kRobat = 'R',              // Khmer only.
    kOther = 'O',              // (digits, measures, non-Indic, etc)
    // Additional classes used only by ValidateGrapheme.
    kWhitespace = ' ',
    kCombiner = 'c', // Combiners other than virama.
  };

Constructor & Destructor Documentation

◆ ~Validator()

tesseract::Validator::~Validator ( )

virtualdefault

◆ Validator()

tesseract::Validator::Validator	(	ViramaScript	script,
		bool	report_errors
	)

inlineprotected

Definition at line 137 of file validator.h.

138 : script_(script), codes_used_(0), output_used_(0), report_errors_(report_errors) {}

tesseract::Validator::script_

ViramaScript script_

Definition: validator.h:223

tesseract::Validator::output_used_

unsigned output_used_

Definition: validator.h:233

tesseract::Validator::codes_used_

unsigned codes_used_

Definition: validator.h:231

tesseract::Validator::report_errors_

bool report_errors_

Definition: validator.h:235

Member Function Documentation

◆ Clear()

void tesseract::Validator::Clear ( )

protected

Definition at line 198 of file validator.cpp.

                      {
  codes_.clear();
  parts_.clear();
  output_.clear();
  codes_used_ = 0;
  output_used_ = 0;
}

◆ CodeOnlyToOutput()

bool tesseract::Validator::CodeOnlyToOutput ( )

inlineprotected

Definition at line 166 of file validator.h.

                          {
    output_.push_back(codes_[codes_used_].second);
    return ++codes_used_ == codes_.size();
  }

◆ ComputeClassCodes()

void tesseract::Validator::ComputeClassCodes ( const std::vector< char32 > & text )

protected

Definition at line 190 of file validator.cpp.

                                                               {
  codes_.reserve(text.size());
  for (char32 c : text) {
    codes_.emplace_back(UnicodeToCharClass(c), c);
  }
}

◆ ConsumeGraphemeIfValid()

virtual bool tesseract::Validator::ConsumeGraphemeIfValid ( )

protectedpure virtual

Implemented in tesseract::ValidateGrapheme, tesseract::ValidateIndic, tesseract::ValidateJavanese, tesseract::ValidateKhmer, and tesseract::ValidateMyanmar.

◆ IsSubscriptScript()

bool tesseract::Validator::IsSubscriptScript ( ) const

protected

Definition at line 184 of file validator.cpp.

                                        {
  return script_ == ViramaScript::kTelugu || script_ == ViramaScript::kKannada ||
         script_ == ViramaScript::kJavanese || script_ == ViramaScript::kMyanmar ||
         script_ == ViramaScript::kKhmer;
}

◆ IsVedicAccent()

bool tesseract::Validator::IsVedicAccent ( char32 unicode )

staticprotected

Definition at line 178 of file validator.cpp.

                                            {
  return (0x1cd0 <= unicode && unicode < 0x1d00) || (0xa8e0 <= unicode && unicode <= 0xa8f7) ||
         (0x951 <= unicode && unicode <= 0x954);
}

◆ IsVirama()

bool tesseract::Validator::IsVirama ( char32 unicode )

staticprotected

Definition at line 169 of file validator.cpp.

                                       {
  return (kMinIndicUnicode <= unicode && unicode <= kMaxSinhalaUnicode &&
          (unicode & 0x7f) == 0x4d) ||
         unicode == kSinhalaVirama || unicode == kJavaneseVirama || unicode == kMyanmarVirama ||
         unicode == kKhmerVirama;
}

◆ IsZeroWidthMark()

static bool tesseract::Validator::IsZeroWidthMark ( char32 ch )

inlinestatic

Definition at line 89 of file validator.h.

                                         {
    return ch == kZeroWidthSpace || ch == kLeftToRightMark || ch == kRightToLeftMark ||
           ch == kInvalid;
  }

◆ MostFrequentViramaScript()

ViramaScript tesseract::Validator::MostFrequentViramaScript ( const std::vector< char32 > & utf32 )

staticprotected

Definition at line 135 of file validator.cpp.

                                                                               {
  std::unordered_map<int, int> histogram;
  for (char32 ch : utf32) {
    // Determine the codepage base. For the Indic scripts, Khmer and Javanese,
    // it is sufficient to divide by kIndicCodePageSize but Myanmar is all over
    // the unicode code space, so use its script id.
    int base = ch / kIndicCodePageSize;
    IcuErrorCode err;
    UScriptCode script_code = uscript_getScript(ch, err);
    if ((kMinIndicUnicode <= ch && ch <= kMaxJavaneseUnicode && script_code != USCRIPT_COMMON) ||
        script_code == USCRIPT_MYANMAR) {
      if (script_code == USCRIPT_MYANMAR) {
        base = static_cast<char32>(ViramaScript::kMyanmar) / kIndicCodePageSize;
      }
      ++histogram[base];
    }
  }
  if (!histogram.empty()) {
    int base = std::max_element(histogram.begin(), histogram.end(), CmpPairSecond)->first;
    auto codebase = static_cast<char32>(base * kIndicCodePageSize);
    // Check for validity.
    if (codebase == static_cast<char32>(ViramaScript::kMyanmar) ||
        codebase == static_cast<char32>(ViramaScript::kJavanese) ||
        codebase == static_cast<char32>(ViramaScript::kKhmer) ||
        (static_cast<char32>(ViramaScript::kDevanagari) <= codebase &&
         codebase <= static_cast<char32>(ViramaScript::kSinhala))) {
      return static_cast<ViramaScript>(codebase);
    }
  }
  return ViramaScript::kNonVirama;
}

◆ MoveResultsToDest()

void tesseract::Validator::MoveResultsToDest	(	GraphemeNormMode	g_mode,
		std::vector< std::vector< char32 > > *	dest
	)

protected

Definition at line 106 of file validator.cpp.

                                                                                             {
  if (g_mode == GraphemeNormMode::kIndividualUnicodes) {
    // Append each element of the combined output_ that we made as a new vector
    // in dest.
    dest->reserve(dest->size() + output_.size());
    for (char32 ch : output_) {
      dest->push_back({ch});
    }
  } else if (g_mode == GraphemeNormMode::kGlyphSplit) {
    // Append all the parts_ that we made onto dest.
    std::move(parts_.begin(), parts_.end(), std::back_inserter(*dest));
  } else if (g_mode == GraphemeNormMode::kCombined || dest->empty()) {
    // Append the combined output_ that we made onto dest as one new vector.
    dest->push_back(std::vector<char32>());
    output_.swap(dest->back());
  } else { // kNone.
    // Append the combined output_ that we made onto the last existing element
    // of dest.
    dest->back().insert(dest->back().end(), output_.begin(), output_.end());
  }
}

◆ MultiCodePart()

void tesseract::Validator::MultiCodePart ( unsigned length )

inlineprotected

Definition at line 176 of file validator.h.

                                      {
    while (output_used_ + length < output_.size()) {
      parts_.emplace_back(std::initializer_list<char32>{output_[output_used_++]});
    }
    parts_.emplace_back(std::initializer_list<char32>{output_[output_used_]});
    while (++output_used_ < output_.size()) {
      parts_.back().push_back(output_[output_used_]);
    }
  }

◆ ScriptValidator()

std::unique_ptr< Validator > tesseract::Validator::ScriptValidator	(	ViramaScript	script,
		bool	report_errors
	)

staticprotected

Definition at line 71 of file validator.cpp.

                                                                                           {
  switch (script) {
#define CASE(e, T) case ViramaScript::e: return std::make_unique<T>(script, report_errors)
    CASE(kNonVirama, ValidateGrapheme);
    CASE(kJavanese, ValidateJavanese);
    CASE(kMyanmar, ValidateMyanmar);
    CASE(kKhmer, ValidateKhmer);
#undef CASE
    default:
      return std::make_unique<ValidateIndic>(script, report_errors);
  }
}

◆ UnicodeToCharClass()

virtual CharClass tesseract::Validator::UnicodeToCharClass ( char32 ch ) const

protectedpure virtual

Implemented in tesseract::ValidateGrapheme, tesseract::ValidateIndic, tesseract::ValidateJavanese, tesseract::ValidateKhmer, and tesseract::ValidateMyanmar.

◆ UseMultiCode()

bool tesseract::Validator::UseMultiCode ( unsigned length )

inlineprotected

Definition at line 189 of file validator.h.

                                     {
    output_.push_back(codes_[codes_used_].second);
    MultiCodePart(length);
    return ++codes_used_ == codes_.size();
  }

◆ ValidateCleanAndSegment()

bool tesseract::Validator::ValidateCleanAndSegment	(	GraphemeNormMode	g_mode,
		bool	report_errors,
		const std::vector< char32 > &	src,
		std::vector< std::vector< char32 > > *	dest
	)

static

Definition at line 40 of file validator.cpp.

                                                                            {
  ValidateGrapheme g_validator(ViramaScript::kNonVirama, report_errors);
  std::vector<std::vector<char32>> graphemes;
  ViramaScript script = MostFrequentViramaScript(src);
  bool success = true;
  if (script == ViramaScript::kNonVirama) {
    // The grapheme segmenter's maximum segmentation is the grapheme unit, so
    // up the mode by 1 to get the desired effect.
    if (g_mode == GraphemeNormMode::kCombined) {
      g_mode = GraphemeNormMode::kGlyphSplit;
    } else if (g_mode == GraphemeNormMode::kGlyphSplit) {
      g_mode = GraphemeNormMode::kIndividualUnicodes;
    }
    // Just do grapheme segmentation.
    success = g_validator.ValidateCleanAndSegmentInternal(g_mode, src, dest);
  } else {
    success =
        g_validator.ValidateCleanAndSegmentInternal(GraphemeNormMode::kGlyphSplit, src, &graphemes);
    std::unique_ptr<Validator> validator(ScriptValidator(script, report_errors));
    for (const auto &grapheme : graphemes) {
      if (!validator->ValidateCleanAndSegmentInternal(g_mode, grapheme, dest)) {
        success = false;
      }
    }
  }
  return success;
}

◆ ValidateCleanAndSegmentInternal()

bool tesseract::Validator::ValidateCleanAndSegmentInternal	(	GraphemeNormMode	g_mode,
		const std::vector< char32 > &	src,
		std::vector< std::vector< char32 > > *	dest
	)

protected

Definition at line 89 of file validator.cpp.

                                                                                    {
  Clear();
  ComputeClassCodes(src);
  bool success = true;
  for (codes_used_ = 0; codes_used_ < codes_.size();) {
    if (!ConsumeGraphemeIfValid()) {
      success = false;
      ++codes_used_;
    }
  }
  MoveResultsToDest(g_mode, dest);
  return success;
}

Member Data Documentation

◆ codes_

std::vector<IndicPair> tesseract::Validator::codes_

protected

Definition at line 225 of file validator.h.

◆ codes_used_

unsigned tesseract::Validator::codes_used_

protected

Definition at line 231 of file validator.h.

◆ kIndicCodePageSize

const int tesseract::Validator::kIndicCodePageSize = 128

staticprotected

Definition at line 207 of file validator.h.

◆ kInvalid

const char32 tesseract::Validator::kInvalid = 0xfffd

static

Definition at line 101 of file validator.h.

◆ kJavaneseVirama

const char32 tesseract::Validator::kJavaneseVirama = 0xa9c0

staticprotected

Definition at line 219 of file validator.h.

◆ kKhmerVirama

const char32 tesseract::Validator::kKhmerVirama = 0x17d2

staticprotected

Definition at line 217 of file validator.h.

◆ kLeftToRightMark

const char32 tesseract::Validator::kLeftToRightMark = 0x200E

static

Definition at line 99 of file validator.h.

◆ kMaxJavaneseUnicode

const char32 tesseract::Validator::kMaxJavaneseUnicode = 0xa9df

staticprotected

Definition at line 220 of file validator.h.

◆ kMaxSinhalaUnicode

const char32 tesseract::Validator::kMaxSinhalaUnicode = 0xdff

staticprotected

Definition at line 211 of file validator.h.

◆ kMaxViramaScriptUnicode

const char32 tesseract::Validator::kMaxViramaScriptUnicode = 0x17ff

staticprotected

Definition at line 213 of file validator.h.

◆ kMinIndicUnicode

const char32 tesseract::Validator::kMinIndicUnicode = 0x900

staticprotected

Definition at line 209 of file validator.h.

◆ kMyanmarVirama

const char32 tesseract::Validator::kMyanmarVirama = 0x1039

staticprotected

Definition at line 216 of file validator.h.

◆ kRightToLeftMark

const char32 tesseract::Validator::kRightToLeftMark = 0x200F

static

Definition at line 100 of file validator.h.

◆ kSinhalaVirama

const char32 tesseract::Validator::kSinhalaVirama = 0xdca

staticprotected

Definition at line 215 of file validator.h.

◆ kZeroWidthJoiner

const char32 tesseract::Validator::kZeroWidthJoiner = 0x200D

static

Definition at line 98 of file validator.h.

◆ kZeroWidthNonJoiner

const char32 tesseract::Validator::kZeroWidthNonJoiner = 0x200C

static

Definition at line 97 of file validator.h.

◆ kZeroWidthSpace

const char32 tesseract::Validator::kZeroWidthSpace = 0x200B

static

Definition at line 96 of file validator.h.

◆ output_

std::vector<char32> tesseract::Validator::output_

protected

Definition at line 229 of file validator.h.

◆ output_used_

unsigned tesseract::Validator::output_used_

protected

Definition at line 233 of file validator.h.

◆ parts_

std::vector<std::vector<char32> > tesseract::Validator::parts_

protected

Definition at line 227 of file validator.h.

◆ report_errors_

bool tesseract::Validator::report_errors_

protected

Definition at line 235 of file validator.h.

◆ script_

ViramaScript tesseract::Validator::script_

protected

Definition at line 223 of file validator.h.

The documentation for this class was generated from the following files:

/media/home/debian/src/github/tesseract-ocr/tesseract/src/training/unicharset/validator.h
/media/home/debian/src/github/tesseract-ocr/tesseract/src/training/unicharset/validator.cpp

Public Member Functions

Static Public Member Functions

Static Public Attributes

Protected Types

Protected Member Functions

Static Protected Member Functions

Protected Attributes

Static Protected Attributes

Detailed Description

Member Typedef Documentation

◆ IndicPair

Member Enumeration Documentation

◆ CharClass

Constructor & Destructor Documentation

◆ ~Validator()

◆ Validator()

Member Function Documentation

◆ Clear()

◆ CodeOnlyToOutput()

◆ ComputeClassCodes()

◆ ConsumeGraphemeIfValid()

◆ IsSubscriptScript()

◆ IsVedicAccent()

◆ IsVirama()

◆ IsZeroWidthMark()

◆ MostFrequentViramaScript()

◆ MoveResultsToDest()

◆ MultiCodePart()

◆ ScriptValidator()

◆ UnicodeToCharClass()

◆ UseMultiCode()

◆ ValidateCleanAndSegment()

◆ ValidateCleanAndSegmentInternal()

Member Data Documentation

◆ codes_

◆ codes_used_

◆ kIndicCodePageSize

◆ kInvalid

◆ kJavaneseVirama

◆ kKhmerVirama

◆ kLeftToRightMark

◆ kMaxJavaneseUnicode

◆ kMaxSinhalaUnicode

◆ kMaxViramaScriptUnicode

◆ kMinIndicUnicode

◆ kMyanmarVirama

◆ kRightToLeftMark

◆ kSinhalaVirama

◆ kZeroWidthJoiner

◆ kZeroWidthNonJoiner

◆ kZeroWidthSpace

◆ output_

◆ output_used_

◆ parts_

◆ report_errors_

◆ script_