#include <validate_grapheme.h>

Inheritance diagram for tesseract::ValidateGrapheme:

Public Member Functions
	ValidateGrapheme (ViramaScript script, bool report_errors)

	~ValidateGrapheme () override=default

Public Member Functions inherited from tesseract::Validator
virtual	~Validator ()

Protected Member Functions
bool	ConsumeGraphemeIfValid () override

CharClass	UnicodeToCharClass (char32 ch) const override

Protected Member Functions inherited from tesseract::Validator
	Validator (ViramaScript script, bool report_errors)

bool	ValidateCleanAndSegmentInternal (GraphemeNormMode g_mode, const std::vector< char32 > &src, std::vector< std::vector< char32 > > *dest)

void	MoveResultsToDest (GraphemeNormMode g_mode, std::vector< std::vector< char32 > > *dest)

bool	IsSubscriptScript () const

bool	CodeOnlyToOutput ()

void	MultiCodePart (unsigned length)

bool	UseMultiCode (unsigned length)

virtual bool	ConsumeGraphemeIfValid ()=0

void	ComputeClassCodes (const std::vector< char32 > &text)

virtual CharClass	UnicodeToCharClass (char32 ch) const =0

void	Clear ()

Additional Inherited Members
Static Public Member Functions inherited from tesseract::Validator
static bool	ValidateCleanAndSegment (GraphemeNormMode g_mode, bool report_errors, const std::vector< char32 > &src, std::vector< std::vector< char32 > > *dest)

static bool	IsZeroWidthMark (char32 ch)

Static Public Attributes inherited from tesseract::Validator
static const char32	kZeroWidthSpace = 0x200B

static const char32	kZeroWidthNonJoiner = 0x200C

static const char32	kZeroWidthJoiner = 0x200D

static const char32	kLeftToRightMark = 0x200E

static const char32	kRightToLeftMark = 0x200F

static const char32	kInvalid = 0xfffd

Protected Types inherited from tesseract::Validator
enum class	CharClass { kConsonant = 'C' , kVowel = 'V' , kVirama = 'H' , kMatra = 'M' , kMatraPiece = 'P' , kVowelModifier = 'D' , kZeroWidthNonJoiner = 'z' , kZeroWidthJoiner = 'Z' , kVedicMark = 'v' , kNukta = 'N' , kRobat = 'R' , kOther = 'O' , kWhitespace = ' ' , kCombiner = 'c' }

using	IndicPair = std::pair< CharClass, char32 >

Static Protected Member Functions inherited from tesseract::Validator
static std::unique_ptr< Validator >	ScriptValidator (ViramaScript script, bool report_errors)

static ViramaScript	MostFrequentViramaScript (const std::vector< char32 > &utf32)

static bool	IsVirama (char32 unicode)

static bool	IsVedicAccent (char32 unicode)

Protected Attributes inherited from tesseract::Validator
ViramaScript	script_

std::vector< IndicPair >	codes_

std::vector< std::vector< char32 > >	parts_

std::vector< char32 >	output_

unsigned	codes_used_

unsigned	output_used_

bool	report_errors_

Static Protected Attributes inherited from tesseract::Validator
static const int	kIndicCodePageSize = 128

static const char32	kMinIndicUnicode = 0x900

static const char32	kMaxSinhalaUnicode = 0xdff

static const char32	kMaxViramaScriptUnicode = 0x17ff

static const char32	kSinhalaVirama = 0xdca

static const char32	kMyanmarVirama = 0x1039

static const char32	kKhmerVirama = 0x17d2

static const char32	kJavaneseVirama = 0xa9c0

static const char32	kMaxJavaneseUnicode = 0xa9df

Detailed Description

Definition at line 10 of file validate_grapheme.h.

Constructor & Destructor Documentation

◆ ValidateGrapheme()

tesseract::ValidateGrapheme::ValidateGrapheme	(	ViramaScript	script,
		bool	report_errors
	)

inline

Definition at line 12 of file validate_grapheme.h.

12: Validator(script, report_errors) {}

tesseract::Validator::Validator

Validator(ViramaScript script, bool report_errors)

Definition: validator.h:137

◆ ~ValidateGrapheme()

tesseract::ValidateGrapheme::~ValidateGrapheme ( )

overridedefault

Member Function Documentation

◆ ConsumeGraphemeIfValid()

bool tesseract::ValidateGrapheme::ConsumeGraphemeIfValid ( )

overrideprotectedvirtual

Implements tesseract::Validator.

Definition at line 7 of file validate_grapheme.cpp.

                                              {
  const unsigned num_codes = codes_.size();
  char32 prev_prev_ch = ' ';
  char32 prev_ch = ' ';
  CharClass prev_cc = CharClass::kWhitespace;
  int num_codes_in_grapheme = 0;
  while (codes_used_ < num_codes) {
    CharClass cc = codes_[codes_used_].first;
    char32 ch = codes_[codes_used_].second;
    const bool is_combiner = cc == CharClass::kCombiner || cc == CharClass::kVirama;
// TODO: Make this code work well with RTL text.
// See
// https://github.com/tesseract-ocr/tesseract/pull/2266#issuecomment-467114751
#if 0
    // Reject easily detected badly formed sequences.
    if (prev_cc == CharClass::kWhitespace && is_combiner) {
      if (report_errors_) tprintf("Word started with a combiner:0x%x\n", ch);
     return false;
    }
#endif
    if (prev_cc == CharClass::kVirama && cc == CharClass::kVirama) {
      if (report_errors_) {
        tprintf("Two grapheme links in a row:0x%x 0x%x\n", prev_ch, ch);
      }
      return false;
    }
    if (prev_cc != CharClass::kWhitespace && cc != CharClass::kWhitespace &&
        IsBadlyFormed(prev_ch, ch)) {
      return false;
    }
    bool prev_is_fwd_combiner = prev_ch == kZeroWidthJoiner || prev_cc == CharClass::kVirama ||
                                (prev_ch == kZeroWidthNonJoiner &&
                                 (cc == CharClass::kVirama || prev_prev_ch == kZeroWidthJoiner));
    if (num_codes_in_grapheme > 0 && !is_combiner && !prev_is_fwd_combiner) {
      break;
    }
    CodeOnlyToOutput();
    ++num_codes_in_grapheme;
    prev_prev_ch = prev_ch;
    prev_ch = ch;
    prev_cc = cc;
  }
  if (num_codes_in_grapheme > 0) {
    MultiCodePart(num_codes_in_grapheme);
  }
  return true;
}

◆ UnicodeToCharClass()

Validator::CharClass tesseract::ValidateGrapheme::UnicodeToCharClass ( char32 ch ) const

overrideprotectedvirtual

Implements tesseract::Validator.

Definition at line 55 of file validate_grapheme.cpp.

                                                                       {
  if (IsVedicAccent(ch)) {
    return CharClass::kVedicMark;
  }
  // The ZeroWidth[Non]Joiner characters are mapped to kCombiner as they
  // always combine with the previous character.
  if (u_hasBinaryProperty(ch, UCHAR_GRAPHEME_LINK)) {
    return CharClass::kVirama;
  }
  if (u_isUWhiteSpace(ch)) {
    return CharClass::kWhitespace;
  }
  // Workaround for Javanese Aksara's Taling, do not label it as a combiner
  if (ch == 0xa9ba) {
    return CharClass::kConsonant;
  }
  int char_type = u_charType(ch);
  if (char_type == U_NON_SPACING_MARK || char_type == U_ENCLOSING_MARK ||
      char_type == U_COMBINING_SPACING_MARK || ch == kZeroWidthNonJoiner ||
      ch == kZeroWidthJoiner) {
    return CharClass::kCombiner;
  }
  return CharClass::kOther;
}

The documentation for this class was generated from the following files:

/media/home/debian/src/github/tesseract-ocr/tesseract/src/training/unicharset/validate_grapheme.h
/media/home/debian/src/github/tesseract-ocr/tesseract/src/training/unicharset/validate_grapheme.cpp

Public Member Functions

Protected Member Functions