tesseract::WeightMatrix Class Reference

#include <weightmatrix.h>

Public Member Functions
	WeightMatrix ()
int	InitWeightsFloat (int no, int ni, bool ada_grad, float weight_range, TRand *randomizer)
void	ConvertToInt ()
bool	is_int_mode () const
int	NumOutputs () const
const double *	GetWeights (int index) const
double	GetDW (int i, int j) const
void	InitBackward ()
bool	Serialize (bool training, TFile *fp) const
bool	DeSerialize (bool training, TFile *fp)
bool	DeSerializeOld (bool training, TFile *fp)
void	MatrixDotVector (const double *u, double *v) const
void	MatrixDotVector (const inT8 *u, double *v) const
void	MultiplyAccumulate (const double *v, double *inout)
void	VectorDotMatrix (const double *u, double *v) const
void	SumOuterTransposed (const TransposedArray &u, const TransposedArray &v, bool parallel)
void	Update (double learning_rate, double momentum, int num_samples)
void	AddDeltas (const WeightMatrix &other)
void	CountAlternators (const WeightMatrix &other, double *same, double *changed) const
void	Debug2D (const char *msg)

Static Public Member Functions
static double	DotProduct (const double *u, const double *v, int n)
static void	FloatToDouble (const GENERIC_2D_ARRAY< float > &wf, GENERIC_2D_ARRAY< double > *wd)

Detailed Description

Definition at line 63 of file weightmatrix.h.

Constructor & Destructor Documentation

WeightMatrix()

tesseract::WeightMatrix::WeightMatrix ( )

inline

Definition at line 65 of file weightmatrix.h.

: int_mode_(false), use_ada_grad_(false) {}

Member Function Documentation

AddDeltas()

void tesseract::WeightMatrix::AddDeltas

(

const WeightMatrix &

other

)

Definition at line 267 of file weightmatrix.cpp.

                                                      {
  ASSERT_HOST(dw_.dim1() == other.dw_.dim1());
  ASSERT_HOST(dw_.dim2() == other.dw_.dim2());
  dw_ += other.dw_;
}

ConvertToInt()

void tesseract::WeightMatrix::ConvertToInt

(

)

Definition at line 62 of file weightmatrix.cpp.

                                {
  wi_.ResizeNoInit(wf_.dim1(), wf_.dim2());
  scales_.init_to_size(wi_.dim1(), 0.0);
  int dim2 = wi_.dim2();
  for (int t = 0; t < wi_.dim1(); ++t) {
    double* f_line = wf_[t];
    inT8* i_line = wi_[t];
    double max_abs = 0.0;
    for (int f = 0; f < dim2; ++f) {
      double abs_val = fabs(f_line[f]);
      if (abs_val > max_abs) max_abs = abs_val;
    }
    double scale = max_abs / MAX_INT8;
    scales_[t] = scale;
    if (scale == 0.0) scale = 1.0;
    for (int f = 0; f < dim2; ++f) {
      i_line[f] = IntCastRounded(f_line[f] / scale);
    }
  }
  wf_.Resize(1, 1, 0.0);
  int_mode_ = true;
}

CountAlternators()

void tesseract::WeightMatrix::CountAlternators	(	const WeightMatrix &	other,
		double *	same,
		double *	changed
	)		const

Definition at line 276 of file weightmatrix.cpp.

                                                           {
  int num_outputs = updates_.dim1();
  int num_inputs = updates_.dim2();
  ASSERT_HOST(num_outputs == other.updates_.dim1());
  ASSERT_HOST(num_inputs == other.updates_.dim2());
  for (int i = 0; i < num_outputs; ++i) {
    const double* this_i = updates_[i];
    const double* other_i = other.updates_[i];
    for (int j = 0; j < num_inputs; ++j) {
      double product = this_i[j] * other_i[j];
      if (product < 0.0)
        *changed -= product;
      else
        *same += product;
    }
  }
}

Debug2D()

void tesseract::WeightMatrix::Debug2D

(

const char *

msg

)

Definition at line 307 of file weightmatrix.cpp.

                                          {
  STATS histogram(0, kHistogramBuckets);
  if (int_mode_) {
    for (int i = 0; i < wi_.dim1(); ++i) {
      for (int j = 0; j < wi_.dim2(); ++j) {
        HistogramWeight(wi_[i][j] * scales_[i], &histogram);
      }
    }
  } else {
    for (int i = 0; i < wf_.dim1(); ++i) {
      for (int j = 0; j < wf_.dim2(); ++j) {
        HistogramWeight(wf_[i][j], &histogram);
      }
    }
  }
  tprintf("%s\n", msg);
  histogram.print();
}

DeSerialize()

bool tesseract::WeightMatrix::DeSerialize	(	bool	training,
		TFile *	fp
	)

Definition at line 125 of file weightmatrix.cpp.

                                                       {
  uinT8 mode = 0;
  if (fp->FRead(&mode, sizeof(mode), 1) != 1) return false;
  int_mode_ = (mode & kInt8Flag) != 0;
  use_ada_grad_ = (mode & kAdaGradFlag) != 0;
  if ((mode & kDoubleFlag) == 0) return DeSerializeOld(training, fp);
  if (int_mode_) {
    if (!wi_.DeSerialize(fp)) return false;
    if (!scales_.DeSerialize(fp)) return false;
  } else {
    if (!wf_.DeSerialize(fp)) return false;
    if (training) {
      InitBackward();
      if (!updates_.DeSerialize(fp)) return false;
      if (use_ada_grad_ && !dw_sq_sum_.DeSerialize(fp)) return false;
    }
  }
  return true;
}

DeSerializeOld()

bool tesseract::WeightMatrix::DeSerializeOld	(	bool	training,
		TFile *	fp
	)

Definition at line 147 of file weightmatrix.cpp.

                                                          {
  GENERIC_2D_ARRAY<float> float_array;
  if (int_mode_) {
    if (!wi_.DeSerialize(fp)) return false;
    GenericVector<float> old_scales;
    if (!old_scales.DeSerialize(fp)) return false;
    scales_.resize_no_init(old_scales.size());
    for (int i = 0; i < old_scales.size(); ++i) scales_[i] = old_scales[i];
  } else {
    if (!float_array.DeSerialize(fp)) return false;
    FloatToDouble(float_array, &wf_);
  }
  if (training) {
    InitBackward();
    if (!float_array.DeSerialize(fp)) return false;
    FloatToDouble(float_array, &updates_);
    // Errs was only used in int training, which is now dead.
    if (!float_array.DeSerialize(fp)) return false;
  }
  return true;
}

DotProduct()

double tesseract::WeightMatrix::DotProduct ( const double *  u, const double *  v, int  n  )

static

Definition at line 328 of file weightmatrix.cpp.

                                                                       {
  // Note: because the order of addition is different among the 3 DotProduct
  // functions, the results can (and do) vary slightly (although they agree
  // to within about 4e-15). This produces different results when running
  // training, despite all random inputs being precisely equal.
  // To get consistent results, use just one of these DotProduct functions.
  // On a test multi-layer network, serial is 57% slower than sse, and avx
  // is about 8% faster than sse. This suggests that the time is memory
  // bandwidth constrained and could benefit from holding the reused vector
  // in AVX registers.
/*
omp simd code
real    4m17,294s
user    12m39,344s
sys     0m2,252s

real    4m22,403s
user    12m53,408s
sys     0m2,116s

old code
real    2m52,396s
user    7m42,624s
sys     0m2,008s
*/

#ifndef _OPENMP
  if (SIMDDetect::IsAVXAvailable()) return DotProductAVX(u, v, n);
  if (SIMDDetect::IsSSEAvailable()) return DotProductSSE(u, v, n);
#endif
  double total = 0.0;
#ifdef _OPENMP
#pragma omp simd
#endif
  for (int k = 0; k < n; ++k) total += u[k] * v[k];
  return total;
}

FloatToDouble()

void tesseract::WeightMatrix::FloatToDouble ( const GENERIC_2D_ARRAY< float > &  wf, GENERIC_2D_ARRAY< double > *  wd  )

static

Definition at line 369 of file weightmatrix.cpp.

                                                               {
  int dim1 = wf.dim1();
  int dim2 = wf.dim2();
  wd->ResizeNoInit(dim1, dim2);
  for (int i = 0; i < dim1; ++i) {
    const float* wfi = wf[i];
    double* wdi = (*wd)[i];
    for (int j = 0; j < dim2; ++j) wdi[j] = static_cast<double>(wfi[j]);
  }
}

GetDW()

double tesseract::WeightMatrix::GetDW ( int  i, int  j  ) const

inline

Definition at line 91 of file weightmatrix.h.

{ return dw_(i, j); }

GetWeights()

const double* tesseract::WeightMatrix::GetWeights ( int  index ) const

inline

Definition at line 89 of file weightmatrix.h.

{ return wf_[index]; }

InitBackward()

void tesseract::WeightMatrix::InitBackward

(

)

Definition at line 87 of file weightmatrix.cpp.

                                {
  int no = int_mode_ ? wi_.dim1() : wf_.dim1();
  int ni = int_mode_ ? wi_.dim2() : wf_.dim2();
  dw_.Resize(no, ni, 0.0);
  updates_.Resize(no, ni, 0.0);
  wf_t_.Transpose(wf_);
  if (use_ada_grad_) dw_sq_sum_.Resize(no, ni, 0.0);
}

InitWeightsFloat()

int tesseract::WeightMatrix::InitWeightsFloat	(	int	no,
		int	ni,
		bool	ada_grad,
		float	weight_range,
		TRand *	randomizer
	)

Definition at line 39 of file weightmatrix.cpp.

                                                                          {
  int_mode_ = false;
  wf_.Resize(no, ni, 0.0);
  if (randomizer != NULL) {
    for (int i = 0; i < no; ++i) {
      for (int j = 0; j < ni; ++j) {
        wf_[i][j] = randomizer->SignedRand(weight_range);
      }
    }
  }
  use_ada_grad_ = ada_grad;
  InitBackward();
  return ni * no;
}

is_int_mode()

bool tesseract::WeightMatrix::is_int_mode ( ) const

inline

Definition at line 84 of file weightmatrix.h.

                           {
    return int_mode_;
  }

MatrixDotVector() [1/2]

void tesseract::WeightMatrix::MatrixDotVector	(	const double *	u,
		double *	v
	)		const

Definition at line 174 of file weightmatrix.cpp.

                                                                   {
  ASSERT_HOST(!int_mode_);
  MatrixDotVectorInternal(wf_, true, false, u, v);
}

MatrixDotVector() [2/2]

void tesseract::WeightMatrix::MatrixDotVector	(	const inT8 *	u,
		double *	v
	)		const

Definition at line 179 of file weightmatrix.cpp.

                                                                 {
  ASSERT_HOST(int_mode_);
  int num_out = wi_.dim1();
  int num_in = wi_.dim2() - 1;
  for (int i = 0; i < num_out; ++i) {
    const inT8* Wi = wi_[i];
    int total = 0;
    if (SIMDDetect::IsSSEAvailable()) {
      total = IntDotProductSSE(u, Wi, num_in);
    } else {
      for (int j = 0; j < num_in; ++j) total += Wi[j] * u[j];
    }
    // Add in the bias and correct for integer values.
    v[i] = (static_cast<double>(total) / MAX_INT8 + Wi[num_in]) * scales_[i];
  }
}

MultiplyAccumulate()

void tesseract::WeightMatrix::MultiplyAccumulate	(	const double *	v,
		double *	inout
	)

Definition at line 198 of file weightmatrix.cpp.

                                                                    {
  ASSERT_HOST(!int_mode_);
  ASSERT_HOST(wf_.dim1() == 1);
  int n = wf_.dim2();
  const double* u = wf_[0];
  for (int i = 0; i < n; ++i) {
    inout[i] += u[i] * v[i];
  }
}

NumOutputs()

int tesseract::WeightMatrix::NumOutputs ( ) const

inline

Definition at line 87 of file weightmatrix.h.

{ return int_mode_ ? wi_.dim1() : wf_.dim1(); }

Serialize()

bool tesseract::WeightMatrix::Serialize	(	bool	training,
		TFile *	fp
	)		const

Definition at line 106 of file weightmatrix.cpp.

                                                           {
  // For backward compatibility, add kDoubleFlag to mode to indicate the doubles
  // format, without errs, so we can detect and read old format weight matrices.
  uinT8 mode = (int_mode_ ? kInt8Flag : 0) |
               (use_ada_grad_ ? kAdaGradFlag : 0) | kDoubleFlag;
  if (fp->FWrite(&mode, sizeof(mode), 1) != 1) return false;
  if (int_mode_) {
    if (!wi_.Serialize(fp)) return false;
    if (!scales_.Serialize(fp)) return false;
  } else {
    if (!wf_.Serialize(fp)) return false;
    if (training && !updates_.Serialize(fp)) return false;
    if (training && use_ada_grad_ && !dw_sq_sum_.Serialize(fp)) return false;
  }
  return true;
}

SumOuterTransposed()

void tesseract::WeightMatrix::SumOuterTransposed	(	const TransposedArray &	u,
		const TransposedArray &	v,
		bool	parallel
	)

Definition at line 222 of file weightmatrix.cpp.

                                                        {
  ASSERT_HOST(!int_mode_);
  int num_outputs = dw_.dim1();
  ASSERT_HOST(u.dim1() == num_outputs);
  ASSERT_HOST(u.dim2() == v.dim2());
  int num_inputs = dw_.dim2() - 1;
  int num_samples = u.dim2();
  // v is missing the last element in dim1.
  ASSERT_HOST(v.dim1() == num_inputs);
#ifdef _OPENMP
#pragma omp parallel for num_threads(4) if (in_parallel)
#endif
  for (int i = 0; i < num_outputs; ++i) {
    double* dwi = dw_[i];
    const double* ui = u[i];
    for (int j = 0; j < num_inputs; ++j) {
      dwi[j] = DotProduct(ui, v[j], num_samples);
    }
    // The last element of v is missing, presumed 1.0f.
    double total = 0.0;
    for (int k = 0; k < num_samples; ++k) total += ui[k];
    dwi[num_inputs] = total;
  }
}

Update()

void tesseract::WeightMatrix::Update	(	double	learning_rate,
		double	momentum,
		int	num_samples
	)

Definition at line 252 of file weightmatrix.cpp.

                                           {
  ASSERT_HOST(!int_mode_);
  if (use_ada_grad_ && num_samples > 0) {
    dw_sq_sum_.SumSquares(dw_);
    dw_.AdaGradScaling(dw_sq_sum_, num_samples);
  }
  dw_ *= learning_rate;
  updates_ += dw_;
  if (momentum > 0.0) wf_ += updates_;
  if (momentum >= 0.0) updates_ *= momentum;
  wf_t_.Transpose(wf_);
}

VectorDotMatrix()

void tesseract::WeightMatrix::VectorDotMatrix	(	const double *	u,
		double *	v
	)		const

Definition at line 212 of file weightmatrix.cpp.

                                                                   {
  ASSERT_HOST(!int_mode_);
  MatrixDotVectorInternal(wf_t_, false, true, u, v);
}

---

The documentation for this class was generated from the following files:

• /home/stefan/src/github/tesseract-ocr/tesseract/lstm/weightmatrix.h
• /home/stefan/src/github/tesseract-ocr/tesseract/lstm/weightmatrix.cpp