*** lm/src/NgramCountLM.cc	2010/06/02 05:49:58	1.14
--- lm/src/NgramCountLM.cc	2012/02/25 03:39:56
***************
*** 56,62 ****
      countsName = 0;
      useGoogle = false;
      writeCounts = true;
-     training = false;
  
      /*
       * create a buffer into which ngrams can be copied in left-to-right order
--- 56,61 ----
***************
*** 188,199 ****
  	}
  
  	totalCount = *root;
- 
- 	if (debug(DEBUG_NGRAM_HITS)) {
- 	    dout() << "vocabsize = " << vocabSize
- 		   << " totalcount = " << totalCount
- 		   << endl;
- 	}
      }
  }
  
--- 187,192 ----
***************
*** 329,338 ****
      	return LogP_Zero;
      }
  
-     if (training) {
-     	return wordProbTrain(word, context);
-     }
- 
      VocabIndex *ngram = setupNgram(word, context);
      unsigned nlen = Vocab::length(ngram);
  
--- 322,327 ----
***************
*** 408,414 ****
  
  /*
   * Mixture weight estimation --
-  * 	- enable training mode
   *	- clear posterior statistics
   *	- run probability computation over training counts, accumulating
   *	  posterior statistics
--- 397,402 ----
***************
*** 418,425 ****
  Boolean
  NgramCountLM::estimate(NgramStats &counts)
  {
-     training = true;
- 
      LogP like;
  
      for (unsigned iter = 0; iter < maxEMiters; iter ++) {
--- 406,411 ----
***************
*** 435,449 ****
  
  	/*
  	 * Compute training set likelihood (collecting counts as a side-effect)
- 	 * Disable debugging output while doing this.
  	 */
! 	unsigned oldlevel = debuglevel();
! 	debugme(0);
! 
! 	TextStats stats;
! 	LogP newLike = countsProb(counts, stats, order);
! 
! 	debugme(oldlevel);
  
  	if (debug(DEBUG_PRINT_LIKELIHOODS)) {
  	   dout() << "iteration " << iter << ": "
--- 421,428 ----
  
  	/*
  	 * Compute training set likelihood (collecting counts as a side-effect)
  	 */
! 	LogP newLike = countsProbTrain(counts);
  
  	if (debug(DEBUG_PRINT_LIKELIHOODS)) {
  	   dout() << "iteration " << iter << ": "
***************
*** 489,506 ****
  	like = newLike;
      }
  
-     training = false;
- 
      return true;
  }
  
  /*
   * Variant of wordProb() that computes and accumulates posterior counts
!  * for each mixture weight
   */
  LogP
! NgramCountLM::wordProbTrain(VocabIndex word, const VocabIndex *context)
  {
      VocabIndex *ngram = setupNgram(word, context);
      unsigned nlen = Vocab::length(ngram);
  
--- 468,536 ----
  	like = newLike;
      }
  
      return true;
  }
  
+ 
+ /*
+  * Simplified version of LM::countsProb() that collects statistics for training
+  */
+ LogP
+ NgramCountLM::countsProbTrain(NgramStats &counts)
+ {
+     makeArray(VocabIndex, ngram, order + 1);
+ 
+     LogP totalProb = 0.0;
+ 
+     /*
+      * Enumerate all counts up to the order indicated
+      */
+     for (unsigned i = 1; i <= order; i++ ) {
+ 	NgramsIter ngramIter(counts, ngram, i);
+ 	NgramCount *count;
+ 
+ 	/*
+ 	 * This enumerates all ngrams of the given order
+ 	 */
+ 	while ((count = ngramIter.next())) {
+ 	    /*
+ 	     * Skip zero counts since they don't contribute anything to
+ 	     * the probability
+ 	     */
+ 	    if (*count == 0) {
+ 		continue;
+ 	    }
+ 
+ 	    /*
+ 	     * reverse ngram for lookup
+ 	     */
+ 	    Vocab::reverse(ngram);
+ 
+ 	    LogP prob = wordProbTrain(ngram[0], &ngram[1], *count);
+ 
+ 	    if (prob != LogP_Zero) {
+ 		totalProb += (*count) * prob;
+ 	    }
+ 
+ 	    Vocab::reverse(ngram);
+ 	}
+     }
+ 
+     return totalProb;
+ }
+ 
  /*
   * Variant of wordProb() that computes and accumulates posterior counts
!  * for each mixture weight.
!  * The count parameter indicates how often this ngram occurs in the data.
   */
  LogP
! NgramCountLM::wordProbTrain(VocabIndex word, const VocabIndex *context, NgramCount count)
  {
+     if (isNonWord(word)) {
+     	return LogP_Zero;
+     }
+ 
      VocabIndex *ngram = setupNgram(word, context);
      unsigned nlen = Vocab::length(ngram);
  
***************
*** 567,574 ****
      for (i -- ; i > 0; i --) {
  	Count contextCount = contextCounts[i-1];
  
! 	mixCounts[contextCount][i-1] += backwardCount * posteriors[i-1];
! 	mixCountTotals[contextCount][i-1] += backwardCount;
  
  	backwardCount = backwardCount * (1.0 - posteriors[i-1]);
      }
--- 597,607 ----
      for (i -- ; i > 0; i --) {
  	Count contextCount = contextCounts[i-1];
  
! 	/*
! 	 * Scale statistics by the ngram's count
! 	 */
! 	mixCounts[contextCount][i-1] += count * backwardCount * posteriors[i-1];
! 	mixCountTotals[contextCount][i-1] += count * backwardCount;
  
  	backwardCount = backwardCount * (1.0 - posteriors[i-1]);
      }
*** lm/src/NgramCountLM.h	2007/01/24 19:47:10	1.6
--- lm/src/NgramCountLM.h	2012/02/25 03:40:05
***************
*** 83,91 ****
      Count countModulus;
      void computeTotals();		/* compute totalCount and vocabSize */
  
!     Boolean training;			/* estimation mode */
!     LogP wordProbTrain(VocabIndex word, const VocabIndex *context);
!     					/* wordProb version used in training */
  };
  
  #endif /* _NgramCountLM_h_ */
--- 83,93 ----
      Count countModulus;
      void computeTotals();		/* compute totalCount and vocabSize */
  
!     /*
!      * helper functions used in training
!      */
!     LogP countsProbTrain(NgramStats &counts);
!     LogP wordProbTrain(VocabIndex word, const VocabIndex *context, NgramCount count);
  };
  
  #endif /* _NgramCountLM_h_ */
*** lm/test/reference/ngram-count-lm.stderr.dist	2006-05-21 16:45:38.000000000 -0700
--- lm/test/reference/ngram-count-lm.stderr	2012-02-24 19:34:05.787530200 -0800
***************
*** 1,24 ****
  ../ngram-count-gt/eval97.text: line 5293: 5290 sentences, 38238 words, 681 OOVs
  0 zeroprobs, logprob= 0 ppl= 1 ppl1= 1
  iteration 0: log likelihood = -91030.2
! posterior counts 0: 0/0 0/773 0/4956
! posterior counts 1: 0/0 16.3445/148.685 128.082/1348
! posterior counts 2: 0/0 6.12878/75.3333 134.307/870
! posterior counts 3: 0/0 11.6851/103 112.177/604
! posterior counts 4: 0/0 12.9503/66.35 106.903/485
! posterior counts 5: 6430.91/8373.19 11255.2/18509.1 9703.08/21051
! iteration 1: log likelihood = -87079.7
! posterior counts 0: 0/0 0/773 0/4956
! posterior counts 1: 0/0 14.4924/149.775 77.1202/1348
! posterior counts 2: 0/0 4.43878/75.3084 83.5558/870
! posterior counts 3: 0/0 9.56042/103 75.1795/604
! posterior counts 4: 0/0 11.4832/66.5068 74.9346/485
! posterior counts 5: 7086.64/8413.71 12302.9/19589 8792.6/21051
! iteration 2: log likelihood = -86886.1
! posterior counts 0: 0/0 0/773 0/4956
! posterior counts 1: 0/0 14.3582/150.311 61.1758/1348
! posterior counts 2: 0/0 4.05152/75.357 65.7269/870
! posterior counts 3: 0/0 9.03058/103 61.9853/604
! posterior counts 4: 0/0 11.1573/66.6052 63.0726/485
! posterior counts 5: 7385.26/8515.63 12837.7/20223.7 8216.12/21051
! iteration 3: log likelihood = -86867.8
--- 1,24 ----
  ../ngram-count-gt/eval97.text: line 5293: 5290 sentences, 38238 words, 681 OOVs
  0 zeroprobs, logprob= 0 ppl= 1 ppl1= 1
  iteration 0: log likelihood = -91030.2
! posterior counts 0: 0/0 0/924 0/5294
! posterior counts 1: 0/0 16.3445/149.685 133.509/1382
! posterior counts 2: 0/0 6.12878/81.3333 141.899/895
! posterior counts 3: 0/0 12.5814/112 113.842/617
! posterior counts 4: 0/0 14.207/69.7001 114.967/504
! posterior counts 5: 8081.78/10182.5 17102.1/25997.1 15008.9/28966
! iteration 1: log likelihood = -86929.5
! posterior counts 0: 0/0 0/924 0/5294
! posterior counts 1: 0/0 14.3764/150.751 79.384/1382
! posterior counts 2: 0/0 4.30199/81.2858 87.2585/895
! posterior counts 3: 0/0 9.89822/112 74.5622/617
! posterior counts 4: 0/0 12.5433/70.0096 79.3222/504
! posterior counts 5: 8238.61/9609.53 18559.9/26872.5 14315.9/28966
! iteration 2: log likelihood = -86764
! posterior counts 0: 0/0 0/924 0/5294
! posterior counts 1: 0/0 14.2557/151.297 62.2298/1382
! posterior counts 2: 0/0 3.89233/81.329 67.8292/895
! posterior counts 3: 0/0 9.23124/112 60.4488/617
! posterior counts 4: 0/0 12.2167/70.2127 65.6065/504
! posterior counts 5: 8289.44/9474.97 19330.9/27506.6 13745.4/28966
! iteration 3: log likelihood = -86729.4
*** lm/test/reference/ngram-count-lm.stdout.dist	2006-06-06 09:47:32.000000000 -0700
--- lm/test/reference/ngram-count-lm.stdout	2012-02-24 19:34:05.886540100 -0800
***************
*** 1,4 ****
  file ../ngram-count-gt/eval97.text: 5290 sentences, 38238 words, 681 OOVs
! 0 zeroprobs, logprob= -86868 ppl= 106.512 ppl1= 205.572
  file ../ngram-count-gt/eval97.text: 5290 sentences, 38238 words, 681 OOVs
! 0 zeroprobs, logprob= -85654.5 ppl= 99.788 ppl1= 190.833
--- 1,4 ----
  file ../ngram-count-gt/eval97.text: 5290 sentences, 38238 words, 681 OOVs
! 0 zeroprobs, logprob= -86729.1 ppl= 105.72 ppl1= 203.829
  file ../ngram-count-gt/eval97.text: 5290 sentences, 38238 words, 681 OOVs
! 0 zeroprobs, logprob= -85698.9 ppl= 100.026 ppl1= 191.353
