miller/docs6/docs/ngrams/mkv3.mlr

begin {
  if (is_absent(@olen)) {
    @olen = 16;
  }
  if (is_absent(@ocount)) {
    @ocount = 16;
  }
}

for (_, v in $*) {
  if (string(v) == "inf" || string(v) == "nan") {
    continue;
  }
  int n = strlen(v);
  if (n >= 2) {
    str a = substr(v, 0, 0);
    @a_histo[a] += 1;

    str b = substr(v, 1, 1);
    @ab_histo[a][b] += 1;

    for (int i = 0; i < n-2; i += 1) {
      str a = substr(v, i, i);
      str b = substr(v, i+1, i+1);
      str c = substr(v, i+2, i+2);
      @abc_histo[a][b][c] += 1;
    }
  }
}

end {

  num a_sum = 0;
  for (a, na in @a_histo) {
    a_sum += na;
  }
  num a_cumu = 0.0;
  for (a, _ in @a_histo) {
    num a_p = @a_histo[a] / a_sum;
    a_cumu += a_p;
    @a_pmf[a] = a_p;
    @a_cmf[a] += a_cumu;
  }

  for (a in @ab_histo) {

    num ab_sum = 0.0;
    for (b, nab in @ab_histo[a]) {
      ab_sum += nab;
    }
    num ab_cumu = 0.0;
    for (b, _ in @ab_histo[a]) {
      num ab_p = @ab_histo[a][b] / ab_sum;
      ab_cumu += ab_p;
      @ab_pmf[a][b] = ab_p;
      @ab_cmf[a][b] = ab_cumu;
    }
  }

  for (a in @abc_histo) {
    for (b in @abc_histo[a]) {

      num abc_sum = 0.0;
      for (c, nabc in @abc_histo[a][b]) {
        abc_sum += nabc;
      }
      num abc_cumu = 0.0;
      for (c, _ in @abc_histo[a][b]) {
        num abc_p = @abc_histo[a][b][c] / abc_sum;
        abc_cumu += abc_p;
        @abc_pmf[a][b][c] = abc_p;
        @abc_cmf[a][b][c] = abc_cumu;
      }
    }
  }

  #dump;

  for (int oi = 0; oi < @ocount; oi += 1) {
    oa_u = urand();
    oa = "?";
    for (a, c in @a_cmf) {
      oa = a;
      if (oa_u < c) {
        break;
      }
    }
    str out = oa;

    ob_u = urand();
    ob = "?";
    for (b, c in @ab_cmf[oa]) {
      ob = b;
      if (ob_u < c) {
        break;
      }
    }
    out .= ob;

    for (int i = 2; i < @olen; i += 1) {
      num abc_u = urand();
      str oc = "???";
      #print;
      #print "i=".i;
      #print "oa = ".oa." ob = ".ob;
      for (c, d in @abc_cmf[oa][ob]) {
        #print "d = ".d;
        oc = c;
        if (abc_u < d) {
          break;
        }
      }
      if (oc == "???") {
        break;
      }
      #print "i=".i.",oa=".oa.",ob=".ob.",oc=".oc;
      #print "oc = ".oc;
      out .= oc;
      oa = ob;
      ob = oc;
    }
    print out;

  }
}