miller/doc/ngrams/mkv5.mlr

begin {
  if (isabsent(@olen)) {
    @olen = 16;
  }
  if (isabsent(@ocount)) {
    @ocount = 16;
  }
}

for (_, v in $*) {
  if (string(v) == "inf" || string(v) == "nan") {
    continue;
  }
  int n = strlen(v);
  if (n >= 4) {
    str a = substr(v, 0, 0);
    str b = substr(v, 1, 1);
    str c = substr(v, 2, 2);
    str d = substr(v, 3, 3);
    @a_histo[a] += 1;
    @ab_histo[a][b] += 1;
    @abc_histo[a][b][c] += 1;
    @abcd_histo[a][b][c][d] += 1;

    for (int i = 0; i < n-4; i += 1) {
      str a = substr(v, i, i);
      str b = substr(v, i+1, i+1);
      str c = substr(v, i+2, i+2);
      str d = substr(v, i+3, i+3);
      str e = substr(v, i+4, i+4);
      @abcde_histo[a][b][c][d][e] += 1;
    }
  }
}

end {

  num a_sum = 0;
  for (a, na in @a_histo) {
    a_sum += na;
  }
  num a_cumu = 0.0;
  for (a, _ in @a_histo) {
    num a_p = @a_histo[a] / a_sum;
    a_cumu += a_p;
    @a_pmf[a] = a_p;
    @a_cmf[a] += a_cumu;
  }

  for (a in @ab_histo) {
    num ab_sum = 0.0;
    for (b, nab in @ab_histo[a]) {
      ab_sum += nab;
    }
    num ab_cumu = 0.0;
    for (b, _ in @ab_histo[a]) {
      num ab_p = @ab_histo[a][b] / ab_sum;
      ab_cumu += ab_p;
      @ab_pmf[a][b] = ab_p;
      @ab_cmf[a][b] = ab_cumu;
    }
  }

  for (a in @abc_histo) {
    for (b in @abc_histo[a]) {
      num abc_sum = 0.0;
      for (c, nabc in @abc_histo[a][b]) {
        abc_sum += nabc;
      }
      num abc_cumu = 0.0;
      for (c, _ in @abc_histo[a][b]) {
        num abc_p = @abc_histo[a][b][c] / abc_sum;
        abc_cumu += abc_p;
        @abc_pmf[a][b][c] = abc_p;
        @abc_cmf[a][b][c] = abc_cumu;
      }
    }
  }

  for (a in @abcd_histo) {
    for (b in @abcd_histo[a]) {
      for (c in @abcd_histo[a][b]) {
        num abcd_sum = 0.0;
        for (d, n in @abcd_histo[a][b][c]) {
          abcd_sum += n;
        }
        num abcd_cumu = 0.0;
        for (d, _ in @abcd_histo[a][b][c]) {
          num abcd_p = @abcd_histo[a][b][c][d] / abcd_sum;
          abcd_cumu += abcd_p;
          @abcd_pmf[a][b][c][d] = abcd_p;
          @abcd_cmf[a][b][c][d] = abcd_cumu;
        }
      }
    }
  }

  for (a in @abcde_histo) {
    for (b in @abcde_histo[a]) {
      for (c in @abcde_histo[a][b]) {
        for (d in @abcde_histo[a][b][c]) {
          num abcde_sum = 0.0;
          for (e, n in @abcde_histo[a][b][c][d]) {
            abcde_sum += n;
          }
          num abcde_cumu = 0.0;
          for (e, _ in @abcde_histo[a][b][c][d]) {
            num abcde_p = @abcde_histo[a][b][c][d][e] / abcde_sum;
            abcde_cumu += abcde_p;
            @abcde_pmf[a][b][c][d][e] = abcde_p;
            @abcde_cmf[a][b][c][d][e] = abcde_cumu;
          }
        }
      }
    }
  }

  #dump;

  for (int oi = 0; oi < @ocount; oi += 1) {

    oa_u = urand();
    oa = "?";
    for (a, c in @a_cmf) {
      oa = a;
      if (oa_u < c) {
        break;
      }
    }
    str out = oa;

    ob_u = urand();
    ob = "?";
    for (b, c in @ab_cmf[oa]) {
      ob = b;
      if (ob_u < c) {
        break;
      }
    }
    out .= ob;

    oc_u = urand();
    oc = "?";
    for (c, d in @abc_cmf[oa][ob]) {
      oc = c;
      if (oc_u < d) {
        break;
      }
    }
    out .= oc;

    od_u = urand();
    od = "?";
    for (d, e in @abcd_cmf[oa][ob][oc]) {
      od = d;
      if (od_u < e) {
        break;
      }
    }
    out .= od;

    for (int i = 4; i < @olen; i += 1) {
      num abcde_u = urand();
      str oe = "???";
      for (e, f in @abcde_cmf[oa][ob][oc][od]) {
        oe = e;
        if (abcde_u < f) {
          break;
        }
      }
      if (oe == "???") {
        break;
      }
      out .= oe;
      oa = ob;
      ob = oc;
      oc = od;
      od = oe;
    }
    print out;

  }
}