From d857653b1a75cd300cfebc8d7908f40502efb4ee Mon Sep 17 00:00:00 2001
From: John Kerl <kerl.john.r@gmail.com>
Date: Fri, 21 Oct 2016 20:04:34 -0400
Subject: [PATCH] initial commit of n-grams examples

---
 c/todo.txt          |   3 +-
 doc/ngrams/mkv.mlr  |  80 ++++++++++++++++++++
 doc/ngrams/mkv3.mlr | 122 ++++++++++++++++++++++++++++++
 doc/ngrams/mkv4.mlr | 152 +++++++++++++++++++++++++++++++++++++
 doc/ngrams/mkv5.mlr | 180 ++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 536 insertions(+), 1 deletion(-)
 create mode 100755 doc/ngrams/mkv.mlr
 create mode 100755 doc/ngrams/mkv3.mlr
 create mode 100755 doc/ngrams/mkv4.mlr
 create mode 100755 doc/ngrams/mkv5.mlr

diff --git a/c/todo.txt b/c/todo.txt
index db55a4e40..c87a9c530 100644
--- a/c/todo.txt
+++ b/c/todo.txt
@@ -38,6 +38,8 @@ FUNCTIONALITY FOR 4.6.0:
 
   ? for-oosvar map-valued boundvars?!?
 
+  ? for-local -- with keylist etc.?
+
 ? runtime type assertion ? either an 'int x' ... needing propagation ...
   or maybe more simply an assert_int(x), assert_numeric, assert_present(x), etc.
 
@@ -66,7 +68,6 @@ COOK:
 PERF:
 * mand bench somewhere; other benches.
 * mkv2/3/4 somewhere; also cookbook. (better w/ mappable vars ... maybe for 4.7.0)
-* also note mkvgen somewhere.
 * xsv notes; update rust/go experiment info
 
 -f/-e:
diff --git a/doc/ngrams/mkv.mlr b/doc/ngrams/mkv.mlr
new file mode 100755
index 000000000..6d1c413d8
--- /dev/null
+++ b/doc/ngrams/mkv.mlr
@@ -0,0 +1,80 @@
+begin {
+  if (isabsent(@olen)) {
+    @olen = 16;
+  }
+  if (isabsent(@ocount)) {
+    @ocount = 16;
+  }
+}
+
+for (_, v in $*) {
+  local n = strlen(v);
+  if (n >= 1) {
+    local a = substr(v, 0, 0);
+    @a_histo[a] += 1;
+    for (local i = 0; i < n-1; i += 1) {
+      local a = substr(v, i, i);
+      local b = substr(v, i+1, i+1);
+      @ab_histo[a][b] += 1;
+    }
+  }
+}
+
+end {
+  local a_sum = 0;
+  for (a, na in @a_histo) {
+    a_sum += na;
+  }
+  local a_cumu = 0.0;
+  for (a, na in @a_histo) {
+    local a_p = @a_histo[a] / a_sum;
+    a_cumu += a_p;
+    @a_pmf[a] = a_p;
+    @a_cmf[a] += a_cumu;
+  }
+
+  for (a in @ab_histo) {
+    local ab_sum = 0.0;
+    for (b, nab in @ab_histo[a]) {
+      ab_sum += nab;
+    }
+    local ab_cumu = 0.0;
+    for (b, _ in @ab_histo[a]) {
+      local ab_p = @ab_histo[a][b] / ab_sum;
+      ab_cumu += ab_p;
+      @ab_pmf[a][b] = ab_p;
+      @ab_cmf[a][b] = ab_cumu;
+    }
+  }
+
+  #dump;
+
+  for (local oi = 0; oi < @ocount; oi += 1) {
+    oa_u = urand();
+    oa = "?";
+    for (a, c in @a_cmf) {
+      if (oa_u < c) {
+        oa = a;
+        break;
+      }
+    }
+    local out = oa;
+    for (local i = 1; i < @olen; i += 1) {
+      local ab_u = urand();
+      ob = "???";
+      for (b, c in @ab_cmf[oa]) {
+        #print "BC ".b.c;
+        if (ab_u < c) {
+          ob = b;
+          break;
+        }
+      }
+      if (ob == "???") {
+        break;
+      }
+      out .= ob;
+      oa = ob;
+    }
+    print out;
+  }
+}
diff --git a/doc/ngrams/mkv3.mlr b/doc/ngrams/mkv3.mlr
new file mode 100755
index 000000000..50fbe2abf
--- /dev/null
+++ b/doc/ngrams/mkv3.mlr
@@ -0,0 +1,122 @@
+begin {
+  if (isabsent(@olen)) {
+    @olen = 16;
+  }
+  if (isabsent(@ocount)) {
+    @ocount = 16;
+  }
+}
+
+for (_, v in $*) {
+  local n = strlen(v);
+  if (n >= 2) {
+    local a = substr(v, 0, 0);
+    @a_histo[a] += 1;
+
+    local b = substr(v, 1, 1);
+    @ab_histo[a][b] += 1;
+
+    for (local i = 0; i < n-2; i += 1) {
+      local a = substr(v, i, i);
+      local b = substr(v, i+1, i+1);
+      local c = substr(v, i+2, i+2);
+      @abc_histo[a][b][c] += 1;
+    }
+  }
+}
+
+end {
+
+  local a_sum = 0;
+  for (a, na in @a_histo) {
+    a_sum += na;
+  }
+  local a_cumu = 0.0;
+  for (a, _ in @a_histo) {
+    local a_p = @a_histo[a] / a_sum;
+    a_cumu += a_p;
+    @a_pmf[a] = a_p;
+    @a_cmf[a] += a_cumu;
+  }
+
+  for (a in @ab_histo) {
+
+    local ab_sum = 0.0;
+    for (b, nab in @ab_histo[a]) {
+      ab_sum += nab;
+    }
+    local ab_cumu = 0.0;
+    for (b, _ in @ab_histo[a]) {
+      local ab_p = @ab_histo[a][b] / ab_sum;
+      ab_cumu += ab_p;
+      @ab_pmf[a][b] = ab_p;
+      @ab_cmf[a][b] = ab_cumu;
+    }
+  }
+
+  for (a in @abc_histo) {
+    for (b in @abc_histo[a]) {
+
+      local abc_sum = 0.0;
+      for (c, nabc in @abc_histo[a][b]) {
+        abc_sum += nabc;
+      }
+      local abc_cumu = 0.0;
+      for (c, _ in @abc_histo[a][b]) {
+        local abc_p = @abc_histo[a][b][c] / abc_sum;
+        abc_cumu += abc_p;
+        @abc_pmf[a][b][c] = abc_p;
+        @abc_cmf[a][b][c] = abc_cumu;
+      }
+    }
+  }
+
+  #dump;
+
+  for (local oi = 0; oi < @ocount; oi += 1) {
+    oa_u = urand();
+    oa = "?";
+    for (a, c in @a_cmf) {
+      oa = a;
+      if (oa_u < c) {
+        break;
+      }
+    }
+    local out = oa;
+
+    ob_u = urand();
+    ob = "?";
+    for (b, c in @ab_cmf[oa]) {
+      ob = b;
+      if (ob_u < c) {
+        break;
+      }
+    }
+    out .= ob;
+
+    for (local i = 2; i < @olen; i += 1) {
+      local abc_u = urand();
+      local oc = "???";
+      #print;
+      #print "i=".i;
+      #print "oa = ".oa." ob = ".ob;
+      for (c, d in @abc_cmf[oa][ob]) {
+        #print "d = ".d;
+        oc = c;
+        if (abc_u < d) {
+          break;
+        }
+      }
+      if (oc == "???") {
+        break;
+      }
+      #print "i=".i.",oa=".oa.",ob=".ob.",oc=".oc;
+      #print "oc = ".oc;
+      out .= oc;
+      oa = ob;
+      ob = oc;
+    }
+    print out;
+
+  }
+}
diff --git a/doc/ngrams/mkv4.mlr b/doc/ngrams/mkv4.mlr
new file mode 100755
index 000000000..499b84400
--- /dev/null
+++ b/doc/ngrams/mkv4.mlr
@@ -0,0 +1,152 @@
+begin {
+  if (isabsent(@olen)) {
+    @olen = 16;
+  }
+  if (isabsent(@ocount)) {
+    @ocount = 16;
+  }
+}
+
+for (_, v in $*) {
+  local n = strlen(v);
+  if (n >= 3) {
+    local a = substr(v, 0, 0);
+    local b = substr(v, 1, 1);
+    local c = substr(v, 2, 2);
+    @a_histo[a] += 1;
+    @ab_histo[a][b] += 1;
+    @abc_histo[a][b][c] += 1;
+
+    for (local i = 0; i < n-3; i += 1) {
+      local a = substr(v, i, i);
+      local b = substr(v, i+1, i+1);
+      local c = substr(v, i+2, i+2);
+      local d = substr(v, i+3, i+3);
+      @abcd_histo[a][b][c][d] += 1;
+    }
+  }
+}
+
+end {
+
+  local a_sum = 0;
+  for (a, na in @a_histo) {
+    a_sum += na;
+  }
+  local a_cumu = 0.0;
+  for (a, _ in @a_histo) {
+    local a_p = @a_histo[a] / a_sum;
+    a_cumu += a_p;
+    @a_pmf[a] = a_p;
+    @a_cmf[a] += a_cumu;
+  }
+
+  for (a in @ab_histo) {
+    local ab_sum = 0.0;
+    for (b, nab in @ab_histo[a]) {
+      ab_sum += nab;
+    }
+    local ab_cumu = 0.0;
+    for (b, _ in @ab_histo[a]) {
+      local ab_p = @ab_histo[a][b] / ab_sum;
+      ab_cumu += ab_p;
+      @ab_pmf[a][b] = ab_p;
+      @ab_cmf[a][b] = ab_cumu;
+    }
+  }
+
+  for (a in @abc_histo) {
+    for (b in @abc_histo[a]) {
+      local abc_sum = 0.0;
+      for (c, nabc in @abc_histo[a][b]) {
+        abc_sum += nabc;
+      }
+      local abc_cumu = 0.0;
+      for (c, _ in @abc_histo[a][b]) {
+        local abc_p = @abc_histo[a][b][c] / abc_sum;
+        abc_cumu += abc_p;
+        @abc_pmf[a][b][c] = abc_p;
+        @abc_cmf[a][b][c] = abc_cumu;
+      }
+    }
+  }
+
+  for (a in @abcd_histo) {
+    for (b in @abcd_histo[a]) {
+      for (c in @abcd_histo[a][b]) {
+        local abcd_sum = 0.0;
+        for (d, n in @abcd_histo[a][b][c]) {
+          abcd_sum += n;
+        }
+        local abcd_cumu = 0.0;
+        for (d, _ in @abcd_histo[a][b][c]) {
+          local abcd_p = @abcd_histo[a][b][c][d] / abcd_sum;
+          abcd_cumu += abcd_p;
+          @abcd_pmf[a][b][c][d] = abcd_p;
+          @abcd_cmf[a][b][c][d] = abcd_cumu;
+        }
+      }
+    }
+  }
+
+  #dump;
+
+  for (local oi = 0; oi < @ocount; oi += 1) {
+    oa_u = urand();
+    oa = "?";
+    for (a, c in @a_cmf) {
+      oa = a;
+      if (oa_u < c) {
+        break;
+      }
+    }
+    local out = oa;
+
+    ob_u = urand();
+    ob = "?";
+    for (b, c in @ab_cmf[oa]) {
+      ob = b;
+      if (ob_u < c) {
+        break;
+      }
+    }
+    out .= ob;
+
+    oc_u = urand();
+    oc = "?";
+    for (c, d in @abc_cmf[oa][ob]) {
+      oc = c;
+      if (oc_u < d) {
+        break;
+      }
+    }
+    out .= oc;
+
+    for (local i = 3; i < @olen; i += 1) {
+      local abcd_u = urand();
+      local od = "???";
+      #print;
+      #print "i=".i;
+      #print "oa = ".oa." ob = ".ob;
+      for (d, e in @abcd_cmf[oa][ob][oc]) {
+        od = d;
+        if (abcd_u < e) {
+          break;
+        }
+      }
+      if (od == "???") {
+        break;
+      }
+      #print "i=".i.",oa=".oa.",ob=".ob.",oc=".oc;
+      #print "oc = ".oc;
+      out .= od;
+      oa = ob;
+      ob = oc;
+      oc = od;
+    }
+    #print "U IS ".oa_u;
+    #print "A IS ".oa;
+    print out;
+
+  }
+}
diff --git a/doc/ngrams/mkv5.mlr b/doc/ngrams/mkv5.mlr
new file mode 100755
index 000000000..65d68b7b6
--- /dev/null
+++ b/doc/ngrams/mkv5.mlr
@@ -0,0 +1,180 @@
+begin {
+  if (isabsent(@olen)) {
+    @olen = 16;
+  }
+  if (isabsent(@ocount)) {
+    @ocount = 16;
+  }
+}
+
+for (_, v in $*) {
+  local n = strlen(v);
+  if (n >= 4) {
+    local a = substr(v, 0, 0);
+    local b = substr(v, 1, 1);
+    local c = substr(v, 2, 2);
+    local d = substr(v, 3, 3);
+    @a_histo[a] += 1;
+    @ab_histo[a][b] += 1;
+    @abc_histo[a][b][c] += 1;
+    @abcd_histo[a][b][c][d] += 1;
+
+    for (local i = 0; i < n-4; i += 1) {
+      local a = substr(v, i, i);
+      local b = substr(v, i+1, i+1);
+      local c = substr(v, i+2, i+2);
+      local d = substr(v, i+3, i+3);
+      local e = substr(v, i+4, i+4);
+      @abcde_histo[a][b][c][d][e] += 1;
+    }
+  }
+}
+
+end {
+
+  local a_sum = 0;
+  for (a, na in @a_histo) {
+    a_sum += na;
+  }
+  local a_cumu = 0.0;
+  for (a, _ in @a_histo) {
+    local a_p = @a_histo[a] / a_sum;
+    a_cumu += a_p;
+    @a_pmf[a] = a_p;
+    @a_cmf[a] += a_cumu;
+  }
+
+  for (a in @ab_histo) {
+    local ab_sum = 0.0;
+    for (b, nab in @ab_histo[a]) {
+      ab_sum += nab;
+    }
+    local ab_cumu = 0.0;
+    for (b, _ in @ab_histo[a]) {
+      local ab_p = @ab_histo[a][b] / ab_sum;
+      ab_cumu += ab_p;
+      @ab_pmf[a][b] = ab_p;
+      @ab_cmf[a][b] = ab_cumu;
+    }
+  }
+
+  for (a in @abc_histo) {
+    for (b in @abc_histo[a]) {
+      local abc_sum = 0.0;
+      for (c, nabc in @abc_histo[a][b]) {
+        abc_sum += nabc;
+      }
+      local abc_cumu = 0.0;
+      for (c, _ in @abc_histo[a][b]) {
+        local abc_p = @abc_histo[a][b][c] / abc_sum;
+        abc_cumu += abc_p;
+        @abc_pmf[a][b][c] = abc_p;
+        @abc_cmf[a][b][c] = abc_cumu;
+      }
+    }
+  }
+
+  for (a in @abcd_histo) {
+    for (b in @abcd_histo[a]) {
+      for (c in @abcd_histo[a][b]) {
+        local abcd_sum = 0.0;
+        for (d, n in @abcd_histo[a][b][c]) {
+          abcd_sum += n;
+        }
+        local abcd_cumu = 0.0;
+        for (d, _ in @abcd_histo[a][b][c]) {
+          local abcd_p = @abcd_histo[a][b][c][d] / abcd_sum;
+          abcd_cumu += abcd_p;
+          @abcd_pmf[a][b][c][d] = abcd_p;
+          @abcd_cmf[a][b][c][d] = abcd_cumu;
+        }
+      }
+    }
+  }
+
+  for (a in @abcde_histo) {
+    for (b in @abcde_histo[a]) {
+      for (c in @abcde_histo[a][b]) {
+        for (d in @abcde_histo[a][b][c]) {
+          local abcde_sum = 0.0;
+          for (e, n in @abcde_histo[a][b][c][d]) {
+            abcde_sum += n;
+          }
+          local abcde_cumu = 0.0;
+          for (e, _ in @abcde_histo[a][b][c][d]) {
+            local abcde_p = @abcde_histo[a][b][c][d][e] / abcde_sum;
+            abcde_cumu += abcde_p;
+            @abcde_pmf[a][b][c][d][e] = abcde_p;
+            @abcde_cmf[a][b][c][d][e] = abcde_cumu;
+          }
+        }
+      }
+    }
+  }
+
+  #dump;
+
+  for (local oi = 0; oi < @ocount; oi += 1) {
+
+    oa_u = urand();
+    oa = "?";
+    for (a, c in @a_cmf) {
+      oa = a;
+      if (oa_u < c) {
+        break;
+      }
+    }
+    local out = oa;
+
+    ob_u = urand();
+    ob = "?";
+    for (b, c in @ab_cmf[oa]) {
+      ob = b;
+      if (ob_u < c) {
+        break;
+      }
+    }
+    out .= ob;
+
+    oc_u = urand();
+    oc = "?";
+    for (c, d in @abc_cmf[oa][ob]) {
+      oc = c;
+      if (oc_u < d) {
+        break;
+      }
+    }
+    out .= oc;
+
+    od_u = urand();
+    od = "?";
+    for (d, e in @abcd_cmf[oa][ob][oc]) {
+      od = d;
+      if (od_u < e) {
+        break;
+      }
+    }
+    out .= od;
+
+    for (local i = 4; i < @olen; i += 1) {
+      local abcde_u = urand();
+      local oe = "???";
+      for (e, f in @abcde_cmf[oa][ob][oc][od]) {
+        oe = e;
+        if (abcde_u < f) {
+          break;
+        }
+      }
+      if (oe == "???") {
+        break;
+      }
+      out .= oe;
+      oa = ob;
+      ob = oc;
+      oc = od;
+      od = oe;
+    }
+    print out;
+
+  }
+}