diff --git a/doc/ngrams/ng1 b/doc/ngrams/ng1 new file mode 100755 index 000000000..bc1271b26 --- /dev/null +++ b/doc/ngrams/ng1 @@ -0,0 +1 @@ +mlr --nidx --from gsl-2000.txt put -q -f ngfuncs.mlr -f ng1.mlr diff --git a/doc/ngrams/ng1.mlr b/doc/ngrams/ng1.mlr new file mode 100755 index 000000000..d2f0c1249 --- /dev/null +++ b/doc/ngrams/ng1.mlr @@ -0,0 +1,45 @@ +# ================================================================ +begin { + if (isabsent(@olen)) { + @olen = 16; + } + if (isabsent(@ocount)) { + @ocount = 16; + } +} + +# ================================================================ +for (_, v in $*) { + if (string(v) == "inf" || string(v) == "nan") { + continue; + } + int n = strlen(v); + for (int i = 0; i < n; i += 1) { + str a = substr(v, i, i); + @a_histo[a] += 1; + } +} + +# ================================================================ +end { + #dump; + # Define this in this scope else it'll be scoped to the for-loop. + map a_cmf = {}; + + a_cmf = compute_cmf_from_histo(@a_histo); + + #print "A CMF"; + #dump a_cmf; + + for (int oi = 0; oi < @ocount; oi += 1) { + str out = ""; + for (int i = 0; i < @olen; i += 1) { + str oa = sample_from_cmf(a_cmf); + if (oa == "") { + break; + } + out .= oa; + } + print out; + } +} diff --git a/doc/ngrams/ng2 b/doc/ngrams/ng2 new file mode 100755 index 000000000..276226af4 --- /dev/null +++ b/doc/ngrams/ng2 @@ -0,0 +1 @@ +mlr --nidx --from gsl-2000.txt put -q -f ngfuncs.mlr -f ng2.mlr diff --git a/doc/ngrams/ng2.mlr b/doc/ngrams/ng2.mlr index f889ada85..1bf87ca1a 100755 --- a/doc/ngrams/ng2.mlr +++ b/doc/ngrams/ng2.mlr @@ -14,11 +14,11 @@ for (_, v in $*) { continue; } int n = strlen(v); - if (n >= 2) { + if (n >= 1) { str a = substr(v, 0, 0); @a_histo[a] += 1; - for (int i = 0; i < n-3; i += 1) { + for (int i = 0; i < n-1; i += 1) { str a = substr(v, i, i); str b = substr(v, i+1, i+1); @ab_histo[a][b] += 1; @@ -28,11 +28,10 @@ for (_, v in $*) { # ================================================================ end { - #dump; - - map a_cmf = {}; - map ab_cmf = {}; + # Define these in this scope else they'll be scoped to the for-loops. + map a_cmf = {}; + map ab_cmf = {}; a_cmf = compute_cmf_from_histo(@a_histo); diff --git a/doc/ngrams/ng3 b/doc/ngrams/ng3 new file mode 100755 index 000000000..a40947f1d --- /dev/null +++ b/doc/ngrams/ng3 @@ -0,0 +1 @@ +mlr --nidx --from gsl-2000.txt put -q -f ngfuncs.mlr -f ng3.mlr diff --git a/doc/ngrams/ng3.mlr b/doc/ngrams/ng3.mlr index 6fdcd24a2..8c9687fba 100755 --- a/doc/ngrams/ng3.mlr +++ b/doc/ngrams/ng3.mlr @@ -20,7 +20,7 @@ for (_, v in $*) { @a_histo[a] += 1; @ab_histo[a][b] += 1; - for (int i = 0; i < n-3; i += 1) { + for (int i = 0; i < n-2; i += 1) { str a = substr(v, i, i); str b = substr(v, i+1, i+1); str c = substr(v, i+2, i+2); @@ -31,12 +31,11 @@ for (_, v in $*) { # ================================================================ end { - #dump; - - map a_cmf = {}; - map ab_cmf = {}; - map abc_cmf = {}; # xxx weirdness when these aren't predeclared as map ... + # Define these in this scope else they'll be scoped to the for-loops. + map a_cmf = {}; + map ab_cmf = {}; + map abc_cmf = {}; a_cmf = compute_cmf_from_histo(@a_histo); diff --git a/doc/ngrams/ng4 b/doc/ngrams/ng4 new file mode 100755 index 000000000..10bf93f13 --- /dev/null +++ b/doc/ngrams/ng4 @@ -0,0 +1 @@ +mlr --nidx --from gsl-2000.txt put -q -f ngfuncs.mlr -f ng4.mlr diff --git a/doc/ngrams/ng4.mlr b/doc/ngrams/ng4.mlr index 8fe4bac24..f7d6c113d 100755 --- a/doc/ngrams/ng4.mlr +++ b/doc/ngrams/ng4.mlr @@ -34,13 +34,12 @@ for (_, v in $*) { # ================================================================ end { - #dump; - - map a_cmf = {}; - map ab_cmf = {}; - map abc_cmf = {}; # xxx weirdness when these aren't predeclared as map ... - map abcd_cmf = {}; + # Define these in this scope else they'll be scoped to the for-loops. + map a_cmf = {}; + map ab_cmf = {}; + map abc_cmf = {}; + map abcd_cmf = {}; a_cmf = compute_cmf_from_histo(@a_histo); diff --git a/doc/ngrams/ng5 b/doc/ngrams/ng5 new file mode 100755 index 000000000..fb27a084b --- /dev/null +++ b/doc/ngrams/ng5 @@ -0,0 +1 @@ +mlr --nidx --from gsl-2000.txt put -q -f ngfuncs.mlr -f ng5.mlr diff --git a/doc/ngrams/ng5.mlr b/doc/ngrams/ng5.mlr index d4053ec34..8a1c670cc 100755 --- a/doc/ngrams/ng5.mlr +++ b/doc/ngrams/ng5.mlr @@ -37,9 +37,7 @@ for (_, v in $*) { # ================================================================ end { - #dump; - # Define these in this scope else they'll be scoped to the for-loops. map a_cmf = {}; map ab_cmf = {}; diff --git a/doc/ngrams/ngfuncs.mlr b/doc/ngrams/ngfuncs.mlr index 0d6c729d2..7f93665b6 100644 --- a/doc/ngrams/ngfuncs.mlr +++ b/doc/ngrams/ngfuncs.mlr @@ -1,6 +1,6 @@ # ================================================================ -func compute_sum_from_histo(map histo) { +func compute_sum_from_histo(map histo) : num { num sum = 0; for (k, n in histo) { sum += n; @@ -9,7 +9,7 @@ func compute_sum_from_histo(map histo) { } # ---------------------------------------------------------------- -func compute_pmf_from_histo(map histo) { +func compute_pmf_from_histo(map histo) : map { num sum = compute_sum_from_histo(histo); num cumu = 0.0; map pmf = {}; @@ -21,7 +21,7 @@ func compute_pmf_from_histo(map histo) { } # ---------------------------------------------------------------- -func compute_cmf_from_pmf(map pmf) { +func compute_cmf_from_pmf(map pmf) : map { num cumu = 0.0; map cmf = {}; for (k, p in pmf) { @@ -32,12 +32,12 @@ func compute_cmf_from_pmf(map pmf) { } # ---------------------------------------------------------------- -func compute_cmf_from_histo(map histo) { +func compute_cmf_from_histo(map histo) : map { return compute_cmf_from_pmf(compute_pmf_from_histo(histo)); } # ---------------------------------------------------------------- -func sample_from_cmf(var cmf) { +func sample_from_cmf(var cmf) : str { u = urand(); output = ""; for (k, c in cmf) { diff --git a/doc/ngrams/ngx.mlr b/doc/ngrams/ngx.mlr deleted file mode 100644 index 2a15f6ec7..000000000 --- a/doc/ngrams/ngx.mlr +++ /dev/null @@ -1,121 +0,0 @@ -# ================================================================ -begin { - if (isabsent(@olen)) { - @olen = 16; - } - if (isabsent(@ocount)) { - @ocount = 16; - } -} - -# ================================================================ -for (_, v in $*) { - if (string(v) == "inf" || string(v) == "nan") { - continue; - } - int n = strlen(v); - if (n >= 4) { - str a = substr(v, 0, 0); - str b = substr(v, 1, 1); - str c = substr(v, 2, 2); - str d = substr(v, 3, 3); - @a_histo[a] += 1; - @ab_histo[a][b] += 1; - @abc_histo[a][b][c] += 1; - @abcd_histo[a][b][c][d] += 1; - - for (int i = 0; i < n-4; i += 1) { - str a = substr(v, i, i); - str b = substr(v, i+1, i+1); - str c = substr(v, i+2, i+2); - str d = substr(v, i+3, i+3); - str e = substr(v, i+4, i+4); - @abcde_histo[a][b][c][d][e] += 1; - } - } -} - -# ================================================================ -end { - - #dump; - - #map a_cmf = {}; - #map ab_cmf = {}; - #map abc_cmf = {}; # xxx weirdness when these aren't predeclared as map ... - #map abcd_cmf = {}; - #map abcde_cmf = {}; - - a_cmf = compute_cmf_from_histo(@a_histo); - - print "A CMF"; - dump a_cmf; - - for (a in @ab_histo) { - #ab_cmf[a] = compute_cmf_from_histo(@ab_histo[a]); - foo = compute_cmf_from_histo(@ab_histo[a]); - print "foo:"; - dump foo; - ab_cmf[a] = foo; - print "ab_cmf:"; - dump ab_cmf; - } - - print "AB CMF"; - dump ab_cmf; - -# for (a in @abc_histo) { -# for (b in @abc_histo[a]) { -# abc_cmf[a][b] = compute_cmf_from_histo(@abc_histo[a][b]); -# } -# } -# -# print "ABC CMF"; -# dump abc_cmf; -# -# for (a in @abcd_histo) { -# for (b in @abcd_histo[a]) { -# for (c in @abcd_histo[a][b]) { -# abcd_cmf[a][b][c] = compute_cmf_from_histo(@abcd_histo[a][b][c]); -# } -# } -# } -# -# print "ABCD CMF"; -# dump abcd_cmf; -# -# for (a in @abcde_histo) { -# for (b in @abcde_histo[a]) { -# for (c in @abcde_histo[a][b]) { -# for (d in @abcde_histo[a][b][c]) { -# abcde_cmf[a][b][c][d] = compute_cmf_from_histo(@abcde_histo[a][b][c][d]); -# } -# } -# } -# } -# -# print "ABCDE CMF"; -# dump abcde_cmf; -# -# for (int oi = 0; oi < @ocount; oi += 1) { -# str oa = sample_from_cmf(a_cmf); -# str ob = sample_from_cmf(ab_cmf[oa]); -# str oc = sample_from_cmf(abc_cmf[oa][ob]); -# str od = sample_from_cmf(abcd_cmf[oa][ob][oc]); -# str out = oa . ob . oc . od; -# -# for (int i = 4; i < @olen; i += 1) { -# str oe = sample_from_cmf(abcde_cmf[oa][ob][oc][od]); -# if (oe == "") { -# break; -# } -# out .= oe; -# oa = ob; -# ob = oc; -# oc = od; -# od = oe; -# } -# print out; -# } - -}