Mirrors/miller
1
0
Fork 0
mirror of https://github.com/johnkerl/miller.git synced 2026-07-18 17:04:50 +00:00
Code Issues Projects Releases Wiki Activity

ngrams examples

Browse source
This commit is contained in:
John Kerl 2016-11-29 23:09:54 -05:00
parent 2b3ee228b9
commit d64db1066f
12 changed files with 70 additions and 146 deletions
Download patch file Download diff file Expand all files Collapse all files

1
doc/ngrams/ng1 Executable file
View file

@ -0,0 +1 @@
mlr --nidx --from gsl-2000.txt put -q -f ngfuncs.mlr -f ng1.mlr

45
doc/ngrams/ng1.mlr Executable file
View file

@ -0,0 +1,45 @@
# ================================================================
begin {
if (isabsent(@olen)) {
@olen = 16;
}
if (isabsent(@ocount)) {
@ocount = 16;
}
}
# ================================================================
for (_, v in $*) {
if (string(v) == "inf" || string(v) == "nan") {
continue;
}
int n = strlen(v);
for (int i = 0; i < n; i += 1) {
str a = substr(v, i, i);
@a_histo[a] += 1;
}
}
# ================================================================
end {
#dump;
# Define this in this scope else it'll be scoped to the for-loop.
map a_cmf = {};
a_cmf = compute_cmf_from_histo(@a_histo);
#print "A CMF";
#dump a_cmf;
for (int oi = 0; oi < @ocount; oi += 1) {
str out = "";
for (int i = 0; i < @olen; i += 1) {
str oa = sample_from_cmf(a_cmf);
if (oa == "") {
break;
}
out .= oa;
}
print out;
}
}

1
doc/ngrams/ng2 Executable file
View file

@ -0,0 +1 @@
mlr --nidx --from gsl-2000.txt put -q -f ngfuncs.mlr -f ng2.mlr

11
doc/ngrams/ng2.mlr
View file

@ -14,11 +14,11 @@ for (_, v in $*) {
continue;
}
int n = strlen(v);
if (n >= 2) {
if (n >= 1) {
str a = substr(v, 0, 0);
@a_histo[a] += 1;
for (int i = 0; i < n-3; i += 1) {
for (int i = 0; i < n-1; i += 1) {
str a = substr(v, i, i);
str b = substr(v, i+1, i+1);
@ab_histo[a][b] += 1;
@ -28,11 +28,10 @@ for (_, v in $*) {
# ================================================================
end {
#dump;
map a_cmf = {};
map ab_cmf = {};
# Define these in this scope else they'll be scoped to the for-loops.
map a_cmf = {};
map ab_cmf = {};
a_cmf = compute_cmf_from_histo(@a_histo);

1
doc/ngrams/ng3 Executable file
View file

@ -0,0 +1 @@
mlr --nidx --from gsl-2000.txt put -q -f ngfuncs.mlr -f ng3.mlr

11
doc/ngrams/ng3.mlr
View file

@ -20,7 +20,7 @@ for (_, v in $*) {
@a_histo[a] += 1;
@ab_histo[a][b] += 1;
for (int i = 0; i < n-3; i += 1) {
for (int i = 0; i < n-2; i += 1) {
str a = substr(v, i, i);
str b = substr(v, i+1, i+1);
str c = substr(v, i+2, i+2);
@ -31,12 +31,11 @@ for (_, v in $*) {
# ================================================================
end {
#dump;
map a_cmf = {};
map ab_cmf = {};
map abc_cmf = {}; # xxx weirdness when these aren't predeclared as map ...
# Define these in this scope else they'll be scoped to the for-loops.
map a_cmf = {};
map ab_cmf = {};
map abc_cmf = {};
a_cmf = compute_cmf_from_histo(@a_histo);

1
doc/ngrams/ng4 Executable file
View file

@ -0,0 +1 @@
mlr --nidx --from gsl-2000.txt put -q -f ngfuncs.mlr -f ng4.mlr

11
doc/ngrams/ng4.mlr
View file

@ -34,13 +34,12 @@ for (_, v in $*) {
# ================================================================
end {
#dump;
map a_cmf = {};
map ab_cmf = {};
map abc_cmf = {}; # xxx weirdness when these aren't predeclared as map ...
map abcd_cmf = {};
# Define these in this scope else they'll be scoped to the for-loops.
map a_cmf = {};
map ab_cmf = {};
map abc_cmf = {};
map abcd_cmf = {};
a_cmf = compute_cmf_from_histo(@a_histo);

1
doc/ngrams/ng5 Executable file
View file

@ -0,0 +1 @@
mlr --nidx --from gsl-2000.txt put -q -f ngfuncs.mlr -f ng5.mlr

2
doc/ngrams/ng5.mlr
View file

@ -37,9 +37,7 @@ for (_, v in $*) {
# ================================================================
end {
#dump;
# Define these in this scope else they'll be scoped to the for-loops.
map a_cmf = {};
map ab_cmf = {};

10
doc/ngrams/ngfuncs.mlr
View file

@ -1,6 +1,6 @@
# ================================================================
func compute_sum_from_histo(map histo) {
func compute_sum_from_histo(map histo) : num {
num sum = 0;
for (k, n in histo) {
sum += n;
@ -9,7 +9,7 @@ func compute_sum_from_histo(map histo) {
}
# ----------------------------------------------------------------
func compute_pmf_from_histo(map histo) {
func compute_pmf_from_histo(map histo) : map {
num sum = compute_sum_from_histo(histo);
num cumu = 0.0;
map pmf = {};
@ -21,7 +21,7 @@ func compute_pmf_from_histo(map histo) {
}
# ----------------------------------------------------------------
func compute_cmf_from_pmf(map pmf) {
func compute_cmf_from_pmf(map pmf) : map {
num cumu = 0.0;
map cmf = {};
for (k, p in pmf) {
@ -32,12 +32,12 @@ func compute_cmf_from_pmf(map pmf) {
}
# ----------------------------------------------------------------
func compute_cmf_from_histo(map histo) {
func compute_cmf_from_histo(map histo) : map {
return compute_cmf_from_pmf(compute_pmf_from_histo(histo));
}
# ----------------------------------------------------------------
func sample_from_cmf(var cmf) {
func sample_from_cmf(var cmf) : str {
u = urand();
output = "";
for (k, c in cmf) {

121
doc/ngrams/ngx.mlr
View file

@ -1,121 +0,0 @@
# ================================================================
begin {
if (isabsent(@olen)) {
@olen = 16;
}
if (isabsent(@ocount)) {
@ocount = 16;
}
}
# ================================================================
for (_, v in $*) {
if (string(v) == "inf" || string(v) == "nan") {
continue;
}
int n = strlen(v);
if (n >= 4) {
str a = substr(v, 0, 0);
str b = substr(v, 1, 1);
str c = substr(v, 2, 2);
str d = substr(v, 3, 3);
@a_histo[a] += 1;
@ab_histo[a][b] += 1;
@abc_histo[a][b][c] += 1;
@abcd_histo[a][b][c][d] += 1;
for (int i = 0; i < n-4; i += 1) {
str a = substr(v, i, i);
str b = substr(v, i+1, i+1);
str c = substr(v, i+2, i+2);
str d = substr(v, i+3, i+3);
str e = substr(v, i+4, i+4);
@abcde_histo[a][b][c][d][e] += 1;
}
}
}
# ================================================================
end {
#dump;
#map a_cmf = {};
#map ab_cmf = {};
#map abc_cmf = {}; # xxx weirdness when these aren't predeclared as map ...
#map abcd_cmf = {};
#map abcde_cmf = {};
a_cmf = compute_cmf_from_histo(@a_histo);
print "A CMF";
dump a_cmf;
for (a in @ab_histo) {
#ab_cmf[a] = compute_cmf_from_histo(@ab_histo[a]);
foo = compute_cmf_from_histo(@ab_histo[a]);
print "foo:";
dump foo;
ab_cmf[a] = foo;
print "ab_cmf:";
dump ab_cmf;
}
print "AB CMF";
dump ab_cmf;
# for (a in @abc_histo) {
# for (b in @abc_histo[a]) {
# abc_cmf[a][b] = compute_cmf_from_histo(@abc_histo[a][b]);
# }
# }
#
# print "ABC CMF";
# dump abc_cmf;
#
# for (a in @abcd_histo) {
# for (b in @abcd_histo[a]) {
# for (c in @abcd_histo[a][b]) {
# abcd_cmf[a][b][c] = compute_cmf_from_histo(@abcd_histo[a][b][c]);
# }
# }
# }
#
# print "ABCD CMF";
# dump abcd_cmf;
#
# for (a in @abcde_histo) {
# for (b in @abcde_histo[a]) {
# for (c in @abcde_histo[a][b]) {
# for (d in @abcde_histo[a][b][c]) {
# abcde_cmf[a][b][c][d] = compute_cmf_from_histo(@abcde_histo[a][b][c][d]);
# }
# }
# }
# }
#
# print "ABCDE CMF";
# dump abcde_cmf;
#
# for (int oi = 0; oi < @ocount; oi += 1) {
# str oa = sample_from_cmf(a_cmf);
# str ob = sample_from_cmf(ab_cmf[oa]);
# str oc = sample_from_cmf(abc_cmf[oa][ob]);
# str od = sample_from_cmf(abcd_cmf[oa][ob][oc]);
# str out = oa . ob . oc . od;
#
# for (int i = 4; i < @olen; i += 1) {
# str oe = sample_from_cmf(abcde_cmf[oa][ob][oc][od]);
# if (oe == "") {
# break;
# }
# out .= oe;
# oa = ob;
# ob = oc;
# oc = od;
# od = oe;
# }
# print out;
# }
}