57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
|
for (let [sFlex, sStem, sTag] of lEntrySrc) {
for (let s2grams of str_transform.getNgrams(sFlex)) {
this.a2grams.add(s2grams);
}
addWordToCharDict(sFlex);
// chars
for (let c of sFlex) {
if (!dChar.get(c)) {
dChar.set(c, nChar);
lChar.push(c);
nChar += 1;
}
dCharOccur.set(c, dCharOccur.gl_get(c, 0) + 1);
}
// affixes to find stem from flexion
let sAff = funcStemmingGen(sFlex, sStem);
if (!dAff.get(sAff)) {
dAff.set(sAff, nAff);
lAff.push(sAff);
nAff += 1;
}
dAffOccur.set(sAff, dAffOccur.gl_get(sAff, 0) + 1);
// tags
if (!dTag.get(sTag)) {
dTag.set(sTag, nTag);
lTag.push(sTag);
nTag += 1;
}
dTagOccur.set(sTag, dTagOccur.gl_get(sTag, 0) + 1);
lEntry.push([sFlex, dAff.get(sAff), dTag.get(sTag)]);
}
|
|
|
|
|
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
|
for (let [sFlex, sStem, sTag] of lEntrySrc) {
for (let s2grams of str_transform.getNgrams(sFlex)) {
this.a2grams.add(s2grams);
}
addWordToCharDict(sFlex);
// chars
for (let c of sFlex) {
if (!dChar.has(c)) {
dChar.set(c, nChar);
lChar.push(c);
nChar += 1;
}
dCharOccur.set(c, dCharOccur.gl_get(c, 0) + 1);
}
// affixes to find stem from flexion
let sAff = funcStemmingGen(sFlex, sStem);
if (!dAff.has(sAff)) {
dAff.set(sAff, nAff);
lAff.push(sAff);
nAff += 1;
}
dAffOccur.set(sAff, dAffOccur.gl_get(sAff, 0) + 1);
// tags
if (!dTag.has(sTag)) {
dTag.set(sTag, nTag);
lTag.push(sTag);
nTag += 1;
}
dTagOccur.set(sTag, dTagOccur.gl_get(sTag, 0) + 1);
lEntry.push([sFlex, dAff.get(sAff), dTag.get(sTag)]);
}
|