176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
|
setLabelsOnToken (oToken) {
if (!this.lexicographer) {
return;
}
if (!oToken.hasOwnProperty("lMorph")) {
oToken["lMorph"] = this.getMorph(oToken["sValue"]);
}
if (oToken["sType"] == "WORD") {
oToken["bValidToken"] = this.isValidToken(oToken["sValue"]);
let [sPrefix, sStem, sSuffix] = this.lexicographer.split(oToken["sValue"]);
if (sStem != oToken["sValue"]) {
oToken["lSubTokens"] = [
{ "sType": "WORD", "sValue": sPrefix, "lMorph": this.getMorph(sPrefix) },
{ "sType": "WORD", "sValue": sStem, "lMorph": this.getMorph(sStem) },
{ "sType": "WORD", "sValue": sSuffix, "lMorph": this.getMorph(sSuffix) }
|
|
|
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
|
setLabelsOnToken (oToken) {
if (!this.lexicographer) {
return;
}
if (!oToken.hasOwnProperty("lMorph")) {
oToken["lMorph"] = this.getMorph(oToken["sValue"]);
}
if (oToken["sType"].startsWith("WORD")) {
oToken["bValidToken"] = this.isValidToken(oToken["sValue"]);
let [sPrefix, sStem, sSuffix] = this.lexicographer.split(oToken["sValue"]);
if (sStem != oToken["sValue"]) {
oToken["lSubTokens"] = [
{ "sType": "WORD", "sValue": sPrefix, "lMorph": this.getMorph(sPrefix) },
{ "sType": "WORD", "sValue": sStem, "lMorph": this.getMorph(sStem) },
{ "sType": "WORD", "sValue": sSuffix, "lMorph": this.getMorph(sSuffix) }
|