98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
|
Object.assign(this, oData);
}
catch (e) {
throw Error("# Error. File not found or not loadable.\n" + e.message + "\n");
}
/*
Properties:
sName, nVersion, sHeader, lArcVal, nArcVal, byDic, sLang, nChar, nBytesArc, nBytesNodeAddress,
nEntries, nNode, nArc, nAff, cStemming, nTag, dChar, nBytesOffset,
*/
/*
Bug workaround.
Mozilla’s JS parser sucks. Can’t read file bigger than 4 Mb!
So we convert huge hexadecimal string to list of numbers…
https://github.com/mozilla/addons-linter/issues/1361
*/
let lTemp = [];
for (let i = 0; i < this.byDic.length; i+=2) {
lTemp.push(parseInt(this.byDic.slice(i, i+2), 16));
}
this.byDic = lTemp;
/* end of bug workaround */
if (!this.sHeader.startsWith("/pyfsa/")) {
throw TypeError("# Error. Not a pyfsa binary dictionary. Header: " + this.sHeader);
}
if (!(this.nVersion == "1" || this.nVersion == "2" || this.nVersion == "3")) {
throw RangeError("# Error. Unknown dictionary version: " + this.nVersion);
}
// <dChar> to get the value of an arc, <dCharVal> to get the char of an arc with its value
this.dChar = helpers.objectToMap(this.dChar);
this.dCharVal = this.dChar.gl_reverse();
//this.byDic = new Uint8Array(this.byDic); // not quicker, even slower
if (this.cStemming == "S") {
this.funcStemming = str_transform.changeWordWithSuffixCode;
} else if (this.cStemming == "A") {
this.funcStemming = str_transform.changeWordWithAffixCode;
} else {
this.funcStemming = str_transform.noStemming;
}
this._arcMask = (2 ** ((this.nBytesArc * 8) - 3)) - 1;
this._finalNodeMask = 1 << ((this.nBytesArc * 8) - 1);
this._lastArcMask = 1 << ((this.nBytesArc * 8) - 2);
// Configuring DAWG functions according to nVersion
switch (this.nVersion) {
case 1:
this.morph = this._morph1;
this.stem = this._stem1;
this._lookupArcNode = this._lookupArcNode1;
this._getArcs = this._getArcs1;
this._writeNodes = this._writeNodes1;
break;
|
|
|
|
|
|
|
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
|
Object.assign(this, oData);
}
catch (e) {
throw Error("# Error. File not found or not loadable.\n" + e.message + "\n");
}
/*
Properties:
sName, nCompressionMethod, sHeader, lArcVal, nArcVal, byDic, sLang, nChar, nBytesArc, nBytesNodeAddress,
nEntries, nNode, nArc, nAff, cStemming, nTag, dChar, nBytesOffset,
*/
/*
Bug workaround.
Mozilla’s JS parser sucks. Can’t read file bigger than 4 Mb!
So we convert huge hexadecimal string to list of numbers…
https://github.com/mozilla/addons-linter/issues/1361
*/
let lTemp = [];
for (let i = 0; i < this.byDic.length; i+=2) {
lTemp.push(parseInt(this.byDic.slice(i, i+2), 16));
}
this.byDic = lTemp;
/* end of bug workaround */
if (!this.sHeader.startsWith("/pyfsa/")) {
throw TypeError("# Error. Not a pyfsa binary dictionary. Header: " + this.sHeader);
}
if (!(this.nCompressionMethod == "1" || this.nCompressionMethod == "2" || this.nCompressionMethod == "3")) {
throw RangeError("# Error. Unknown dictionary compression method: " + this.nCompressionMethod);
}
// <dChar> to get the value of an arc, <dCharVal> to get the char of an arc with its value
this.dChar = helpers.objectToMap(this.dChar);
this.dCharVal = this.dChar.gl_reverse();
//this.byDic = new Uint8Array(this.byDic); // not quicker, even slower
if (this.cStemming == "S") {
this.funcStemming = str_transform.changeWordWithSuffixCode;
} else if (this.cStemming == "A") {
this.funcStemming = str_transform.changeWordWithAffixCode;
} else {
this.funcStemming = str_transform.noStemming;
}
this._arcMask = (2 ** ((this.nBytesArc * 8) - 3)) - 1;
this._finalNodeMask = 1 << ((this.nBytesArc * 8) - 1);
this._lastArcMask = 1 << ((this.nBytesArc * 8) - 2);
// Configuring DAWG functions according to nCompressionMethod
switch (this.nCompressionMethod) {
case 1:
this.morph = this._morph1;
this.stem = this._stem1;
this._lookupArcNode = this._lookupArcNode1;
this._getArcs = this._getArcs1;
this._writeNodes = this._writeNodes1;
break;
|
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
|
this.morph = this._morph3;
this.stem = this._stem3;
this._lookupArcNode = this._lookupArcNode3;
this._getArcs = this._getArcs3;
this._writeNodes = this._writeNodes3;
break;
default:
throw ValueError("# Error: unknown code: " + this.nVersion);
}
//console.log(this.getInfo());
this.bOptNumSigle = true;
this.bOptNumAtLast = false;
}
getInfo () {
return ` Language: ${this.sLang} Version: ${this.nVersion} Date: ${this.sDate} Stemming: ${this.cStemming}FX\n` +
` Arcs values: ${this.nArcVal} = ${this.nChar} characters, ${this.nAff} affixes, ${this.nTag} tags\n` +
` Dictionary: ${this.nEntries} entries, ${this.nNode} nodes, ${this.nArc} arcs\n` +
` Address size: ${this.nBytesNodeAddress} bytes, Arc size: ${this.nBytesArc} bytes\n`;
}
isValidToken (sToken) {
// checks if sToken is valid (if there is hyphens in sToken, sToken is split, each part is checked)
|
|
|
|
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
|
this.morph = this._morph3;
this.stem = this._stem3;
this._lookupArcNode = this._lookupArcNode3;
this._getArcs = this._getArcs3;
this._writeNodes = this._writeNodes3;
break;
default:
throw ValueError("# Error: unknown code: " + this.nCompressionMethod);
}
//console.log(this.getInfo());
this.bOptNumSigle = true;
this.bOptNumAtLast = false;
}
getInfo () {
return ` Language: ${this.sLang} Version: ${this.nCompressionMethod} Date: ${this.sDate} Stemming: ${this.cStemming}FX\n` +
` Arcs values: ${this.nArcVal} = ${this.nChar} characters, ${this.nAff} affixes, ${this.nTag} tags\n` +
` Dictionary: ${this.nEntries} entries, ${this.nNode} nodes, ${this.nArc} arcs\n` +
` Address size: ${this.nBytesNodeAddress} bytes, Arc size: ${this.nBytesArc} bytes\n`;
}
isValidToken (sToken) {
// checks if sToken is valid (if there is hyphens in sToken, sToken is split, each part is checked)
|