Overview
| Comment: | [graphspell][js] ibdawg optimization: precalculate bytes in binary dictionary |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | graphspell | bdic_opt |
| Files: | files | file ages | folders |
| SHA3-256: |
443f28094b386d4144cc6fcd905497e9 |
| User & Date: | olr on 2020-09-11 15:53:49 |
| Other Links: | branch diff | manifest | tags |
Context
|
2020-09-11
| ||
| 17:22 | [graphspell][py] ibdawg optimization: precalculate bytes in binary dictionary check-in: ae767aaff5 user: olr tags: graphspell, bdic_opt | |
| 15:53 | [graphspell][js] ibdawg optimization: precalculate bytes in binary dictionary check-in: 443f28094b user: olr tags: graphspell, bdic_opt | |
| 14:21 | [fr] faux positif check-in: 86f302f4ef user: olr tags: trunk, fr | |
Changes
Modified graphspell-js/ibdawg.js from [69d7490b82] to [1dc2c625a7].
| ︙ | ︙ | |||
126 127 128 129 130 131 132 133 134 |
/*
Bug workaround.
Mozilla’s JS parser sucks. Can’t read file bigger than 4 Mb!
So we convert huge hexadecimal string to list of numbers…
https://github.com/mozilla/addons-linter/issues/1361
*/
let lTemp = [];
for (let i = 0; i < this.sByDic.length; i+=2) {
| > > > > > > > > | > > > | > > > > > > > < | 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 |
/*
Bug workaround.
Mozilla’s JS parser sucks. Can’t read file bigger than 4 Mb!
So we convert huge hexadecimal string to list of numbers…
https://github.com/mozilla/addons-linter/issues/1361
*/
/*
Performance trick:
Instead of converting bytes to integers each times we parse the binary dictionary,
we do it once, then parse the array
*/
let nAcc = 0;
let lBytesBuffer = [];
let lTemp = [];
let nDivisor = (this.nBytesArc + this.nBytesNodeAddress) / 2;
for (let i = 0; i < this.sByDic.length; i+=2) {
lBytesBuffer.push(parseInt(this.sByDic.slice(i, i+2), 16));
if (nAcc == (this.nBytesArc - 1)) {
lTemp.push(this._convBytesToInteger(lBytesBuffer));
lBytesBuffer = [];
}
else if (nAcc == (this.nBytesArc + this.nBytesNodeAddress - 1)) {
lTemp.push(Math.round(this._convBytesToInteger(lBytesBuffer) / nDivisor)); // Math.round should be useless, BUT with JS who knowns what can happen…
lBytesBuffer = [];
nAcc = -1;
}
nAcc = nAcc + 1;
}
this.byDic = lTemp;
/* end of bug workaround */
if (!(this.sHeader.startsWith("/grammalecte-fsa/") || this.sHeader.startsWith("/pyfsa/"))) {
throw TypeError("# Error. Not a grammalecte-fsa binary dictionary. Header: " + this.sHeader);
}
if (!(this.nCompressionMethod == 1 || this.nCompressionMethod == 2 || this.nCompressionMethod == 3)) {
throw RangeError("# Error. Unknown dictionary compression method: " + this.nCompressionMethod);
|
| ︙ | ︙ | |||
194 195 196 197 198 199 200 |
// lexicographer module ?
this.lexicographer = null;
// JS still sucks: we’ll try importation when importation will be available in Workers. Still waiting...
if (self && self.hasOwnProperty("lexgraph_"+this.sLangCode)) { // self is the Worker
this.lexicographer = self["lexgraph_"+this.sLangCode];
}
| < | 211 212 213 214 215 216 217 218 219 220 221 222 223 224 |
// lexicographer module ?
this.lexicographer = null;
// JS still sucks: we’ll try importation when importation will be available in Workers. Still waiting...
if (self && self.hasOwnProperty("lexgraph_"+this.sLangCode)) { // self is the Worker
this.lexicographer = self["lexgraph_"+this.sLangCode];
}
}
getInfo () {
return ` Language: ${this.sLangName} Lang code: ${this.sLangCode} Dictionary name: ${this.sDicName}\n` +
` Compression method: ${this.nCompressionMethod} Date: ${this.sDate} Stemming: ${this.cStemming}FX\n` +
` Arcs values: ${this.nArcVal} = ${this.nChar} characters, ${this.nAff} affixes, ${this.nTag} tags\n` +
` Dictionary: ${this.nEntry} entries, ${this.nNode} nodes, ${this.nArc} arcs\n` +
|
| ︙ | ︙ | |||
304 305 306 307 308 309 310 |
return false;
}
iAddr = this._lookupArcNode(this.dChar.get(c), iAddr);
if (iAddr === null) {
return false;
}
}
| | | 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 |
return false;
}
iAddr = this._lookupArcNode(this.dChar.get(c), iAddr);
if (iAddr === null) {
return false;
}
}
return Boolean(this.byDic[iAddr] & this._finalNodeMask);
}
getMorph (sWord) {
// retrieves morphologies list, different casing allowed
if (!sWord) {
return [];
}
|
| ︙ | ︙ | |||
376 377 378 379 380 381 382 |
}
}
}
_suggest (oSuggResult, sRemain, nMaxSwitch=0, nMaxDel=0, nMaxHardRepl=0, nMaxJump=0, nDist=0, nDeep=0, iAddr=0, sNewWord="", bAvoidLoop=false) {
// returns a set of suggestions
// recursive function
| | | 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 |
}
}
}
_suggest (oSuggResult, sRemain, nMaxSwitch=0, nMaxDel=0, nMaxHardRepl=0, nMaxJump=0, nDist=0, nDeep=0, iAddr=0, sNewWord="", bAvoidLoop=false) {
// returns a set of suggestions
// recursive function
if (this.byDic[iAddr] & this._finalNodeMask) {
if (sRemain == "") {
oSuggResult.addSugg(sNewWord);
for (let sTail of this._getTails(iAddr)) {
oSuggResult.addSugg(sNewWord+sTail);
}
return;
}
|
| ︙ | ︙ | |||
486 487 488 489 490 491 492 |
}
_getTails (iAddr, sTail="", n=2) {
// return a list of suffixes ending at a distance of <n> from <iAddr>
let aTails = new Set();
for (let [nVal, jAddr] of this._getArcs(iAddr)) {
if (nVal <= this.nChar) {
| | | 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 |
}
_getTails (iAddr, sTail="", n=2) {
// return a list of suffixes ending at a distance of <n> from <iAddr>
let aTails = new Set();
for (let [nVal, jAddr] of this._getArcs(iAddr)) {
if (nVal <= this.nChar) {
if (this.byDic[jAddr] & this._finalNodeMask) {
aTails.add(sTail + this.dCharVal.get(nVal));
}
if (n && aTails.size == 0) {
aTails.gl_update(this._getTails(jAddr, sTail+this.dCharVal.get(nVal), n-1));
}
}
}
|
| ︙ | ︙ | |||
564 565 566 567 568 569 570 |
return [];
}
iAddr = this._lookupArcNode(this.dChar.get(c), iAddr);
if (iAddr === null) {
return [];
}
}
| | | | | | | | | | | | | | | | | | | | | | 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 |
return [];
}
iAddr = this._lookupArcNode(this.dChar.get(c), iAddr);
if (iAddr === null) {
return [];
}
}
if (this.byDic[iAddr] & this._finalNodeMask) {
let l = [];
let nRawArc = 0;
while (!(nRawArc & this._lastArcMask)) {
let iEndArcAddr = iAddr + 1;
nRawArc = this.byDic[iAddr];
let nArc = nRawArc & this._arcMask;
if (nArc > this.nChar) {
// This value is not a char, this is a stemming code
let sStem = ">" + this.funcStemming(sWord, this.lArcVal[nArc]);
// Now , we go to the next node and retrieve all following arcs values, all of them are tags
let iAddr2 = this.byDic[iEndArcAddr];
let nRawArc2 = 0;
while (!(nRawArc2 & this._lastArcMask)) {
let iEndArcAddr2 = iAddr2 + 1;
nRawArc2 = this.byDic[iAddr2];
l.push(sStem + "/" + this.lArcVal[nRawArc2 & this._arcMask]);
iAddr2 = iEndArcAddr2 + 1;
}
}
iAddr = iEndArcAddr + 1;
}
return l;
}
return [];
}
_stem1 (sWord) {
// returns stems list of sWord
let iAddr = 0;
for (let c of sWord) {
if (!this.dChar.has(c)) {
return [];
}
iAddr = this._lookupArcNode(this.dChar.get(c), iAddr);
if (iAddr === null) {
return [];
}
}
if (this.byDic[iAddr] & this._finalNodeMask) {
let l = [];
let nRawArc = 0;
while (!(nRawArc & this._lastArcMask)) {
let iEndArcAddr = iAddr + 1;
nRawArc = this.byDic[iAddr];
let nArc = nRawArc & this._arcMask;
if (nArc > this.nChar) {
// This value is not a char, this is a stemming code
l.push(this.funcStemming(sWord, this.lArcVal[nArc]));
}
iAddr = iEndArcAddr + 1;
}
return l;
}
return [];
}
_lookupArcNode1 (nVal, iAddr) {
// looks if nVal is an arc at the node at iAddr, if yes, returns address of next node else None
while (true) {
let iEndArcAddr = iAddr+1;
let nRawArc = this.byDic[iAddr];
if (nVal == (nRawArc & this._arcMask)) {
// the value we are looking for
// we return the address of the next node
return this.byDic[iEndArcAddr];
}
else {
// value not found
if (nRawArc & this._lastArcMask) {
return null;
}
iAddr = iEndArcAddr + 1;
}
}
}
* _getArcs1 (iAddr) {
// generator: return all arcs at <iAddr> as tuples of (nVal, iAddr)
while (true) {
let iEndArcAddr = iAddr+1;
let nRawArc = this.byDic[iAddr];
yield [nRawArc & this._arcMask, this.byDic[iEndArcAddr]];
if (nRawArc & this._lastArcMask) {
break;
}
iAddr = iEndArcAddr+1;
}
}
// VERSION 2
_morph2 (sWord) {
// to do
}
|
| ︙ | ︙ |