Index: gc_lang/fr/perf_memo.txt ================================================================== --- gc_lang/fr/perf_memo.txt +++ gc_lang/fr/perf_memo.txt @@ -21,5 +21,6 @@ 0.5.15 2017.01.22 11:47 4.85593 1.15248 0.762924 0.22744 0.243461 0.254609 0.586741 0.317503 0.0588827 0.00701016 (unicode normalisation NFC) 0.5.15 2017.01.31 12:06 4.88227 1.18008 0.782217 0.232617 0.247672 0.257628 0.596903 0.32169 0.0603505 0.00695196 0.5.15 2017.02.05 10:10 4.90222 1.18444 0.786696 0.233413 0.25071 0.260214 0.602112 0.325235 0.0609932 0.00706897 0.5.16 2017.05.12 07:41 4.92201 1.19269 0.80639 0.239147 0.257518 0.266523 0.62111 0.33359 0.0634668 0.00757178 0.6.1 2018.02.12 09:58 5.25924 1.2649 0.878442 0.257465 0.280558 0.293903 0.686887 0.391275 0.0672474 0.00824723 +0.6.1 2018.02.19 09:06 6.20116 1.44334 1.02936 0.272956 0.311561 0.362367 0.812705 0.419061 0.0773003 0.00845671 (spelling normalization) Index: graphspell-js/char_player.js ================================================================== --- graphspell-js/char_player.js +++ graphspell-js/char_player.js @@ -4,29 +4,42 @@ ${map} var char_player = { - _dTransChars: new Map([ + _xTransCharsForSpelling: new Map([ + ['ſ', 's'], ['ffi', 'ffi'], ['ffl', 'ffl'], ['ff', 'ff'], ['ſt', 'ft'], ['fi', 'fi'], ['fl', 'fl'], ['st', 'st'] + ]), + + spellingNormalization: function (sWord) { + let sNewWord = ""; + for (let c of sWord) { + sNewWord += this._xTransCharsForSpelling.gl_get(c, c); + } + return sNewWord.normalize("NFC"); + }, + + _xTransCharsForSimplification: new Map([ ['à', 'a'], ['é', 'e'], ['î', 'i'], ['ô', 'o'], ['û', 'u'], ['ÿ', 'i'], ['y', 'i'], ['â', 'a'], ['è', 'e'], ['ï', 'i'], ['ö', 'o'], ['ù', 'u'], ['ŷ', 'i'], ['ä', 'a'], ['ê', 'e'], ['í', 'i'], ['ó', 'o'], ['ü', 'u'], ['ý', 'i'], ['á', 'a'], ['ë', 'e'], ['ì', 'i'], ['ò', 'o'], ['ú', 'u'], ['ỳ', 'i'], ['ā', 'a'], ['ē', 'e'], ['ī', 'i'], ['ō', 'o'], ['ū', 'u'], ['ȳ', 'i'], ['ñ', 'n'], ['k', 'q'], ['w', 'v'], ['œ', 'oe'], ['æ', 'ae'], + ['ſ', 's'], ['ffi', 'ffi'], ['ffl', 'ffl'], ['ff', 'ff'], ['ſt', 'ft'], ['fi', 'fi'], ['fl', 'fl'], ['st', 'st'] ]), simplifyWord: function (sWord) { // word simplication before calculating distance between words sWord = sWord.toLowerCase(); let sNewWord = ""; let i = 1; for (let c of sWord) { - let cNew = this._dTransChars.gl_get(c, c); + let cNew = this._xTransCharsForSimplification.gl_get(c, c); let cNext = sWord.slice(i, i+1) - if (cNew != this._dTransChars.gl_get(cNext, cNext)) { + if (cNew != this._xTransCharsForSimplification.gl_get(cNext, cNext)) { sNewWord += cNew; } i++; } return sNewWord.replace(/eau/g, "o").replace(/au/g, "o").replace(/ai/g, "e").replace(/ei/g, "e").replace(/ph/g, "f"); Index: graphspell-js/ibdawg.js ================================================================== --- graphspell-js/ibdawg.js +++ graphspell-js/ibdawg.js @@ -208,10 +208,11 @@ return oJSON; } isValidToken (sToken) { // checks if sToken is valid (if there is hyphens in sToken, sToken is split, each part is checked) + sToken = char_player.spellingNormalization(sToken) if (this.isValid(sToken)) { return true; } if (sToken.includes("-")) { if (sToken.gl_count("-") > 4) { @@ -278,10 +279,11 @@ return Boolean(this._convBytesToInteger(this.byDic.slice(iAddr, iAddr+this.nBytesArc)) & this._finalNodeMask); } getMorph (sWord) { // retrieves morphologies list, different casing allowed + sWord = char_player.spellingNormalization(sWord) let l = this.morph(sWord); if (sWord[0].gl_isUpperCase()) { l.push(...this.morph(sWord.toLowerCase())); if (sWord.gl_isUpperCase() && sWord.length > 1) { l.push(...this.morph(sWord.gl_toCapitalize())); @@ -290,10 +292,11 @@ return l; } suggest (sWord, nSuggLimit=10) { // returns a array of suggestions for + sWord = char_player.spellingNormalization(sWord) let sPfx = ""; let sSfx = ""; [sPfx, sWord, sSfx] = char_player.cut(sWord); let nMaxSwitch = Math.max(Math.floor(sWord.length / 3), 1); let nMaxDel = Math.floor(sWord.length / 5); Index: graphspell/char_player.py ================================================================== --- graphspell/char_player.py +++ graphspell/char_player.py @@ -1,24 +1,34 @@ # list of similar chars # useful for suggestion mechanism import re +import unicodedata + + +_xTransCharsForSpelling = str.maketrans({ + 'ſ': 's', 'ffi': 'ffi', 'ffl': 'ffl', 'ff': 'ff', 'ſt': 'ft', 'fi': 'fi', 'fl': 'fl', 'st': 'st' +}) + +def spellingNormalization (sWord): + return unicodedata.normalize("NFC", sWord.translate(_xTransCharsForSpelling)) -_xTransChars = str.maketrans({ +_xTransCharsForSimplification = str.maketrans({ 'à': 'a', 'é': 'e', 'î': 'i', 'ô': 'o', 'û': 'u', 'ÿ': 'i', "y": "i", 'â': 'a', 'è': 'e', 'ï': 'i', 'ö': 'o', 'ù': 'u', 'ŷ': 'i', 'ä': 'a', 'ê': 'e', 'í': 'i', 'ó': 'o', 'ü': 'u', 'ý': 'i', 'á': 'a', 'ë': 'e', 'ì': 'i', 'ò': 'o', 'ú': 'u', 'ỳ': 'i', 'ā': 'a', 'ē': 'e', 'ī': 'i', 'ō': 'o', 'ū': 'u', 'ȳ': 'i', 'ñ': 'n', 'k': 'q', 'w': 'v', - 'œ': 'oe', 'æ': 'ae', + 'œ': 'oe', 'æ': 'ae', + 'ſ': 's', 'ffi': 'ffi', 'ffl': 'ffl', 'ff': 'ff', 'ſt': 'ft', 'fi': 'fi', 'fl': 'fl', 'st': 'st', }) def simplifyWord (sWord): "word simplication before calculating distance between words" - sWord = sWord.lower().translate(_xTransChars) + sWord = sWord.lower().translate(_xTransCharsForSimplification) sNewWord = "" for i, c in enumerate(sWord, 1): if c != sWord[i:i+1]: sNewWord += c return sNewWord.replace("eau", "o").replace("au", "o").replace("ai", "e").replace("ei", "e").replace("ph", "f") Index: graphspell/ibdawg.py ================================================================== --- graphspell/ibdawg.py +++ graphspell/ibdawg.py @@ -216,10 +216,11 @@ if bInJSModule: hDst.write(";\n\nexports.dictionary = dictionary;\n") def isValidToken (self, sToken): "checks if is valid (if there is hyphens in , is split, each part is checked)" + sToken = cp.spellingNormalization(sToken) if self.isValid(sToken): return True if "-" in sToken: if sToken.count("-") > 4: return True @@ -258,10 +259,11 @@ return False return bool(int.from_bytes(self.byDic[iAddr:iAddr+self.nBytesArc], byteorder='big') & self._finalNodeMask) def getMorph (self, sWord): "retrieves morphologies list, different casing allowed" + sWord = cp.spellingNormalization(sWord) l = self.morph(sWord) if sWord[0:1].isupper(): l.extend(self.morph(sWord.lower())) if sWord.isupper() and len(sWord) > 1: l.extend(self.morph(sWord.capitalize())) @@ -268,10 +270,11 @@ return l #@timethis def suggest (self, sWord, nSuggLimit=10): "returns a set of suggestions for " + sWord = cp.spellingNormalization(sWord) sPfx, sWord, sSfx = cp.cut(sWord) nMaxSwitch = max(len(sWord) // 3, 1) nMaxDel = len(sWord) // 5 nMaxHardRepl = max((len(sWord) - 5) // 4, 1) oSuggResult = SuggResult(sWord) @@ -328,10 +331,11 @@ self._suggest(oSuggResult, sRepl, nMaxSwitch, nMaxDel, nMaxHardRepl, nDeep+1, iAddr, sNewWord, True) #@timethis def suggest2 (self, sWord, nMaxSugg=10): "returns a set of suggestions for " + sWord = cp.spellingNormalization(sWord) sPfx, sWord, sSfx = cp.cut(sWord) oSuggResult = SuggResult(sWord) self._suggest2(oSuggResult) aSugg = oSuggResult.getSuggestions() if sSfx or sPfx: @@ -384,10 +388,11 @@ aTails.update(self._getTails(jAddr, sTail+self.dCharVal[nVal], n-1)) return aTails def drawPath (self, sWord, iAddr=0): "show the path taken by in the graph" + sWord = cp.spellingNormalization(sWord) c1 = sWord[0:1] if sWord else " " iPos = -1 n = 0 print(c1 + ": ", end="") for c2, jAddr in self._getCharArcs(iAddr):