Grammalecte  Diff

Differences From Artifact [b7c1b974f3]:

To Artifact [6eb071b95f]:


23
24
25
26
27
28
29







































30
31
32
33
34
35
36
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75







+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+







        fStart = time.time()
        result = func(*args, **kwargs)
        fEnd = time.time()
        print(func.__name__, fEnd - fStart)
        return result
    return wrapper


class SuggResult:

    def __init__ (self, sWord, nDistLimit=-1):
        self.sWord = sWord
        self.sCleanWord = cp.cleanWord(sWord)
        self.nDistLimit = nDistLimit  if nDistLimit >= 0  else  (len(sWord) // 3) + 1
        self.nMinDist = 1000
        self.nMaxDist = 0
        self.aSugg = set()
        self.dSugg = { 0: [],  1: [],  2: [] }

    def addSugg (self, sSugg, nDeep=0):
        "add a suggestion"
        if sSugg not in self.aSugg:
            nDist = st.distanceSift4(self.sCleanWord, cp.cleanWord(sSugg))
            if nDist <= self.nDistLimit:
                if nDist not in self.dSugg:
                    self.dSugg[nDist] = []
                self.dSugg[nDist].append(sSugg)
                logging.info((nDeep * "  ") + "__" + sSugg + "__")
                if nDist > self.nMaxDist:
                    self.nMaxDist = nDist
                if nDist < self.nMinDist:
                    self.nMinDist = nDist
                self.nDistLimit = min(self.nDistLimit, self.nMinDist+2)

    def getSuggestions (self, nSuggLimit=10, nDistLimit=-1):
        "return a list of suggestions"
        lRes = []
        for lSugg in self.dSugg.values():
            lRes.extend(lSugg)
            if len(lRes) > nSuggLimit:
                break
        lRes = list(cp.filterSugg(lRes))
        if self.sWord.istitle():
            lRes = list(map(lambda sSugg: sSugg.title(), lRes))
        return lRes[:nSuggLimit]


class IBDAWG:
    """INDEXABLE BINARY DIRECT ACYCLIC WORD GRAPH"""

    def __init__ (self, sDicName):
        self.by = pkgutil.get_data(__package__, "_dictionaries/" + sDicName)
        if not self.by:
210
211
212
213
214
215
216
217

218
219
220
221
222
223
224
249
250
251
252
253
254
255

256
257
258
259
260
261
262
263







-
+







        if sWord.istitle():
            aSugg.update(self._suggest(sWord.lower(), nMaxDel=nMaxDel, nMaxHardRepl=nMaxHardRepl))
            aSugg = set(map(lambda sSugg: sSugg.title(), aSugg))
        elif sWord.islower():
            aSugg.update(self._suggest(sWord.title(), nMaxDel=nMaxDel, nMaxHardRepl=nMaxHardRepl))
        aSugg = cp.filterSugg(aSugg)
        sCleanWord = cp.cleanWord(sWord)
        aSugg = sorted(aSugg, key=lambda sSugg: cp.distanceDamerauLevenshtein(sCleanWord, cp.cleanWord(sSugg)))[:nMaxSugg]
        aSugg = sorted(aSugg, key=lambda sSugg: st.distanceDamerauLevenshtein(sCleanWord, cp.cleanWord(sSugg)))[:nMaxSugg]
        if sSfx or sPfx:
            # we add what we removed
            return list(map(lambda sSug: sPfx + sSug + sSfx, aSugg))
        return aSugg

    def _suggest (self, sRemain, nMaxDel=0, nMaxHardRepl=0, nDeep=0, iAddr=0, sNewWord="", sAction="", bAvoidLoop=False):
        "returns a set of suggestions"
268
269
270
271
272
273
274
275

276
277

278
279
280
281
282
283

284
285
286
287
288
289
290

291
292

293
294
295


296
297
298
299
300
301

302
303
304


305
306
307
308
309
310
311
307
308
309
310
311
312
313

314


315






316

317
318
319
320
321

322
323

324
325


326
327

328
329



330
331


332
333
334
335
336
337
338
339
340







-
+
-
-
+
-
-
-
-
-
-
+
-





-
+

-
+

-
-
+
+
-


-
-
-
+

-
-
+
+







                    aSugg.update(self._suggest(sRepl, nMaxDel, nMaxHardRepl, nDeep+1, iAddr, sNewWord, sRemain + " >> " + sRepl, True))
        return aSugg

    @timethis
    def suggest2 (self, sWord, nMaxSugg=10):
        "returns a set of suggestions for <sWord>"
        sPfx, sWord, sSfx = cp.cut(sWord)
        nMaxDist = (len(sWord) // 3) + 1
        oSuggResult = SuggResult(sWord)
        sCleanWord = cp.cleanWord(sWord)
        aSugg = self._suggest2(sWord, sCleanWord, nMaxDist)
        self._suggest2(oSuggResult)
        if sWord.istitle():
            aSugg.update(self._suggest2(sWord.lower(), sCleanWord, nMaxDist))
            aSugg = set(map(lambda sSugg: sSugg.title(), aSugg))
        elif sWord.islower():
            aSugg.update(self._suggest2(sWord.title(), sCleanWord, nMaxDist))
        aSugg = cp.filterSugg(aSugg)
        aSugg = oSuggResult.getSuggestions()
        aSugg = sorted(aSugg, key=lambda sSugg: cp.distanceSift4(sCleanWord, cp.cleanWord(sSugg)))[:nMaxSugg]
        if sSfx or sPfx:
            # we add what we removed
            return list(map(lambda sSug: sPfx + sSug + sSfx, aSugg))
        return aSugg

    def _suggest2 (self, sWord, sCleanWord, nMaxDist, nDeep=0, iAddr=0, sNewWord=""):
    def _suggest2 (self, oSuggResult, nDeep=0, iAddr=0, sNewWord=""):
        #logging.info((nDeep * "  ") + sNewWord)
        if nDeep >= nMaxDist:
        if nDeep >= oSuggResult.nDistLimit:
            sCleanNewWord = cp.cleanWord(sNewWord)
            if cp.distanceSift4(sCleanWord[:len(sCleanNewWord)], sCleanNewWord) > nMaxDist:
                return set()
            if st.distanceSift4(oSuggResult.sCleanWord[:len(sCleanNewWord)], sCleanNewWord) > oSuggResult.nDistLimit:
                return
        aSugg = set()
        if int.from_bytes(self.byDic[iAddr:iAddr+self.nBytesArc], byteorder='big') & self._finalNodeMask:
            #logging.info((nDeep * "  ") + "__" + sNewWord + "__")
            sCleanNewWord = cp.cleanWord(sNewWord)
            if cp.distanceSift4(sCleanWord, sCleanNewWord) <= nMaxDist:
                aSugg.add(sNewWord)
            oSuggResult.addSugg(sNewWord, nDeep)
        for cChar, jAddr in self._getCharArcs(iAddr):
            aSugg.update(self._suggest2(sWord, sCleanWord, nMaxDist, nDeep+1, jAddr, sNewWord+cChar))
        return aSugg
            self._suggest2(oSuggResult, nDeep+1, jAddr, sNewWord+cChar)
        return

    def _getCharArcs (self, iAddr):
        "generator: yield all chars and addresses from node at address <iAddr>"
        for nVal, jAddr in self._getArcs(iAddr):
            if nVal < self.nChar:
                yield (self.dCharVal[nVal], jAddr)