396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
|
dToken["aTags"] = self.dTokenPos[dToken["nStart"]]["aTags"]
self.lTokens = lNewTokens
self.dTokenPos = { dToken["nStart"]: dToken for dToken in self.lTokens if dToken["sType"] != "INFO" }
if bDebug:
echo("UPDATE:")
echo(self)
def _getNextPointers (self, dToken, dGraph, dPointer, bDebug=False):
"generator: return nodes where <dToken> “values” match <dNode> arcs"
dNode = dGraph[dPointer["iNode"]]
iToken1 = dPointer["iToken1"]
bTokenFound = False
# token value
if dToken["sValue"] in dNode:
if bDebug:
echo(" MATCH: " + dToken["sValue"])
yield { "iToken1": iToken1, "iNode": dNode[dToken["sValue"]] }
if bDebug:
echo(" MATCH: " + sValue)
yield { "iToken1": iToken1, "iNode": dNode[sValue] }
if bDebug:
echo(" MATCH: " + sValue)
yield { "iToken1": iToken1, "iNode": dNode[sValue] }
if bDebug:
echo(" MATCH: " + sValue)
yield { "iToken1": iToken1, "iNode": dNode[sValue] }
if bDebug:
echo(" MATCH: ~" + sRegex)
yield { "iToken1": iToken1, "iNode": dNode["<re_value>"][sRegex] }
if bDebug:
echo(" MATCH: ~" + sRegex)
yield { "iToken1": iToken1, "iNode": dNode["<re_value>"][sRegex] }
if bDebug:
echo(" MATCH: >" + sLemma)
yield { "iToken1": iToken1, "iNode": dNode["<lemmas>"][sLemma] }
if bDebug:
echo(" MATCH: %" + sPhonet)
yield { "iToken1": iToken1, "iNode": dNode["<phonet>"][sPhonet] }
if bDebug:
echo(" MATCH: $" + sSearch)
yield { "iToken1": iToken1, "iNode": dNode["<morph>"][sSearch] }
if bDebug:
echo(" MATCH: $" + sSearch)
yield { "iToken1": iToken1, "iNode": dNode["<morph>"][sSearch] }
if bDebug:
echo(" MATCH: $" + sSearch)
yield { "iToken1": iToken1, "iNode": dNode["<morph>"][sSearch] }
if bDebug:
echo(" MATCH: @" + sRegex)
yield { "iToken1": iToken1, "iNode": dNode["<re_morph>"][sRegex] }
if bDebug:
echo(" MATCH: @" + sRegex)
yield { "iToken1": iToken1, "iNode": dNode["<re_morph>"][sRegex] }
if bDebug:
echo(" MATCH: @" + sRegex)
yield { "iToken1": iToken1, "iNode": dNode["<re_morph>"][sRegex] }
if bDebug:
echo(" MATCH: /" + sTag)
yield { "iToken1": iToken1, "iNode": dNode["<tags>"][sTag] }
if bDebug:
echo(" MATCH: *" + sMeta)
yield { "iToken1": iToken1, "iNode": dNode["<meta>"][sMeta] }
if bDebug:
echo(" MATCH: *" + sMeta)
yield { "iToken1": iToken1, "iNode": dNode["<meta>"][sMeta] }
if not bTokenFound and "bKeep" in dPointer:
yield dPointer
# JUMP
# Warning! Recurssion!
if "<>" in dNode:
dPointer2 = { "iToken1": iToken1, "iNode": dNode["<>"], "bKeep": True }
yield from self._getNextPointers(dToken, dGraph, dPointer2, bDebug)
lNextPointer.extend(self._getNextPointers(dToken, dGraph, dPointer, bDebug))
lPointer = lNextPointer
# check arcs of first nodes
lPointer.extend(self._getNextPointers(dToken, dGraph, { "iToken1": iToken, "iNode": 0 }, bDebug))
# check if there is rules to check for each pointer
for dPointer in lPointer:
#if bDebug:
# echo("+", dPointer)
if "<rules>" in dGraph[dPointer["iNode"]]:
bChange = self._executeActions(dGraph, dGraph[dPointer["iNode"]]["<rules>"], dPointer["iToken1"]-1, iToken, dOptions, sCountry, bShowRuleId, bDebug, bContext)
if bChange:
|
|
|
<
<
<
<
|
<
<
|
<
<
|
<
<
|
<
<
|
<
<
|
<
<
|
<
<
|
<
<
|
<
<
|
<
<
|
<
<
|
<
<
|
<
<
|
<
<
|
<
<
|
<
<
|
|
|
|
<
|
>
>
|
>
>
>
>
>
>
>
>
>
|
|
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
|
dToken["aTags"] = self.dTokenPos[dToken["nStart"]]["aTags"]
self.lTokens = lNewTokens
self.dTokenPos = { dToken["nStart"]: dToken for dToken in self.lTokens if dToken["sType"] != "INFO" }
if bDebug:
echo("UPDATE:")
echo(self)
def _getMatches (self, dGraph, dToken, dNode, bKeep=False):
"generator: return matches where <dToken> “values” match <dNode> arcs"
bTokenFound = False
# token value
if dToken["sValue"] in dNode:
yield (" ", dToken["sValue"], dNode[dToken["sValue"]])
bTokenFound = True
if dToken["sValue"][0:2].istitle(): # we test only 2 first chars, to make valid words such as "Laissez-les", "Passe-partout".
sValue = dToken["sValue"].lower()
if sValue in dNode:
yield (" ", sValue, dNode[sValue])
bTokenFound = True
elif dToken["sValue"].isupper():
sValue = dToken["sValue"].lower()
if sValue in dNode:
yield (" ", sValue, dNode[sValue])
bTokenFound = True
sValue = dToken["sValue"].capitalize()
if sValue in dNode:
yield (" ", sValue, dNode[sValue])
bTokenFound = True
# regex value arcs
if dToken["sType"] not in frozenset(["INFO", "PUNC", "SIGN"]):
if "<re_value>" in dNode:
for sRegex in dNode["<re_value>"]:
if "¬" not in sRegex:
# no anti-pattern
if re.search(sRegex, dToken["sValue"]):
yield ("~", sRegex, dNode["<re_value>"][sRegex])
bTokenFound = True
else:
# there is an anti-pattern
sPattern, sNegPattern = sRegex.split("¬", 1)
if sNegPattern and re.search(sNegPattern, dToken["sValue"]):
continue
if not sPattern or re.search(sPattern, dToken["sValue"]):
yield ("~", sRegex, dNode["<re_value>"][sRegex])
bTokenFound = True
# analysable tokens
if dToken["sType"][0:4] == "WORD":
# token lemmas
if "<lemmas>" in dNode:
for sLemma in _oSpellChecker.getLemma(dToken["sValue"]):
if sLemma in dNode["<lemmas>"]:
yield (">", sLemma, dNode["<lemmas>"][sLemma])
bTokenFound = True
# phonetic similarity
if "<phonet>" in dNode:
for sPhonet in dNode["<phonet>"]:
if sPhonet.endswith("!"):
sPhon = sPhonet[0:-1]
if dToken["sValue"] == sPhon:
continue
if dToken["sValue"][0:1].isupper():
if dToken["sValue"].lower() == sPhon:
continue
if dToken["sValue"].isupper() and dToken["sValue"].capitalize() == sPhon:
continue
if phonet.isSimilAs(dToken["sValue"], sPhonet.rstrip("!")):
yield ("#", sPhonet, dNode["<phonet>"][sPhonet])
bTokenFound = True
# morph arcs
if "<morph>" in dNode:
lMorph = dToken.get("lMorph", _oSpellChecker.getMorph(dToken["sValue"]))
if lMorph:
for sSearch in dNode["<morph>"]:
if "¬" not in sSearch:
# no anti-pattern
if any(sSearch in sMorph for sMorph in lMorph):
yield ("$", sSearch, dNode["<morph>"][sSearch])
bTokenFound = True
else:
# there is an anti-pattern
sPattern, sNegPattern = sSearch.split("¬", 1)
if sNegPattern == "*":
# all morphologies must match with <sPattern>
if sPattern:
if all(sPattern in sMorph for sMorph in lMorph):
yield ("$", sSearch, dNode["<morph>"][sSearch])
bTokenFound = True
else:
if sNegPattern and any(sNegPattern in sMorph for sMorph in lMorph):
continue
if not sPattern or any(sPattern in sMorph for sMorph in lMorph):
yield ("$", sSearch, dNode["<morph>"][sSearch])
bTokenFound = True
# regex morph arcs
if "<re_morph>" in dNode:
lMorph = dToken.get("lMorph", _oSpellChecker.getMorph(dToken["sValue"]))
if lMorph:
for sRegex in dNode["<re_morph>"]:
if "¬" not in sRegex:
# no anti-pattern
if any(re.search(sRegex, sMorph) for sMorph in lMorph):
yield ("@", sRegex, dNode["<re_morph>"][sRegex])
bTokenFound = True
else:
# there is an anti-pattern
sPattern, sNegPattern = sRegex.split("¬", 1)
if sNegPattern == "*":
# all morphologies must match with <sPattern>
if sPattern:
if all(re.search(sPattern, sMorph) for sMorph in lMorph):
yield ("@", sRegex, dNode["<re_morph>"][sRegex])
bTokenFound = True
else:
if sNegPattern and any(re.search(sNegPattern, sMorph) for sMorph in lMorph):
continue
if not sPattern or any(re.search(sPattern, sMorph) for sMorph in lMorph):
yield ("@", sRegex, dNode["<re_morph>"][sRegex])
bTokenFound = True
# token tags
if "aTags" in dToken and "<tags>" in dNode:
for sTag in dToken["aTags"]:
if sTag in dNode["<tags>"]:
yield ("/", sTag, dNode["<tags>"][sTag])
bTokenFound = True
# meta arc (for token type)
if "<meta>" in dNode:
for sMeta in dNode["<meta>"]:
# no regex here, we just search if <dNode["sType"]> exists within <sMeta>
if sMeta == "*" or dToken["sType"] == sMeta:
yield ("*", sMeta, dNode["<meta>"][sMeta])
bTokenFound = True
elif "¬" in sMeta:
if dToken["sType"] not in sMeta:
yield ("*", sMeta, dNode["<meta>"][sMeta])
bTokenFound = True
if not bTokenFound and bKeep:
yield (None, "", -1)
# JUMP
# Warning! Recursion!
if "<>" in dNode:
yield from self._getMatches(dGraph, dToken, dGraph[dNode["<>"]], bKeep=True)
def parseGraph (self, dGraph, sCountry="${country_default}", dOptions=None, bShowRuleId=False, bDebug=False, bContext=False):
"parse graph with tokens from the text and execute actions encountered"
lPointer = []
bTagAndRewrite = False
for iToken, dToken in enumerate(self.lTokens):
if bDebug:
echo("TOKEN: " + dToken["sValue"])
# check arcs for each existing pointer
lNextPointer = []
for dPointer in lPointer:
for cActionType, sMatch, iNode in self._getMatches(dGraph, dToken, dGraph[dPointer["iNode"]]):
if cActionType is None:
lNextPointer.append(dPointer)
continue
if bDebug:
echo(" MATCH: " + cActionType + sMatch)
lNextPointer.append({ "iToken1": dPointer["iToken1"], "iNode": iNode })
lPointer = lNextPointer
# check arcs of first nodes
for cActionType, sMatch, iNode in self._getMatches(dGraph, dToken, dGraph[0]):
if cActionType is None:
continue
if bDebug:
echo(" MATCH: " + cActionType + sMatch)
lPointer.append({ "iToken1": iToken, "iNode": iNode })
# check if there is rules to check for each pointer
for dPointer in lPointer:
#if bDebug:
# echo("+", dPointer)
if "<rules>" in dGraph[dPointer["iNode"]]:
bChange = self._executeActions(dGraph, dGraph[dPointer["iNode"]]["<rules>"], dPointer["iToken1"]-1, iToken, dOptions, sCountry, bShowRuleId, bDebug, bContext)
if bChange:
|