146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
|
def countWordsOccurrences (self, sText, bByLemma=False, bOnlyUnknownWords=False, dWord={}):
"""count word occurrences.
<dWord> can be used to cumulate count from several texts."""
if not self.oTokenizer:
self._loadTokenizer()
for dToken in self.oTokenizer.genTokens(sText):
if dToken['sType'] == "WORD":
if bOnlyUnknownWords:
if not self.isValidToken(dToken['sValue']):
dWord[dToken['sValue']] = dWord.get(dToken['sValue'], 0) + 1
else:
if not bByLemma:
dWord[dToken['sValue']] = dWord.get(dToken['sValue'], 0) + 1
else:
|
|
|
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
|
def countWordsOccurrences (self, sText, bByLemma=False, bOnlyUnknownWords=False, dWord={}):
"""count word occurrences.
<dWord> can be used to cumulate count from several texts."""
if not self.oTokenizer:
self._loadTokenizer()
for dToken in self.oTokenizer.genTokens(sText):
if dToken['sType'].startswith("WORD"):
if bOnlyUnknownWords:
if not self.isValidToken(dToken['sValue']):
dWord[dToken['sValue']] = dWord.get(dToken['sValue'], 0) + 1
else:
if not bByLemma:
dWord[dToken['sValue']] = dWord.get(dToken['sValue'], 0) + 1
else:
|