Overview
| Comment: | [core][js] tokenizer: yield separator one by one |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | core | kill_innerHTML |
| Files: | files | file ages | folders |
| SHA3-256: |
492169bd6ffb8f9e5493fe78da200db1 |
| User & Date: | olr on 2017-07-13 07:08:53 |
| Original Comment: | [core] tokenizer: yield separator one by one |
| Other Links: | branch diff | manifest | tags |
Context
|
2017-07-13
| ||
| 07:12 | [fr][js] lexicographe: nettoyage check-in: 3996597b39 user: olr tags: fr, kill_innerHTML | |
| 07:08 | [core][js] tokenizer: yield separator one by one check-in: 492169bd6f user: olr tags: core, kill_innerHTML | |
| 07:01 | [fr][js] corrige le commit précédent check-in: 622850e787 user: olr tags: fr, kill_innerHTML | |
Changes
Modified gc_core/js/tokenizer.js from [4eb5311054] to [8720e6c367].
| ︙ | ︙ | |||
52 53 54 55 56 57 58 |
let m;
let i = 0;
while (sText) {
let nCut = 1;
for (let [zRegex, sType] of this.aRules) {
try {
if ((m = zRegex.exec(sText)) !== null) {
| > > > > > | > | 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 |
let m;
let i = 0;
while (sText) {
let nCut = 1;
for (let [zRegex, sType] of this.aRules) {
try {
if ((m = zRegex.exec(sText)) !== null) {
if (sType == 'SEPARATOR') {
for (let c of m[0]) {
yield { "sType": sType, "sValue": c, "nStart": i, "nEnd": i + m[0].length }
}
} else {
yield { "sType": sType, "sValue": m[0], "nStart": i, "nEnd": i + m[0].length }
}
nCut = m[0].length;
break;
}
}
catch (e) {
helpers.logerror(e);
}
|
| ︙ | ︙ |