1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
// JavaScript
// Very simple tokenizer
/*jslint esversion: 6*/
/*global require,exports*/
"use strict";
if (typeof(require) !== 'undefined') {
var helpers = require("resource://grammalecte/graphspell/helpers.js");
}
|
<
<
<
<
<
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
|
// JavaScript
// Very simple tokenizer
/*jslint esversion: 6*/
/*global require,exports*/
"use strict";
const aTkzPatterns = {
// All regexps must start with ^.
"default":
[
[/^[ \t]+/, 'SPACE'],
[/^\/(?:~|bin|boot|dev|etc|home|lib|mnt|opt|root|sbin|tmp|usr|var|Bureau|Documents|Images|Musique|Public|Téléchargements|Vidéos)(?:\/[a-zA-Zà-öÀ-Ö0-9ø-ÿØ-ßĀ-ʯfi-st_.()-]+)*/, 'FOLDERUNIX'],
|
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
|
yield { "sType": sType, "sValue": m[0], "nStart": i, "nEnd": i + m[0].length }
}
nCut = m[0].length;
break;
}
}
catch (e) {
helpers.logerror(e);
}
}
i += nCut;
sText = sText.slice(nCut);
}
}
}
if (typeof(exports) !== 'undefined') {
exports.Tokenizer = Tokenizer;
}
|
|
|
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
|
yield { "sType": sType, "sValue": m[0], "nStart": i, "nEnd": i + m[0].length }
}
nCut = m[0].length;
break;
}
}
catch (e) {
console.error(e);
}
}
i += nCut;
sText = sText.slice(nCut);
}
}
}
if (typeof(exports) !== 'undefined') {
exports.Tokenizer = Tokenizer;
}
|