@@ -108,7 +108,7 @@ var OPERATORS = makePredicate([
|
|||||||
"||"
|
"||"
|
||||||
]);
|
]);
|
||||||
|
|
||||||
var WHITESPACE_CHARS = makePredicate(characters(" \u00a0\n\r\t\f\u000b\u200b\u180e\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u202f\u205f\u3000"));
|
var WHITESPACE_CHARS = makePredicate(characters(" \u00a0\n\r\t\f\u000b\u200b\u180e\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u202f\u205f\u3000\uFEFF"));
|
||||||
|
|
||||||
var PUNC_BEFORE_EXPRESSION = makePredicate(characters("[{(,.;:"));
|
var PUNC_BEFORE_EXPRESSION = makePredicate(characters("[{(,.;:"));
|
||||||
|
|
||||||
@@ -213,7 +213,7 @@ var EX_EOF = {};
|
|||||||
function tokenizer($TEXT, filename, html5_comments) {
|
function tokenizer($TEXT, filename, html5_comments) {
|
||||||
|
|
||||||
var S = {
|
var S = {
|
||||||
text : $TEXT.replace(/^\uFEFF/g, ''),
|
text : $TEXT,
|
||||||
filename : filename,
|
filename : filename,
|
||||||
pos : 0,
|
pos : 0,
|
||||||
tokpos : 0,
|
tokpos : 0,
|
||||||
|
|||||||
Reference in New Issue
Block a user