current_column+=yyleng;
}
-trie_t*active_namespaces = 0;
-/*void tokenizer_register_namespace(const char*id)
-{
- trie_put(namespaces, id, 0);
-}
-void tokenizer_unregister_namespace(const char*id)
-{
- trie_remove(namespaces, id);
-}*/
-static inline char tokenizer_is_namespace(const char*id)
-{
- return trie_contains(active_namespaces, (const unsigned char*)id);
-}
-
static inline int handleIdentifier()
{
char*s = malloc(yyleng+1);
memcpy(s, yytext, yyleng);
s[yyleng]=0;
a3_lval.id = s;
- if(tokenizer_is_namespace(s))
- return T_NAMESPACE;
- else
- return T_IDENTIFIER;
+ return T_IDENTIFIER;
}
static int tokenerror();
%x XMLTEXT
%x XML
-NAME [a-zA-Z_\x80-\xff][a-zA-Z0-9_\\\x80-\xff]*
-_ [^a-zA-Z0-9_\\\x80-\xff]
+X1 parsing identifiers with a non unicode lexer is a knightmare we have to skip all possible
+X2 combinations of byte order markers or utf8 space chars and i dont quite like the fact that
+X3 lex doesnt support proper comments in this section either...
+X4 {NAME_HEAD}{NAME_TAIL}
+
+NAME_NOC2EF [a-zA-Z_\x80-\xc1\xc3-\xee\xf0-\xff]
+NAME_EF [\xef][a-zA-Z0-9_\\\x80-\xba\xbc-\xff]
+NAME_C2 [\xc2][a-zA-Z0-9_\\\x80-\x9f\xa1-\xff]
+NAME_EFBB [\xef][\xbb][a-zA-Z0-9_\\\x80-\xbe\xc0-\xff]
+NAME_TAIL [a-zA-Z_0-9\\\x80-\xff]*
+NAME_HEAD (({NAME_NOC2EF})|({NAME_EF})|({NAME_C2})|({NAME_EFBB}))
+NAME {NAME_HEAD}{NAME_TAIL}
+
+_ [^a-zA-Z0-9_\\\x80-\xff]
HEXINT 0x[a-zA-Z0-9]+
HEXFLOAT 0x[a-zA-Z0-9]*\.[a-zA-Z0-9]*
XMLSTRING ["][^"]*["]
STRING ["](\\[\x00-\xff]|[^\\"\n])*["]|['](\\[\x00-\xff]|[^\\'\n])*[']
-S ([ \n\r\t\xa0]|\xc2\xa0)
+S ([ \n\r\t\xa0]|[\xc2][\xa0])
MULTILINE_COMMENT [/][*]+([*][^/]|[^/*]|[^*][/]|[\x00-\x1f])*[*]+[/]
SINGLELINE_COMMENT \/\/[^\n\r]*[\n\r]
REGEXP [/]([^/\n]|\\[/])*[/][a-zA-Z]*
<REGEXPOK>[\{] {c(); BEGIN(REGEXPOK);return m(T_DICTSTART);}
[\{] {c(); BEGIN(DEFAULT); return m('{');}
-\xef\xbb\xbf {/* utf 8 bom */}
+\xef\xbb\xbf {/* utf 8 bom (0xfeff) */}
{S} {l();}
{HEXINT}/{_} {c(); BEGIN(DEFAULT);return handlehex();}