allow for utf8 names

[swftools.git] / lib / as3 / tokenizer.lex

diff --git a/lib/as3/tokenizer.lex b/lib/as3/tokenizer.lex
index 7b57674..4e0495d 100644 (file)
--- a/lib/as3/tokenizer.lex
+++ b/lib/as3/tokenizer.lex
@@ -110,10 +110,10 @@ void handleInclude(char*text, int len, char quotes)
     } else {
         int i1=0,i2=len;
         // find start
-        while(!strchr(" \n\r\t", text[i1])) i1++;
+        while(!strchr(" \n\r\t\xa0", text[i1])) i1++;
         // strip
-        while(strchr(" \n\r\t", text[i1])) i1++;
-        while(strchr(" \n\r\t", text[i2-1])) i2--;
+        while(strchr(" \n\r\t\xa0", text[i1])) i1++;
+        while(strchr(" \n\r\t\xa0", text[i2-1])) i2--;
         if(i2!=len) text[i2]=0;
         filename = strdup(&text[i1]);
     }
@@ -481,7 +481,7 @@ void tokenizer_unregister_namespace(const char*id)
 }*/
 static inline char tokenizer_is_namespace(const char*id)
 {
-    return trie_contains(active_namespaces, id);
+    return trie_contains(active_namespaces, (const unsigned char*)id);
 }
 
 static inline int handleIdentifier()
@@ -505,6 +505,7 @@ static int tokenerror();
 //XMLCOMMENT  <!--([^->]|(-/[^-])|(--/[^>]))*-->
 
 //{XMLCOMMENT}                 
+
 %}
 
 %s REGEXPOK
@@ -513,8 +514,8 @@ static int tokenerror();
 %x XMLTEXT
 %x XML
 
-NAME    [a-zA-Z_][a-zA-Z0-9_\\]*
-_        [^a-zA-Z0-9_\\]
+NAME    [a-zA-Z_\x80-\xff][a-zA-Z0-9_\\\x80-\xff]*
+_        [^a-zA-Z0-9_\\\x80-\xff]
 
 HEXINT    0x[a-zA-Z0-9]+
 HEXFLOAT  0x[a-zA-Z0-9]*\.[a-zA-Z0-9]*
@@ -533,7 +534,7 @@ XMLID       [A-Za-z0-9_\x80-\xff]+([:][A-Za-z0-9_\x80-\xff]+)?
 XMLSTRING   ["][^"]*["]
 
 STRING   ["](\\[\x00-\xff]|[^\\"\n])*["]|['](\\[\x00-\xff]|[^\\'\n])*[']
-S       [ \n\r\t]
+S       ([ \n\r\t\xa0]|\xc2\xa0)
 MULTILINE_COMMENT [/][*]+([*][^/]|[^/*]|[^*][/]|[\x00-\x1f])*[*]+[/]
 SINGLELINE_COMMENT \/\/[^\n\r]*[\n\r]
 REGEXP   [/]([^/\n]|\\[/])*[/][a-zA-Z]*
@@ -545,7 +546,7 @@ REGEXP   [/]([^/\n]|\\[/])*[/][a-zA-Z]*
 [/][*]                       {syntaxerror("syntax error: unterminated comment", yytext);}
 
 ^include{S}+{STRING}{S}*/\n    {l();handleInclude(yytext, yyleng, 1);}
-^include{S}+[^" \t\r\n][\x20-\xff]*{S}*/\n    {l();handleInclude(yytext, yyleng, 0);}
+^include{S}+[^" \t\xa0\r\n][\x20-\xff]*{S}*/\n    {l();handleInclude(yytext, yyleng, 0);}
 {STRING}                     {l(); BEGIN(DEFAULT);handleString(yytext, yyleng);return T_STRING;}
 {CDATA}                      {l(); BEGIN(DEFAULT);handleCData(yytext, yyleng);return T_STRING;}
 
@@ -730,7 +731,7 @@ static int tokenerror()
     if(c1>='0' && c1<='9')
         syntaxerror("syntax error: %s (identifiers must not start with a digit)");
     else
-        syntaxerror("syntax error [%d]: %s", (yy_start-1)/2, buf);
+        syntaxerror("syntax error [state=%d]: %s", (yy_start-1)/2, buf);
     printf("\n");
     exit(1);
     yyterminate();