handle filesystems where lowercase(file)==file more gracefully

[swftools.git] / lib / as3 / tokenizer.lex

diff --git a/lib/as3/tokenizer.lex b/lib/as3/tokenizer.lex
index 7557a88..ece94d0 100644 (file)
--- a/lib/as3/tokenizer.lex
+++ b/lib/as3/tokenizer.lex
@@ -76,6 +76,9 @@ void as3_buffer_input(void*buffer, int len)
     as3_in = 0;
 }
 
+//#undef BEGIN
+//#define BEGIN(x) {(yy_start) = 1 + 2 *x;dbg("entering state %d", x);}
+
 #define YY_INPUT(buf,result,max_size) { \
   if(!as3_buffer) { \
       errno=0; \
@@ -107,10 +110,10 @@ void handleInclude(char*text, int len, char quotes)
     } else {
         int i1=0,i2=len;
         // find start
-        while(!strchr(" \n\r\t", text[i1])) i1++;
+        while(!strchr(" \n\r\t\xa0", text[i1])) i1++;
         // strip
-        while(strchr(" \n\r\t", text[i1])) i1++;
-        while(strchr(" \n\r\t", text[i2-1])) i2--;
+        while(strchr(" \n\r\t\xa0", text[i1])) i1++;
+        while(strchr(" \n\r\t\xa0", text[i2-1])) i2--;
         if(i2!=len) text[i2]=0;
         filename = strdup(&text[i1]);
     }
@@ -502,6 +505,7 @@ static int tokenerror();
 //XMLCOMMENT  <!--([^->]|(-/[^-])|(--/[^>]))*-->
 
 //{XMLCOMMENT}                 
+
 %}
 
 %s REGEXPOK
@@ -526,9 +530,11 @@ FLOATWITHSIGN [+-]?({FLOAT})
 CDATA       <!\[CDATA\[([^]]|\][^]]|\]\][^>])*\]*\]\]\>
 XMLCOMMENT  <!--([^->]|[-]+[^>-]|>)*-*-->
 XML         <[^>]+{S}>
+XMLID       [A-Za-z0-9_\x80-\xff]+([:][A-Za-z0-9_\x80-\xff]+)?
+XMLSTRING   ["][^"]*["]
 
 STRING   ["](\\[\x00-\xff]|[^\\"\n])*["]|['](\\[\x00-\xff]|[^\\'\n])*[']
-S       [ \n\r\t]
+S       [ \n\r\t\xa0]
 MULTILINE_COMMENT [/][*]+([*][^/]|[^/*]|[^*][/]|[\x00-\x1f])*[*]+[/]
 SINGLELINE_COMMENT \/\/[^\n\r]*[\n\r]
 REGEXP   [/]([^/\n]|\\[/])*[/][a-zA-Z]*
@@ -540,7 +546,7 @@ REGEXP   [/]([^/\n]|\\[/])*[/][a-zA-Z]*
 [/][*]                       {syntaxerror("syntax error: unterminated comment", yytext);}
 
 ^include{S}+{STRING}{S}*/\n    {l();handleInclude(yytext, yyleng, 1);}
-^include{S}+[^" \t\r\n][\x20-\xff]*{S}*/\n    {l();handleInclude(yytext, yyleng, 0);}
+^include{S}+[^" \t\xa0\r\n][\x20-\xff]*{S}*/\n    {l();handleInclude(yytext, yyleng, 0);}
 {STRING}                     {l(); BEGIN(DEFAULT);handleString(yytext, yyleng);return T_STRING;}
 {CDATA}                      {l(); BEGIN(DEFAULT);handleCData(yytext, yyleng);return T_STRING;}
 
@@ -549,18 +555,20 @@ REGEXP   [/]([^/\n]|\\[/])*[/][a-zA-Z]*
 }
 
 <XML>{
-{STRING}                     {l(); handleString(yytext, yyleng);return T_STRING;}
+{XMLSTRING}                  {l(); handleRaw(yytext, yyleng);return T_STRING;}
+[{]                          {c(); BEGIN(REGEXPOK);return m('{');}
 [<]                          {c(); return m('<');}
 [/]                          {c(); return m('/');}
 [>]                          {c(); return m('>');}
 [=]                          {c(); return m('=');}
-{NAME}                       {c(); handleRaw(yytext, yyleng);return T_IDENTIFIER;}
+{XMLID}                      {c(); handleRaw(yytext, yyleng);return T_IDENTIFIER;}
 {S}                          {l();}
 <<EOF>>                      {syntaxerror("unexpected end of file");}
 }
 
 <XMLTEXT>{
-[^<>]+                       {l(); BEGIN(DEFAULT);handleRaw(yytext, yyleng);return T_STRING;}
+[^<>{]+                      {l(); handleRaw(yytext, yyleng);return T_STRING;}
+[{]                          {c(); BEGIN(REGEXPOK);return m('{');}
 [<]                          {c(); BEGIN(XML);return m('<');}
 [>]                          {c(); return m('>');}
 {XMLCOMMENT}                 {l(); handleRaw(yytext, yyleng);return T_STRING;}
@@ -595,6 +603,7 @@ NaN                          {c(); BEGIN(DEFAULT);return m(KW_NAN);}
 {NAME}{S}*:{S}*do/{_}        {l();BEGIN(DEFAULT);handleLabel(yytext, yyleng-2);return T_DO;}
 {NAME}{S}*:{S}*while/{_}     {l();BEGIN(DEFAULT);handleLabel(yytext, yyleng-5);return T_WHILE;}
 {NAME}{S}*:{S}*switch/{_}    {l();BEGIN(DEFAULT);handleLabel(yytext, yyleng-6);return T_SWITCH;}
+default{S}xml                {l();BEGIN(DEFAULT);return m(KW_DEFAULT_XML);}
 for                          {c();BEGIN(DEFAULT);a3_lval.id="";return T_FOR;}
 do                           {c();BEGIN(DEFAULT);a3_lval.id="";return T_DO;}
 while                        {c();BEGIN(DEFAULT);a3_lval.id="";return T_WHILE;}
@@ -632,9 +641,10 @@ switch                       {c();BEGIN(DEFAULT);a3_lval.id="";return T_SWITCH;}
 instanceof                   {c();BEGIN(REGEXPOK);return m(KW_INSTANCEOF);}
 implements                   {c();BEGIN(REGEXPOK);return m(KW_IMPLEMENTS);}
 interface                    {c();BEGIN(DEFAULT);return m(KW_INTERFACE);}
-namespace                    {c();BEGIN(DEFAULT);return m(KW_NAMESPACE);}
 protected                    {c();BEGIN(DEFAULT);return m(KW_PROTECTED);}
+namespace                    {c();BEGIN(DEFAULT);return m(KW_NAMESPACE);}
 undefined                    {c();BEGIN(DEFAULT);return m(KW_UNDEFINED);}
+arguments                    {c();BEGIN(DEFAULT);return m(KW_ARGUMENTS);}
 continue                     {c();BEGIN(DEFAULT);return m(KW_CONTINUE);}
 override                     {c();BEGIN(DEFAULT);return m(KW_OVERRIDE);}
 internal                     {c();BEGIN(DEFAULT);return m(KW_INTERNAL);}
@@ -721,7 +731,7 @@ static int tokenerror()
     if(c1>='0' && c1<='9')
         syntaxerror("syntax error: %s (identifiers must not start with a digit)");
     else
-        syntaxerror("syntax error: %s", buf);
+        syntaxerror("syntax error [%d]: %s", (yy_start-1)/2, buf);
     printf("\n");
     exit(1);
     yyterminate();
@@ -797,18 +807,22 @@ char*token2string(enum yytokentype nr, YYSTYPE v)
 
 void tokenizer_begin_xml()
 {
+    dbg("begin reading xml");
     BEGIN(XML);
 }
 void tokenizer_begin_xmltext()
 {
+    dbg("begin reading xml text");
     BEGIN(XMLTEXT);
 }
 void tokenizer_end_xmltext()
 {
+    dbg("end reading xml text");
     BEGIN(XML);
 }
 void tokenizer_end_xml()
 {
+    dbg("end reading xml");
     BEGIN(DEFAULT);
 }