finished ast implementation

[swftools.git] / lib / as3 / tokenizer.lex

diff --git a/lib/as3/tokenizer.lex b/lib/as3/tokenizer.lex
index f253b1b..127379a 100644 (file)
--- a/lib/as3/tokenizer.lex
+++ b/lib/as3/tokenizer.lex
@@ -28,53 +28,12 @@
 #include <stdio.h>
 #include <stdarg.h>
 #include "../utf8.h"
+#include "common.h"
 #include "tokenizer.h"
 #include "files.h"
 
-int as3_pass = 0;
-int as3_verbosity = 1;
 unsigned int as3_tokencount = 0;
 
-void as3_error(const char*format, ...)
-{
-    char buf[1024];
-    int l;
-    va_list arglist;
-    if(as3_verbosity<0)
-        exit(1);
-    va_start(arglist, format);
-    vsprintf(buf, format, arglist);
-    va_end(arglist);
-    fprintf(stderr, "%s:%d:%d: error: %s\n", current_filename_short, current_line, current_column, buf);
-    fflush(stderr);
-    exit(1);
-}
-void as3_warning(const char*format, ...)
-{
-    char buf[1024];
-    int l;
-    va_list arglist;
-    if(as3_verbosity<1)
-        return;
-    va_start(arglist, format);
-    vsprintf(buf, format, arglist);
-    va_end(arglist);
-    fprintf(stderr, "%s:%d:%d: warning: %s\n", current_filename_short, current_line, current_column, buf);
-    fflush(stderr);
-}
-void as3_softwarning(const char*format, ...)
-{
-    char buf[1024];
-    int l;
-    va_list arglist;
-    if(as3_verbosity<2)
-       return;
-    va_start(arglist, format);
-    vsprintf(buf, format, arglist);
-    va_end(arglist);
-    fprintf(stderr, "%s:%d:%d: warning: %s\n", current_filename_short, current_line, current_column, buf);
-    fflush(stderr);
-}
 static void dbg(const char*format, ...)
 {
     char buf[1024];
@@ -156,7 +115,7 @@ void handleInclude(char*text, int len, char quotes)
         filename = strdup(&text[i1]);
     }
     
-    char*fullfilename = find_file(filename);
+    char*fullfilename = find_file(filename, 1);
     enter_file2(filename, fullfilename, YY_CURRENT_BUFFER);
     yyin = fopen(fullfilename, "rb");
     if (!yyin) {
@@ -259,8 +218,15 @@ static int do_unescape(const char*s, const char*end, char*n)
                 }
                break;
            }
-            default:
-                syntaxerror("unknown escape sequence: \"\\%c\"", *s);
+            default: {
+               if(o) {
+                    o[len+0] = '\\';
+                    o[len+1] = *s;
+                }
+                s++;
+                len+=2;
+                break;
+            }
         }
     }
     if(o) o[len]=0;
@@ -288,12 +254,6 @@ static void handleCData(char*s, int len)
 
 static void handleString(char*s, int len)
 {
-    if(as3_pass < 2) {
-        // don't bother decoding strings in pass 1
-        memset(&a3_lval, 0, sizeof(a3_lval));
-        return;
-    }
-
     if(s[0]=='"') {
         if(s[len-1]!='"') syntaxerror("String doesn't end with '\"'");
         s++;len-=2;
@@ -303,7 +263,6 @@ static void handleString(char*s, int len)
         s++;len-=2;
     }
     else syntaxerror("String incorrectly terminated");
-
     
     a3_lval.str = string_unescape(s, len);
 }
@@ -332,22 +291,7 @@ static char*nrbuf()
 static inline int setint(int v)
 {
     a3_lval.number_int = v;
-    if(v>-128)
-        return T_BYTE;
-    else if(v>=-32768)
-        return T_SHORT;
-    else
-        return T_INT;
-}
-static inline int setuint(unsigned int v)
-{
-    a3_lval.number_uint = v;
-    if(v<128)
-        return T_BYTE;
-    else if(v<32768)
-        return T_SHORT;
-    else
-        return T_UINT;
+    return T_INT;
 }
 static inline int setfloat(double v)
 {
@@ -367,7 +311,9 @@ static inline int handleint()
     char*s = nrbuf();
     char l = (yytext[0]=='-');
 
-    char*max = l?"1073741824":"2147483647";
+    //char*max = l?"1073741824":"2147483647";
+    char*max = l?"2147483648":"2147483647";
+
     if(yyleng-l>10) {
         as3_softwarning("integer overflow: %s (converted to Number)", s);
         return handlefloat();
@@ -393,7 +339,7 @@ static inline int handleint()
             v*=10;
             v+=yytext[t]-'0';
         }
-        return setuint(v);
+        return setint(v);
     }
 }
 
@@ -442,21 +388,21 @@ static inline int handlehex()
         else if((c>='a' && c<='f') || (c>='A' && c<='F'))
             v|=(c&0x0f)+9;
     }
-    if(l && v>1073741824) {
+    if(l && v>=0x80000000) {
         char*s = nrbuf();
-        as3_softwarning("signed integer overflow: %s (converted to Number)", s);
+        as3_softwarning("integer overflow: %s (converted to Number)", s);
         return setfloat(v);
     }
-    if(!l && v>2147483647) {
+    if(!l && v>0x7fffffff) {
         char*s = nrbuf();
-        as3_softwarning("unsigned integer overflow: %s (converted to Number)", s);
+        as3_softwarning("integer overflow: %s (converted to Number)", s);
         return setfloat(v);
     }
 
     if(l==3) {
         return setint(-(int)v);
     } else {
-        return setuint(v);
+        return setint(v);
     }
 }
 
@@ -516,14 +462,18 @@ static inline void c() {
     current_column+=yyleng;
 }
 
-static trie_t*namespaces = 0;
-void tokenizer_register_namespace(const char*id)
+trie_t*active_namespaces = 0;
+/*void tokenizer_register_namespace(const char*id)
 {
-    trie_put(&namespaces, id);
+    trie_put(namespaces, id, 0);
 }
+void tokenizer_unregister_namespace(const char*id)
+{
+    trie_remove(namespaces, id);
+}*/
 static inline tokenizer_is_namespace(const char*id)
 {
-    return trie_lookup(namespaces, id);
+    return trie_contains(active_namespaces, id);
 }
 
 static inline int handleIdentifier()
@@ -598,6 +548,7 @@ REGEXP   [/]([^/\n]|\\[/])*[/][a-zA-Z]*
 {HEXFLOAT}/{_}               {c(); BEGIN(INITIAL);return handlehexfloat();}
 {INT}/{_}                    {c(); BEGIN(INITIAL);return handleint();}
 {FLOAT}/{_}                  {c(); BEGIN(INITIAL);return handlefloat();}
+NaN                          {c(); BEGIN(INITIAL);return m(KW_NAN);}
 
 3rr0r                        {/* for debugging: generates a tokenizer-level error */
                               syntaxerror("3rr0r");}
@@ -628,6 +579,7 @@ switch                       {c();BEGIN(INITIAL);a3_lval.id="";return T_SWITCH;}
 [%][=]                       {c();BEGIN(REGEXPOK);return m(T_MODBY);}
 [*][=]                       {c();BEGIN(REGEXPOK);return m(T_MULBY);}
 [|][=]                       {c();BEGIN(REGEXPOK);return m(T_ORBY);}
+[&][=]                       {c();BEGIN(REGEXPOK);return m(T_ANDBY);}
 [>][>][=]                    {c();BEGIN(REGEXPOK);return m(T_SHRBY);}
 [<][<][=]                    {c();BEGIN(REGEXPOK);return m(T_SHLBY);}
 [>][>][>][=]                 {c();BEGIN(REGEXPOK);return m(T_USHRBY);}
@@ -638,7 +590,7 @@ switch                       {c();BEGIN(INITIAL);a3_lval.id="";return T_SWITCH;}
 \.\.                         {c();BEGIN(REGEXPOK);return m(T_DOTDOT);}
 \.                           {c();BEGIN(REGEXPOK);return m('.');}
 ::                           {c();BEGIN(REGEXPOK);return m(T_COLONCOLON);}
-:                            {c();BEGIN(INITIAL);return m(':');}
+:                            {c();BEGIN(REGEXPOK);return m(':');}
 instanceof                   {c();BEGIN(REGEXPOK);return m(KW_INSTANCEOF);}
 implements                   {c();BEGIN(REGEXPOK);return m(KW_IMPLEMENTS);}
 interface                    {c();BEGIN(INITIAL);return m(KW_INTERFACE);}
@@ -687,7 +639,7 @@ is                           {c();BEGIN(REGEXPOK);return m(KW_IS) ;}
 in                           {c();BEGIN(REGEXPOK);return m(KW_IN) ;}
 if                           {c();BEGIN(INITIAL);return m(KW_IF) ;}
 as                           {c();BEGIN(REGEXPOK);return m(KW_AS);}
-{NAME}                       {c();BEGIN(INITIAL);return handleIdentifier();}
+$?{NAME}                       {c();BEGIN(INITIAL);return handleIdentifier();}
 
 [\]\}*]                       {c();BEGIN(INITIAL);return m(yytext[0]);}
 [+-\/^~@$!%&\(=\[|?:;,<>]   {c();BEGIN(REGEXPOK);return m(yytext[0]);}
@@ -754,7 +706,6 @@ char*token2string(enum yytokentype nr, YYSTYPE v)
     }
     else if(nr==T_INT)     return "<int>";
     else if(nr==T_UINT)     return "<uint>";
-    else if(nr==T_BYTE)     return "<byte>";
     else if(nr==T_FLOAT)     return "<float>";
     else if(nr==T_EOF)        return "***END***";
     else if(nr==T_GE)         return ">=";