X-Git-Url: http://git.asbjorn.biz/?a=blobdiff_plain;f=lib%2Fas3%2Ftokenizer.lex;h=c84e3b3035609d62120a91209d503b5e52da6230;hb=f4992808059471b4f7f0f75f0b2f526d37d7715f;hp=27317a599b4cf3fa734ee765967bdc2894757137;hpb=c291a3dfd11cd189a00e7989e66db824a6a08cf2;p=swftools.git diff --git a/lib/as3/tokenizer.lex b/lib/as3/tokenizer.lex index 27317a5..c84e3b3 100644 --- a/lib/as3/tokenizer.lex +++ b/lib/as3/tokenizer.lex @@ -31,48 +31,43 @@ #include "tokenizer.h" #include "files.h" -static int verbose = 1; -static void dbg(const char*format, ...) +int as3_pass = 0; +int as3_verbosity = 1; +unsigned int as3_tokencount = 0; + +void as3_error(const char*format, ...) { char buf[1024]; int l; va_list arglist; - if(!verbose) - return; + if(as3_verbosity<0) + exit(1); va_start(arglist, format); vsprintf(buf, format, arglist); va_end(arglist); - l = strlen(buf); - while(l && buf[l-1]=='\n') { - buf[l-1] = 0; - l--; - } - printf("(tokenizer) "); - printf("%s\n", buf); - fflush(stdout); + fprintf(stderr, "%s:%d:%d: error: %s\n", current_filename_short, current_line, current_column, buf); + fflush(stderr); + exit(1); } - -void syntaxerror(const char*format, ...) +void as3_warning(const char*format, ...) { char buf[1024]; int l; va_list arglist; - if(!verbose) - return; + if(as3_verbosity<1) + return; va_start(arglist, format); vsprintf(buf, format, arglist); va_end(arglist); - fprintf(stderr, "%s:%d:%d: error: %s\n", current_filename_short, current_line, current_column, buf); + fprintf(stderr, "%s:%d:%d: warning: %s\n", current_filename_short, current_line, current_column, buf); fflush(stderr); - exit(1); } -void warning(const char*format, ...) +void as3_softwarning(const char*format, ...) { - return; char buf[1024]; int l; va_list arglist; - if(!verbose) + if(as3_verbosity<2) return; va_start(arglist, format); vsprintf(buf, format, arglist); @@ -80,12 +75,63 @@ void warning(const char*format, ...) fprintf(stderr, "%s:%d:%d: warning: %s\n", current_filename_short, current_line, current_column, buf); fflush(stderr); } - +static void dbg(const char*format, ...) +{ + char buf[1024]; + int l; + va_list arglist; + if(as3_verbosity<3) + return; + va_start(arglist, format); + vsprintf(buf, format, arglist); + va_end(arglist); + l = strlen(buf); + while(l && buf[l-1]=='\n') { + buf[l-1] = 0; + l--; + } + printf("(tokenizer) "); + printf("%s\n", buf); + fflush(stdout); +} #ifndef YY_CURRENT_BUFFER #define YY_CURRENT_BUFFER yy_current_buffer #endif +static void*as3_buffer=0; +static int as3_buffer_pos=0; +static int as3_buffer_len=0; +void as3_file_input(FILE*fi) +{ + as3_in = fi; + as3_buffer = 0; +} +void as3_buffer_input(void*buffer, int len) +{ + as3_buffer = buffer; + as3_buffer_len = len; + as3_buffer_pos = 0; + as3_in = 0; +} + +#define YY_INPUT(buf,result,max_size) { \ + if(!as3_buffer) { \ + errno=0; \ + while((result = fread(buf, 1, max_size, as3_in))==0 && ferror(as3_in)) \ + { if(errno != EINTR) {YY_FATAL_ERROR("input in flex scanner failed"); break;} \ + errno=0; clearerr(as3_in); \ + } \ + } else { \ + int to_read = max_size; \ + if(to_read + as3_buffer_pos > as3_buffer_len) \ + to_read = as3_buffer_len - as3_buffer_pos; \ + memcpy(buf, as3_buffer+as3_buffer_pos, to_read); \ + as3_buffer_pos += to_read; \ + result=to_read; \ + } \ +} + void handleInclude(char*text, int len, char quotes) { char*filename = 0; @@ -108,7 +154,8 @@ void handleInclude(char*text, int len, char quotes) filename = strdup(&text[i1]); } - char*fullfilename = enter_file(filename, YY_CURRENT_BUFFER); + char*fullfilename = find_file(filename); + enter_file2(filename, fullfilename, YY_CURRENT_BUFFER); yyin = fopen(fullfilename, "rb"); if (!yyin) { syntaxerror("Couldn't open include file \"%s\"\n", fullfilename); @@ -230,8 +277,21 @@ static string_t string_unescape(const char*in, int l) return out; } +static void handleCData(char*s, int len) +{ + a3_lval.str.str = s+9; // + a3_lval.str.str = strdup_n(a3_lval.str.str, a3_lval.str.len); +} + static void handleString(char*s, int len) { + if(as3_pass < 2) { + // don't bother decoding strings in pass 1 + memset(&a3_lval, 0, sizeof(a3_lval)); + return; + } + if(s[0]=='"') { if(s[len-1]!='"') syntaxerror("String doesn't end with '\"'"); s++;len-=2; @@ -243,24 +303,24 @@ static void handleString(char*s, int len) else syntaxerror("String incorrectly terminated"); - avm2_lval.str = string_unescape(s, len); + a3_lval.str = string_unescape(s, len); } char start_of_expression; -static inline int mkid(int type) +static inline int handleIdentifier(int type) { char*s = malloc(yyleng+1); memcpy(s, yytext, yyleng); s[yyleng]=0; - avm2_lval.id = s; + a3_lval.id = s; return type; } static inline int m(int type) { - avm2_lval.token = type; + a3_lval.token = type; return type; } @@ -278,7 +338,7 @@ static char*nrbuf() static inline int setint(int v) { - avm2_lval.number_int = v; + a3_lval.number_int = v; if(v>-128) return T_BYTE; else if(v>=-32768) @@ -288,7 +348,7 @@ static inline int setint(int v) } static inline int setuint(unsigned int v) { - avm2_lval.number_uint = v; + a3_lval.number_uint = v; if(v<128) return T_BYTE; else if(v<32768) @@ -298,14 +358,14 @@ static inline int setuint(unsigned int v) } static inline int setfloat(double v) { - avm2_lval.number_float = v; + a3_lval.number_float = v; return T_FLOAT; } static inline int handlefloat() { char*s = nrbuf(); - avm2_lval.number_float = atof(s); + a3_lval.number_float = atof(s); return T_FLOAT; } @@ -316,14 +376,14 @@ static inline int handleint() char*max = l?"1073741824":"2147483647"; if(yyleng-l>10) { - warning("integer overflow: %s", s); + as3_warning("integer overflow: %s (converted to Number)", s); return handlefloat(); } if(yyleng-l==10) { int t; for(t=0;tmax[t]) { - warning("integer overflow: %s", s); + as3_warning("integer overflow: %s (converted to Number)", s); return handlefloat(); } else if(yytext[l+t]='0' && c<='9') + d+=(c&15)*base; + else if((c>='a' && c<='f') || (c>='A' && c<='F')) + d+=((c&0x0f)+9)*base; + } + return setfloat(d); +} static inline int handlehex() { char l = (yytext[0]=='-')+2; + int len = yyleng; - if(yyleng-l>8) { + if(len-l>8) { char*s = nrbuf(); syntaxerror("integer overflow %s", s); } int t; unsigned int v = 0; - for(t=l;t='0' && c<='9') v|=(c&15); - else if(c>='a' && c<='f' || - c>='A' && c<='F') + else if((c>='a' && c<='f') || (c>='A' && c<='F')) v|=(c&0x0f)+9; } if(l && v>1073741824) { char*s = nrbuf(); - warning("signed integer overflow: %s", s); + as3_warning("signed integer overflow: %s (converted to Number)", s); return setfloat(v); } if(!l && v>2147483647) { char*s = nrbuf(); - warning("unsigned integer overflow: %s", s); + as3_warning("unsigned integer overflow: %s (converted to Number)", s); return setfloat(v); } @@ -393,7 +478,7 @@ void handleLabel(char*text, int len) char*s = malloc(t+1); memcpy(s, yytext, t); s[t]=0; - avm2_lval.id = s; + a3_lval.id = s; } static int handleregexp() @@ -409,11 +494,11 @@ static int handleregexp() break; } } - avm2_lval.regexp.pattern = s; + a3_lval.regexp.pattern = s; if(t==len) { - avm2_lval.regexp.options = 0; + a3_lval.regexp.options = 0; } else { - avm2_lval.regexp.options = s+t+1; + a3_lval.regexp.options = s+t+1; } return T_REGEXP; } @@ -443,7 +528,6 @@ static inline void c() { //uint {c();return m(KW_UINT);} //Number {c();return m(KW_NUMBER);} - %} %s REGEXPOK @@ -453,13 +537,16 @@ NAME [a-zA-Z_][a-zA-Z0-9_\\]* _ [^a-zA-Z0-9_\\] HEXINT 0x[a-zA-Z0-9]+ +HEXFLOAT 0x[a-zA-Z0-9]*\.[a-zA-Z0-9]* INT [0-9]+ FLOAT [0-9]+(\.[0-9]*)?|\.[0-9]+ HEXWITHSIGN [+-]?({HEXINT}) +HEXFLOATWITHSIGN [+-]?({HEXFLOAT}) INTWITHSIGN [+-]?({INT}) FLOATWITHSIGN [+-]?({FLOAT}) +CDATA ])*\]*\]\]\> STRING ["](\\[\x00-\xff]|[^\\"\n])*["]|['](\\[\x00-\xff]|[^\\'\n])*['] S [ \n\r\t] MULTILINE_COMMENT [/][*]+([*][^/]|[^/*]|[^*][/]|[\x00-\x1f])*[*]+[/] @@ -475,10 +562,12 @@ REGEXP [/]([^/\n]|\\[/])*[/][a-zA-Z]* ^include{S}+{STRING}{S}*/\n {l();handleInclude(yytext, yyleng, 1);} ^include{S}+[^" \t\r\n][\x20-\xff]*{S}*/\n {l();handleInclude(yytext, yyleng, 0);} {STRING} {l(); BEGIN(INITIAL);handleString(yytext, yyleng);return T_STRING;} +{CDATA} {l(); BEGIN(INITIAL);handleCData(yytext, yyleng);return T_STRING;} { {REGEXP} {c(); BEGIN(INITIAL);return handleregexp();} {HEXWITHSIGN} {c(); BEGIN(INITIAL);return handlehex();} +{HEXFLOATWITHSIGN} {c(); BEGIN(INITIAL);return handlehexfloat();} {INTWITHSIGN} {c(); BEGIN(INITIAL);return handleint();} {FLOATWITHSIGN} {c(); BEGIN(INITIAL);return handlefloat();} } @@ -487,6 +576,7 @@ REGEXP [/]([^/\n]|\\[/])*[/][a-zA-Z]* {S} {l();} {HEXINT} {c(); BEGIN(INITIAL);return handlehex();} +{HEXFLOAT} {c(); BEGIN(INITIAL);return handlehexfloat();} {INT} {c(); BEGIN(INITIAL);return handleint();} {FLOAT} {c(); BEGIN(INITIAL);return handlefloat();} @@ -497,10 +587,10 @@ REGEXP [/]([^/\n]|\\[/])*[/][a-zA-Z]* {NAME}{S}*:{S}*do/{_} {l();handleLabel(yytext, yyleng-2);return T_DO;} {NAME}{S}*:{S}*while/{_} {l();handleLabel(yytext, yyleng-5);return T_WHILE;} {NAME}{S}*:{S}*switch/{_} {l();handleLabel(yytext, yyleng-6);return T_SWITCH;} -for {c();avm2_lval.id="";return T_FOR;} -do {c();avm2_lval.id="";return T_DO;} -while {c();avm2_lval.id="";return T_WHILE;} -switch {c();avm2_lval.id="";return T_SWITCH;} +for {c();a3_lval.id="";return T_FOR;} +do {c();a3_lval.id="";return T_DO;} +while {c();a3_lval.id="";return T_WHILE;} +switch {c();a3_lval.id="";return T_SWITCH;} [&][&] {c();BEGIN(REGEXPOK);return m(T_ANDAND);} [|][|] {c();BEGIN(REGEXPOK);return m(T_OROR);} @@ -539,6 +629,7 @@ continue {c();return m(KW_CONTINUE);} override {c();return m(KW_OVERRIDE);} internal {c();return m(KW_INTERNAL);} function {c();return m(KW_FUNCTION);} +finally {c();return m(KW_FINALLY);} default {c();return m(KW_DEFAULT);} package {c();return m(KW_PACKAGE);} private {c();return m(KW_PRIVATE);} @@ -576,12 +667,13 @@ is {c();return m(KW_IS) ;} in {c();return m(KW_IN) ;} if {c();return m(KW_IF) ;} as {c();return m(KW_AS);} -{NAME} {c();BEGIN(INITIAL);return mkid(T_IDENTIFIER);} +{NAME} {c();BEGIN(INITIAL);return handleIdentifier(T_IDENTIFIER);} [+-\/*^~@$!%&\(=\[\]\{\}|?:;,<>] {c();BEGIN(REGEXPOK);return m(yytext[0]);} [\)\]] {c();BEGIN(INITIAL);return m(yytext[0]);} -. {char c1=yytext[0]; +. {/* ERROR */ + char c1=yytext[0]; char buf[128]; buf[0] = yytext[0]; int t; @@ -678,3 +770,4 @@ void initialize_scanner() BEGIN(BEGINNING); } +