From 9e103111a918b0d69c2b25a8a199c54eb719b8ac Mon Sep 17 00:00:00 2001 From: kramm Date: Tue, 30 Dec 2008 23:04:38 +0000 Subject: [PATCH 1/1] removed token_t type --- lib/as3/tokenizer.h | 16 +++-- lib/as3/tokenizer.lex | 163 +++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 154 insertions(+), 25 deletions(-) diff --git a/lib/as3/tokenizer.h b/lib/as3/tokenizer.h index 6acc38e..b8e5be3 100644 --- a/lib/as3/tokenizer.h +++ b/lib/as3/tokenizer.h @@ -36,12 +36,6 @@ DECLARE(params); DECLARE(typedcode); DECLARE_LIST(typedcode); -struct _token { - int type; - char*text; - token_list_t*tokens; -}; - struct _param { char*name; classinfo_t*type; @@ -61,6 +55,16 @@ struct _typedcode { memberinfo_t*f; }; +#define FLAG_PUBLIC 1 +#define FLAG_PRIVATE 2 +#define FLAG_PROTECTED 4 +#define FLAG_STATIC 8 +#define FLAG_DYNAMIC 16 +#define FLAG_FINAL 32 +#define FLAG_OVERRIDE 64 +#define FLAG_NATIVE 128 +#define FLAG_INTERNAL 256 + extern char start_of_expression; typedef token_t*tokenptr_t; diff --git a/lib/as3/tokenizer.lex b/lib/as3/tokenizer.lex index 95fa16c..2238ace 100644 --- a/lib/as3/tokenizer.lex +++ b/lib/as3/tokenizer.lex @@ -78,6 +78,19 @@ void syntaxerror(const char*format, ...) fflush(stderr); exit(1); } +void warning(const char*format, ...) +{ + char buf[1024]; + int l; + va_list arglist; + if(!verbose) + return; + va_start(arglist, format); + vsprintf(buf, format, arglist); + va_end(arglist); + fprintf(stderr, "%s:%d:%d: warning: %s\n", current_filename_short, current_line, current_column, buf); + fflush(stderr); +} #ifndef YY_CURRENT_BUFFER @@ -116,6 +129,109 @@ void handleInclude(char*text, int len, char quotes) //BEGIN(INITIAL); keep context } +string_t string_unescape(const char*in, int l) +{ + int len=0; + const char*s = in; + const char*end = &in[l]; + char*n = (char*)malloc(l); + char*o = n; + while(s256) + syntaxerror("octal number out of range (0-255): %d", num); + o[len++] = num; + continue; + } + case 'x': case 'u': { + int max=2; + char bracket = 0; + char unicode = 0; + if(*s == 'u') { + max = 6; + unicode = 1; + } + s++; + if(s==end) syntaxerror("invalid \\u or \\x at end of string"); + if(*s == '{') { + s++; + if(s==end) syntaxerror("invalid \\u{ at end of string"); + bracket=1; + } + unsigned int num=0; + int nr = 0; + while(strchr("0123456789abcdefABCDEF", *s) && (bracket || nr < max) && s='0' && *s<='9') num |= *s - '0'; + if(*s>='a' && *s<='f') num |= *s - 'a' + 10; + if(*s>='A' && *s<='F') num |= *s - 'A' + 10; + nr++; + s++; + } + if(bracket) { + if(*s=='}' && s256) + syntaxerror("byte out of range (0-255): %d", num); + o[len++] = num; + } + break; + } + default: + syntaxerror("unknown escape sequence: \"\\%c\"", *s); + } + } + string_t out = string_new(n, len); + o[len]=0; + return out; +} + static void handleString(char*s, int len) { if(s[0]=='"') { @@ -127,26 +243,30 @@ static void handleString(char*s, int len) s++;len-=2; } else syntaxerror("String incorrectly terminated"); - s[len] = 0; - avm2_lval.string = s; + + + avm2_lval.str = string_unescape(s, len); } char start_of_expression; -static inline int m(int type) +static inline int mkid(int type) { char*s = malloc(yyleng+1); memcpy(s, yytext, yyleng); s[yyleng]=0; + avm2_lval.id = s; + return type; +} - NEW(token_t,t); - t->type = type; - t->text = s; - avm2_lval.token = t; +static inline int m(int type) +{ + avm2_lval.token = type; return type; } + static char numberbuf[64]; static inline int handlenumber() { @@ -227,7 +347,7 @@ NUMBER -?[0-9]+(\.[0-9]*)? STRING ["](\\[\x00-\xff]|[^\\"\n])*["]|['](\\[\x00-\xff]|[^\\'\n])*['] S [ \n\r\t] -MULTILINE_COMMENT [/][*]+([*][^/]|[^/*]|[\x00-\x1f])*[*]+[/] +MULTILINE_COMMENT [/][*]+([*][^/]|[^/*]|[^*][/]|[\x00-\x1f])*[*]+[/] SINGLELINE_COMMENT \/\/[^\n]*\n REGEXP [/]([^/\n]|\\[/])*[/][a-zA-Z]* %% @@ -260,10 +380,20 @@ REGEXP [/]([^/\n]|\\[/])*[/][a-zA-Z]* [=][=] {c();BEGIN(REGEXPOK);return m(T_EQEQ);} [>][=] {c();return m(T_GE);} [<][=] {c();return m(T_LE);} -[+][=] {c();return m(T_PLUSBY);} -[-][=] {c();return m(T_MINUSBY);} [-][-] {c();BEGIN(INITIAL);return m(T_MINUSMINUS);} [+][+] {c();BEGIN(INITIAL);return m(T_PLUSPLUS);} +[+][=] {c();return m(T_PLUSBY);} +[-][=] {c();return m(T_MINUSBY);} +[/][=] {c();return m(T_DIVBY);} +[%][=] {c();return m(T_MODBY);} +[*][=] {c();return m(T_MULBY);} +[>][>][=] {c();return m(T_SHRBY);} +[<][<][=] {c();return m(T_SHLBY);} +[>][>][>][=] {c();return m(T_USHRBY);} +[<][<] {c();return m(T_SHL);} +[>][>][>] {c();return m(T_USHR);} +[>][>] {c();return m(T_SHR);} +\.\.\. {c();return m(T_DOTDOTDOT);} \.\. {c();return m(T_DOTDOT);} \. {c();return m('.');} :: {c();return m(T_COLONCOLON);} @@ -306,7 +436,7 @@ var {c();return m(KW_VAR);} is {c();return m(KW_IS) ;} if {c();return m(KW_IF) ;} as {c();return m(KW_AS);} -{NAME} {c();BEGIN(INITIAL);return m(T_IDENTIFIER);} +{NAME} {c();BEGIN(INITIAL);return mkid(T_IDENTIFIER);} [+-\/*^~@$!%&\(=\[\]\{\}|?:;,.<>] {c();BEGIN(REGEXPOK);return m(yytext[0]);} [\)\]] {c();BEGIN(INITIAL);return m(yytext[0]);} @@ -350,9 +480,8 @@ int yywrap() } static char mbuf[256]; -char*token2string(token_t*t) +char*token2string(enum yytokentype nr) { - int nr=t->type; if(nr==T_STRING) return ""; else if(nr==T_INT) return ""; else if(nr==T_UINT) return ""; @@ -397,12 +526,8 @@ char*token2string(token_t*t) else if(nr==KW_VAR) return "var"; else if(nr==KW_IS) return "is"; else if(nr==KW_AS) return "as"; - else if(nr==T_IDENTIFIER) { - if(strlen(t->text)>sizeof(mbuf)-1) - return "ID(...)"; - sprintf(mbuf, "ID(%s)", t->text); - return mbuf; - } else { + else if(nr==T_IDENTIFIER) return "ID"; + else { sprintf(mbuf, "%d", nr); return mbuf; } -- 1.7.10.4