X-Git-Url: http://git.asbjorn.biz/?a=blobdiff_plain;f=lib%2Fas3%2Ftokenizer.lex;h=3da6c22e04a216fd580f6177713b04fe3cc0de4a;hb=0f1baaeab21e4bfd0c48e8d0d025bf8a15ccbc2c;hp=7557a8894b6f6660232c0e316f5d8246dae04d69;hpb=e4687b3aa2aed49fb16ba9e9561344d808750297;p=swftools.git diff --git a/lib/as3/tokenizer.lex b/lib/as3/tokenizer.lex index 7557a88..3da6c22 100644 --- a/lib/as3/tokenizer.lex +++ b/lib/as3/tokenizer.lex @@ -76,6 +76,9 @@ void as3_buffer_input(void*buffer, int len) as3_in = 0; } +//#undef BEGIN +//#define BEGIN(x) {(yy_start) = 1 + 2 *x;dbg("entering state %d", x);} + #define YY_INPUT(buf,result,max_size) { \ if(!as3_buffer) { \ errno=0; \ @@ -107,10 +110,10 @@ void handleInclude(char*text, int len, char quotes) } else { int i1=0,i2=len; // find start - while(!strchr(" \n\r\t", text[i1])) i1++; + while(!strchr(" \n\r\t\xa0", text[i1])) i1++; // strip - while(strchr(" \n\r\t", text[i1])) i1++; - while(strchr(" \n\r\t", text[i2-1])) i2--; + while(strchr(" \n\r\t\xa0", text[i1])) i1++; + while(strchr(" \n\r\t\xa0", text[i2-1])) i2--; if(i2!=len) text[i2]=0; filename = strdup(&text[i1]); } @@ -526,9 +529,11 @@ FLOATWITHSIGN [+-]?({FLOAT}) CDATA ])*\]*\]\]\> XMLCOMMENT XML <[^>]+{S}> +XMLID [A-Za-z0-9_\x80-\xff]+([:][A-Za-z0-9_\x80-\xff]+)? +XMLSTRING ["][^"]*["] STRING ["](\\[\x00-\xff]|[^\\"\n])*["]|['](\\[\x00-\xff]|[^\\'\n])*['] -S [ \n\r\t] +S [ \n\r\t\xa0] MULTILINE_COMMENT [/][*]+([*][^/]|[^/*]|[^*][/]|[\x00-\x1f])*[*]+[/] SINGLELINE_COMMENT \/\/[^\n\r]*[\n\r] REGEXP [/]([^/\n]|\\[/])*[/][a-zA-Z]* @@ -540,7 +545,7 @@ REGEXP [/]([^/\n]|\\[/])*[/][a-zA-Z]* [/][*] {syntaxerror("syntax error: unterminated comment", yytext);} ^include{S}+{STRING}{S}*/\n {l();handleInclude(yytext, yyleng, 1);} -^include{S}+[^" \t\r\n][\x20-\xff]*{S}*/\n {l();handleInclude(yytext, yyleng, 0);} +^include{S}+[^" \t\xa0\r\n][\x20-\xff]*{S}*/\n {l();handleInclude(yytext, yyleng, 0);} {STRING} {l(); BEGIN(DEFAULT);handleString(yytext, yyleng);return T_STRING;} {CDATA} {l(); BEGIN(DEFAULT);handleCData(yytext, yyleng);return T_STRING;} @@ -549,18 +554,20 @@ REGEXP [/]([^/\n]|\\[/])*[/][a-zA-Z]* } { -{STRING} {l(); handleString(yytext, yyleng);return T_STRING;} +{XMLSTRING} {l(); handleRaw(yytext, yyleng);return T_STRING;} +[{] {c(); BEGIN(REGEXPOK);return m('{');} [<] {c(); return m('<');} [/] {c(); return m('/');} [>] {c(); return m('>');} [=] {c(); return m('=');} -{NAME} {c(); handleRaw(yytext, yyleng);return T_IDENTIFIER;} +{XMLID} {c(); handleRaw(yytext, yyleng);return T_IDENTIFIER;} {S} {l();} <> {syntaxerror("unexpected end of file");} } { -[^<>]+ {l(); BEGIN(DEFAULT);handleRaw(yytext, yyleng);return T_STRING;} +[^<>{]+ {l(); handleRaw(yytext, yyleng);return T_STRING;} +[{] {c(); BEGIN(REGEXPOK);return m('{');} [<] {c(); BEGIN(XML);return m('<');} [>] {c(); return m('>');} {XMLCOMMENT} {l(); handleRaw(yytext, yyleng);return T_STRING;} @@ -595,6 +602,7 @@ NaN {c(); BEGIN(DEFAULT);return m(KW_NAN);} {NAME}{S}*:{S}*do/{_} {l();BEGIN(DEFAULT);handleLabel(yytext, yyleng-2);return T_DO;} {NAME}{S}*:{S}*while/{_} {l();BEGIN(DEFAULT);handleLabel(yytext, yyleng-5);return T_WHILE;} {NAME}{S}*:{S}*switch/{_} {l();BEGIN(DEFAULT);handleLabel(yytext, yyleng-6);return T_SWITCH;} +default{S}xml {l();BEGIN(DEFAULT);return m(KW_DEFAULT_XML);} for {c();BEGIN(DEFAULT);a3_lval.id="";return T_FOR;} do {c();BEGIN(DEFAULT);a3_lval.id="";return T_DO;} while {c();BEGIN(DEFAULT);a3_lval.id="";return T_WHILE;} @@ -632,9 +640,10 @@ switch {c();BEGIN(DEFAULT);a3_lval.id="";return T_SWITCH;} instanceof {c();BEGIN(REGEXPOK);return m(KW_INSTANCEOF);} implements {c();BEGIN(REGEXPOK);return m(KW_IMPLEMENTS);} interface {c();BEGIN(DEFAULT);return m(KW_INTERFACE);} -namespace {c();BEGIN(DEFAULT);return m(KW_NAMESPACE);} protected {c();BEGIN(DEFAULT);return m(KW_PROTECTED);} +namespace {c();BEGIN(DEFAULT);return m(KW_NAMESPACE);} undefined {c();BEGIN(DEFAULT);return m(KW_UNDEFINED);} +arguments {c();BEGIN(DEFAULT);return m(KW_ARGUMENTS);} continue {c();BEGIN(DEFAULT);return m(KW_CONTINUE);} override {c();BEGIN(DEFAULT);return m(KW_OVERRIDE);} internal {c();BEGIN(DEFAULT);return m(KW_INTERNAL);} @@ -721,7 +730,7 @@ static int tokenerror() if(c1>='0' && c1<='9') syntaxerror("syntax error: %s (identifiers must not start with a digit)"); else - syntaxerror("syntax error: %s", buf); + syntaxerror("syntax error [%d]: %s", (yy_start-1)/2, buf); printf("\n"); exit(1); yyterminate(); @@ -797,18 +806,22 @@ char*token2string(enum yytokentype nr, YYSTYPE v) void tokenizer_begin_xml() { + dbg("begin reading xml"); BEGIN(XML); } void tokenizer_begin_xmltext() { + dbg("begin reading xml text"); BEGIN(XMLTEXT); } void tokenizer_end_xmltext() { + dbg("end reading xml text"); BEGIN(XML); } void tokenizer_end_xml() { + dbg("end reading xml"); BEGIN(DEFAULT); }