allow .123 format numbers
[swftools.git] / lib / as3 / tokenizer.lex
1 /* tokenizer.lex
2
3    Routines for compiling Flash2 AVM2 ABC Actionscript
4
5    Extension module for the rfxswf library.
6    Part of the swftools package.
7
8    Copyright (c) 2008 Matthias Kramm <kramm@quiss.org>
9  
10    This program is free software; you can redistribute it and/or modify
11    it under the terms of the GNU General Public License as published by
12    the Free Software Foundation; either version 2 of the License, or
13    (at your option) any later version.
14
15    This program is distributed in the hope that it will be useful,
16    but WITHOUT ANY WARRANTY; without even the implied warranty of
17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18    GNU General Public License for more details.
19
20    You should have received a copy of the GNU General Public License
21    along with this program; if not, write to the Free Software
22    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
23 %{
24
25
26 #include <string.h>
27 #include <stdlib.h>
28 #include <stdio.h>
29 #include <stdarg.h>
30 #include "../utf8.h"
31 #include "tokenizer.h"
32 #include "files.h"
33
34 static void countlines(char*text, int len) {
35     int t;
36     for(t=0;t<len;t++) {
37         if(text[t]=='\n') {
38             current_line++;
39             current_column=0;
40         } else {
41             current_column++;
42         }
43     }
44 }
45
46 static int verbose = 1;
47 static void dbg(const char*format, ...)
48 {
49     char buf[1024];
50     int l;
51     va_list arglist;
52     if(!verbose)
53         return;
54     va_start(arglist, format);
55     vsprintf(buf, format, arglist);
56     va_end(arglist);
57     l = strlen(buf);
58     while(l && buf[l-1]=='\n') {
59         buf[l-1] = 0;
60         l--;
61     }
62     printf("(tokenizer) ");
63     printf("%s\n", buf);
64     fflush(stdout);
65 }
66
67 void syntaxerror(const char*format, ...)
68 {
69     char buf[1024];
70     int l;
71     va_list arglist;
72     if(!verbose)
73         return;
74     va_start(arglist, format);
75     vsprintf(buf, format, arglist);
76     va_end(arglist);
77     fprintf(stderr, "%s:%d:%d: error: %s\n", current_filename_short, current_line, current_column, buf);
78     fflush(stderr);
79     exit(1);
80 }
81 void warning(const char*format, ...)
82 {
83     char buf[1024];
84     int l;
85     va_list arglist;
86     if(!verbose)
87         return;
88     va_start(arglist, format);
89     vsprintf(buf, format, arglist);
90     va_end(arglist);
91     fprintf(stderr, "%s:%d:%d: warning: %s\n", current_filename_short, current_line, current_column, buf);
92     fflush(stderr);
93 }
94
95
96 #ifndef YY_CURRENT_BUFFER
97 #define YY_CURRENT_BUFFER yy_current_buffer
98 #endif
99
100 void handleInclude(char*text, int len, char quotes)
101 {
102     char*filename = 0;
103     if(quotes) {
104         char*p1 = strchr(text, '"');
105         char*p2 = strrchr(text, '"');
106         if(!p1 || !p2 || p1==p2) {
107             syntaxerror("Invalid include in line %d\n", current_line);
108         }
109         *p2 = 0;
110         filename = strdup(p1+1);
111     } else {
112         int i1=0,i2=len;
113         // find start
114         while(!strchr(" \n\r\t", text[i1])) i1++;
115         // strip
116         while(strchr(" \n\r\t", text[i1])) i1++;
117         while(strchr(" \n\r\t", text[i2-1])) i2--;
118         if(i2!=len) text[i2]=0;
119         filename = strdup(&text[i1]);
120     }
121     
122     char*fullfilename = enter_file(filename, YY_CURRENT_BUFFER);
123     yyin = fopen(fullfilename, "rb");
124     if (!yyin) {
125         syntaxerror("Couldn't open include file \"%s\"\n", fullfilename);
126     }
127
128     yy_switch_to_buffer(yy_create_buffer( yyin, YY_BUF_SIZE ) );
129     //BEGIN(INITIAL); keep context
130 }
131
132 string_t string_unescape(const char*in, int l)
133 {
134     int len=0;
135     const char*s = in;
136     const char*end = &in[l];
137     char*n = (char*)malloc(l);
138     char*o = n;
139     while(s<end) {
140         if(*s!='\\') {
141             o[len++] = *s;
142             s++;
143             continue;
144         }
145         s++; //skip past '\'
146         if(s==end) syntaxerror("invalid \\ at end of string");
147
148         /* handle the various line endings (mac, dos, unix) */
149         if(*s=='\r') { 
150             s++; 
151             if(s==end) break;
152             if(*s=='\n') 
153                 s++;
154             continue;
155         }
156         if(*s=='\n')  {
157             s++;
158             continue;
159         }
160         switch(*s) {
161             case '\\': o[len++] = '\\';s++; break;
162             case '"': o[len++] = '"';s++; break;
163             case 'b': o[len++] = '\b';s++; break;
164             case 'f': o[len++] = '\f';s++; break;
165             case 'n': o[len++] = '\n';s++; break;
166             case 'r': o[len++] = '\r';s++; break;
167             case 't': o[len++] = '\t';s++; break;
168             case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': {
169                 unsigned int num=0;
170                 int nr = 0;
171                 while(strchr("01234567", *s) && nr<3 && s<end) {
172                     num <<= 3;
173                     num |= *s-'0';
174                     nr++;
175                     s++;
176                 }
177                 if(num>256) 
178                     syntaxerror("octal number out of range (0-255): %d", num);
179                 o[len++] = num;
180                 continue;
181             }
182             case 'x': case 'u': {
183                 int max=2;
184                 char bracket = 0;
185                 char unicode = 0;
186                 if(*s == 'u') {
187                     max = 6;
188                     unicode = 1;
189                 }
190                 s++;
191                 if(s==end) syntaxerror("invalid \\u or \\x at end of string");
192                 if(*s == '{')  {
193                     s++;
194                     if(s==end) syntaxerror("invalid \\u{ at end of string");
195                     bracket=1;
196                 }
197                 unsigned int num=0;
198                 int nr = 0;
199                 while(strchr("0123456789abcdefABCDEF", *s) && (bracket || nr < max) && s<end) {
200                     num <<= 4;
201                     if(*s>='0' && *s<='9') num |= *s - '0';
202                     if(*s>='a' && *s<='f') num |= *s - 'a' + 10;
203                     if(*s>='A' && *s<='F') num |= *s - 'A' + 10;
204                     nr++;
205                     s++;
206                 }
207                 if(bracket) {
208                     if(*s=='}' && s<end) {
209                         s++;
210                     } else {
211                         syntaxerror("missing terminating '}'");
212                     }
213                 }
214                 if(unicode) {
215                     char*utf8 = getUTF8(num);
216                     while(*utf8) {
217                         o[len++] = *utf8++;
218                     }
219                 } else {
220                     if(num>256) 
221                         syntaxerror("byte out of range (0-255): %d", num);
222                     o[len++] = num;
223                 }
224                 break;
225             }
226             default:
227                 syntaxerror("unknown escape sequence: \"\\%c\"", *s);
228         }
229     }
230     string_t out = string_new(n, len);
231     o[len]=0;
232     return out; 
233 }
234
235 static void handleString(char*s, int len)
236 {
237     if(s[0]=='"') {
238         if(s[len-1]!='"') syntaxerror("String doesn't end with '\"'");
239         s++;len-=2;
240     }
241     else if(s[0]=='\'') {
242         if(s[len-1]!='\'') syntaxerror("String doesn't end with '\"'");
243         s++;len-=2;
244     }
245     else syntaxerror("String incorrectly terminated");
246
247     
248     avm2_lval.str = string_unescape(s, len);
249 }
250
251
252 char start_of_expression;
253
254 static inline int mkid(int type)
255 {
256     char*s = malloc(yyleng+1);
257     memcpy(s, yytext, yyleng);
258     s[yyleng]=0;
259     avm2_lval.id = s;
260     return type;
261 }
262
263 static inline int m(int type)
264 {
265     avm2_lval.token = type;
266     return type;
267 }
268
269
270 static char numberbuf[64];
271 static inline int handlenumber()
272 {
273     if(yyleng>sizeof(numberbuf)-1)
274         syntaxerror("decimal number overflow");
275
276     char*s = numberbuf;
277     memcpy(s, yytext, yyleng);
278     s[yyleng]=0;
279
280     int t;
281     char is_float=0;
282     for(t=0;t<yyleng;t++) {
283         if(yytext[t]=='.') {
284             if(is_float)
285                 syntaxerror("Invalid number");
286             is_float=1;
287         } else if(!strchr("-0123456789", yytext[t])) {
288             syntaxerror("Invalid number");
289         }
290     }
291     if(is_float) {
292         avm2_lval.number_float = atof(s);
293         return T_FLOAT;
294     } 
295     char l = (yytext[0]=='-');
296
297     char*max = l?"1073741824":"2147483647";
298     if(yyleng-l>10)
299         syntaxerror("integer overflow");
300     if(yyleng-l==10) {
301         int t;
302         for(t=0;t<yyleng-l;t++) {
303             if(yytext[l+t]>max[t])
304                 syntaxerror("integer overflow %s > %s", s+l,max);
305             else if(yytext[l+t]<max[t])
306                 break;
307         }
308     }
309     if(yytext[0]=='-') {
310         int v = atoi(s);
311         avm2_lval.number_int = v;
312         if(v>-128)
313             return T_BYTE;
314         else if(v>=-32768)
315             return T_SHORT;
316         else
317             return T_INT;
318     } else {
319         unsigned int v = 0;
320         for(t=0;t<yyleng;t++) {
321             v*=10;
322             v+=yytext[t]-'0';
323         }
324         avm2_lval.number_uint = v;
325         if(v<128)
326             return T_BYTE;
327         else if(v<32768)
328             return T_SHORT;
329         else
330             return T_UINT;
331     }
332 }
333
334 void initialize_scanner();
335 #define YY_USER_INIT initialize_scanner();
336
337 #define c() {countlines(yytext, yyleng);}
338
339 %}
340
341 %s REGEXPOK
342 %s BEGINNING
343
344 NAME     [a-zA-Z_][a-zA-Z0-9_\\]*
345
346 NUMBER   -?[0-9]+(\.[0-9]*)?|-?\.[0-9]+
347
348 STRING   ["](\\[\x00-\xff]|[^\\"\n])*["]|['](\\[\x00-\xff]|[^\\'\n])*[']
349 S        [ \n\r\t]
350 MULTILINE_COMMENT [/][*]+([*][^/]|[^/*]|[^*][/]|[\x00-\x1f])*[*]+[/]
351 SINGLELINE_COMMENT \/\/[^\n]*\n
352 REGEXP   [/]([^/\n]|\\[/])*[/][a-zA-Z]*
353 %%
354
355
356 {SINGLELINE_COMMENT}         {c(); /* single line comment */}
357 {MULTILINE_COMMENT}          {c(); /* multi line comment */}
358 [/][*]                       {syntaxerror("syntax error: unterminated comment", yytext);}
359
360 ^include{S}+{STRING}{S}*/\n    {c();handleInclude(yytext, yyleng, 1);}
361 ^include{S}+[^" \t\r\n][\x20-\xff]*{S}*/\n    {c();handleInclude(yytext, yyleng, 0);}
362 {STRING}                     {c(); BEGIN(INITIAL);handleString(yytext, yyleng);return T_STRING;}
363
364 <BEGINNING,REGEXPOK>{
365 {REGEXP}                     {c(); BEGIN(INITIAL);return m(T_REGEXP);} 
366 }
367
368 \xef\xbb\xbf                 {/* utf 8 bom */}
369 {S}                          {c();}
370
371 {NUMBER}                     {c(); BEGIN(INITIAL);return handlenumber();}
372
373 3rr0r                        {/* for debugging: generates a tokenizer-level error */
374                               syntaxerror("3rr0r");}
375
376 [&][&]                       {c();BEGIN(REGEXPOK);return m(T_ANDAND);}
377 [|][|]                       {c();BEGIN(REGEXPOK);return m(T_OROR);}
378 [!][=]                       {c();BEGIN(REGEXPOK);return m(T_NE);}
379 [=][=][=]                    {c();BEGIN(REGEXPOK);return m(T_EQEQEQ);}
380 [=][=]                       {c();BEGIN(REGEXPOK);return m(T_EQEQ);}
381 [>][=]                       {c();return m(T_GE);}
382 [<][=]                       {c();return m(T_LE);}
383 [-][-]                       {c();BEGIN(INITIAL);return m(T_MINUSMINUS);}
384 [+][+]                       {c();BEGIN(INITIAL);return m(T_PLUSPLUS);}
385 [+][=]                       {c();return m(T_PLUSBY);}
386 [-][=]                       {c();return m(T_MINUSBY);}
387 [/][=]                       {c();return m(T_DIVBY);}
388 [%][=]                       {c();return m(T_MODBY);}
389 [*][=]                       {c();return m(T_MULBY);}
390 [>][>][=]                    {c();return m(T_SHRBY);}
391 [<][<][=]                    {c();return m(T_SHLBY);}
392 [>][>][>][=]                 {c();return m(T_USHRBY);}
393 [<][<]                       {c();return m(T_SHL);}
394 [>][>][>]                    {c();return m(T_USHR);}
395 [>][>]                       {c();return m(T_SHR);}
396 \.\.\.                       {c();return m(T_DOTDOTDOT);}
397 \.\.                         {c();return m(T_DOTDOT);}
398 \.                           {c();return m('.');}
399 ::                           {c();return m(T_COLONCOLON);}
400 :                            {c();return m(':');}
401 implements                   {c();return m(KW_IMPLEMENTS);}
402 interface                    {c();return m(KW_INTERFACE);}
403 namespace                    {c();return m(KW_NAMESPACE);}
404 protected                    {c();return m(KW_PROTECTED);}
405 override                     {c();return m(KW_OVERRIDE);}
406 internal                     {c();return m(KW_INTERNAL);}
407 function                     {c();return m(KW_FUNCTION);}
408 package                      {c();return m(KW_PACKAGE);}
409 private                      {c();return m(KW_PRIVATE);}
410 Boolean                      {c();return m(KW_BOOLEAN);}
411 dynamic                      {c();return m(KW_DYNAMIC);}
412 extends                      {c();return m(KW_EXTENDS);}
413 return                       {c();return m(KW_RETURN);}
414 public                       {c();return m(KW_PUBLIC);}
415 native                       {c();return m(KW_NATIVE);}
416 static                       {c();return m(KW_STATIC);}
417 import                       {c();return m(KW_IMPORT);}
418 Number                       {c();return m(KW_NUMBER);}
419 while                        {c();return m(KW_WHILE);}
420 class                        {c();return m(KW_CLASS);}
421 const                        {c();return m(KW_CONST);}
422 final                        {c();return m(KW_FINAL);}
423 false                        {c();return m(KW_FALSE);}
424 break                        {c();return m(KW_BREAK);}
425 true                         {c();return m(KW_TRUE);}
426 uint                         {c();return m(KW_UINT);}
427 null                         {c();return m(KW_NULL);}
428 else                         {c();return m(KW_ELSE);}
429 use                          {c();return m(KW_USE);}
430 int                          {c();return m(KW_INT);}
431 new                          {c();return m(KW_NEW);}
432 get                          {c();return m(KW_GET);}
433 for                          {c();return m(KW_FOR);}
434 set                          {c();return m(KW_SET);}
435 var                          {c();return m(KW_VAR);}
436 is                           {c();return m(KW_IS) ;}
437 if                           {c();return m(KW_IF) ;}
438 as                           {c();return m(KW_AS);}
439 {NAME}                       {c();BEGIN(INITIAL);return mkid(T_IDENTIFIER);}
440
441 [+-\/*^~@$!%&\(=\[\]\{\}|?:;,<>] {c();BEGIN(REGEXPOK);return m(yytext[0]);}
442 [\)\]]                            {c();BEGIN(INITIAL);return m(yytext[0]);}
443
444 .                            {char c1=yytext[0];
445                               char buf[128];
446                               buf[0] = yytext[0];
447                               int t;
448                               for(t=1;t<128;t++) {
449                                   char c = buf[t]=input();
450                                   if(c=='\n' || c==EOF)  {
451                                       buf[t] = 0;
452                                       break;
453                                   }
454                               }
455                               if(c1>='0' && c1<='9')
456                                   syntaxerror("syntax error: %s (identifiers must not start with a digit)");
457                               else
458                                   syntaxerror("syntax error: %s", buf);
459                               printf("\n");
460                               exit(1);
461                               yyterminate();
462                              }
463 <<EOF>>                      {c();
464                               void*b = leave_file();
465                               if (!b) {
466                                  yyterminate();
467                                  yy_delete_buffer(YY_CURRENT_BUFFER);
468                                  return m(T_EOF);
469                               } else {
470                                   yy_delete_buffer(YY_CURRENT_BUFFER);
471                                   yy_switch_to_buffer(b);
472                               }
473                              }
474
475 %%
476
477 int yywrap()
478 {
479     return 1;
480 }
481
482 static char mbuf[256];
483 char*token2string(enum yytokentype nr, YYSTYPE v)
484 {
485     if(nr==T_STRING)     return "<string>";
486     else if(nr==T_INT)     return "<int>";
487     else if(nr==T_UINT)     return "<uint>";
488     else if(nr==T_FLOAT)     return "<float>";
489     else if(nr==T_REGEXP)     return "REGEXP";
490     else if(nr==T_EOF)        return "***END***";
491     else if(nr==T_GE)         return ">=";
492     else if(nr==T_LE)         return "<=";
493     else if(nr==T_MINUSMINUS) return "--";
494     else if(nr==T_PLUSPLUS)   return "++";
495     else if(nr==KW_IMPLEMENTS) return "implements";
496     else if(nr==KW_INTERFACE)  return "interface";
497     else if(nr==KW_NAMESPACE)  return "namespace";
498     else if(nr==KW_PROTECTED)  return "protected";
499     else if(nr==KW_OVERRIDE)   return "override";
500     else if(nr==KW_INTERNAL)   return "internal";
501     else if(nr==KW_FUNCTION)   return "function";
502     else if(nr==KW_PACKAGE)    return "package";
503     else if(nr==KW_PRIVATE)    return "private";
504     else if(nr==KW_BOOLEAN)    return "Boolean";
505     else if(nr==KW_DYNAMIC)    return "dynamic";
506     else if(nr==KW_EXTENDS)    return "extends";
507     else if(nr==KW_PUBLIC)     return "public";
508     else if(nr==KW_NATIVE)     return "native";
509     else if(nr==KW_STATIC)     return "static";
510     else if(nr==KW_IMPORT)     return "import";
511     else if(nr==KW_NUMBER)     return "number";
512     else if(nr==KW_CLASS)      return "class";
513     else if(nr==KW_CONST)      return "const";
514     else if(nr==KW_FINAL)      return "final";
515     else if(nr==KW_FALSE)      return "False";
516     else if(nr==KW_TRUE)       return "True";
517     else if(nr==KW_UINT)       return "uint";
518     else if(nr==KW_NULL)       return "null";
519     else if(nr==KW_ELSE)       return "else";
520     else if(nr==KW_USE)        return "use";
521     else if(nr==KW_INT)        return "int";
522     else if(nr==KW_NEW)        return "new";
523     else if(nr==KW_GET)        return "get";
524     else if(nr==KW_FOR)        return "for";
525     else if(nr==KW_SET)        return "set";
526     else if(nr==KW_VAR)        return "var";
527     else if(nr==KW_IS)         return "is";
528     else if(nr==KW_AS)         return "as";
529     else if(nr==T_IDENTIFIER)  return "ID";
530     else {
531         sprintf(mbuf, "%d", nr);
532         return mbuf;
533     }
534 }
535
536 void initialize_scanner()
537 {
538     BEGIN(BEGINNING);
539 }
540