From: Matthias Kramm <kramm@quiss.org>
Date: Fri, 31 Jul 2009 18:22:39 +0000 (+0200)
Subject: fixed a nasty floating point bug
X-Git-Tag: version-0-9-1~328
X-Git-Url: http://git.asbjorn.biz/?p=swftools.git;a=commitdiff_plain;h=a8a2650b187181ff507303ff9cf181427445255a

fixed a nasty floating point bug
---

diff --git a/lib/gfxpoly/convert.c b/lib/gfxpoly/convert.c
index 9d8ecab..dae58b6 100644
--- a/lib/gfxpoly/convert.c
+++ b/lib/gfxpoly/convert.c
@@ -12,12 +12,13 @@
 
 static inline int32_t convert_coord(double x, double z)
 {
-    /* we clamp to 31 bit instead of 32 bit because we use
-       a (x1-x2) shortcut when comparing coordinates
+    /* we clamp to 26 bit because: 
+       a) we use a (x1-x2) shortcut when comparing coordinates
+       b) we need to be able to multiply two coordinates and store them in a double w/o loss of precision
     */
     x *= z;
-    if(x < -0x40000000) x = -0x40000000;
-    if(x >  0x3fffffff) x =  0x3fffffff;
+    if(x < -0x2000000) x = -0x2000000;
+    if(x >  0x1ffffff) x =  0x1ffffff;
     return ceil(x);
 }
 
diff --git a/lib/gfxpoly/poly.c b/lib/gfxpoly/poly.c
index 9f7e8e8..2be5efb 100644
--- a/lib/gfxpoly/poly.c
+++ b/lib/gfxpoly/poly.c
@@ -305,6 +305,11 @@ static void segment_init(segment_t*s, int32_t x1, int32_t y1, int32_t x2, int32_
     s->nr = segment_count++;
 
 #ifdef CHECKS
+    /* notice: on some systems (with some compilers), for the line 
+       (1073741823,-1073741824)->(1073741823,1073741823)
+       we get LINE_EQ(s->a, s) == 1. 
+       That's why we now clamp to 26 bit.
+    */
     assert(LINE_EQ(s->a, s) == 0);
     assert(LINE_EQ(s->b, s) == 0);