upgraded to xpdf-3.01pl1

[swftools.git] / pdf2swf / xpdf / Decrypt.cc

diff --git a/pdf2swf/xpdf/Decrypt.cc b/pdf2swf/xpdf/Decrypt.cc
index 8de4091..91eb452 100644 (file)
--- a/pdf2swf/xpdf/Decrypt.cc
+++ b/pdf2swf/xpdf/Decrypt.cc
@@ -2,15 +2,17 @@
 //
 // Decrypt.cc
 //
-// Copyright 1996-2002 Glyph & Cog, LLC
+// Copyright 1996-2003 Glyph & Cog, LLC
 //
 //========================================================================
 
-#ifdef __GNUC__
+#include <aconf.h>
+
+#ifdef USE_GCC_PRAGMAS
 #pragma implementation
 #endif
 
-#include <aconf.h>
+#include <string.h>
 #include "gmem.h"
 #include "Decrypt.h"
 
@@ -64,14 +66,17 @@ GBool Decrypt::makeFileKey(int encVersion, int encRevision, int keyLength,
                           GString *ownerKey, GString *userKey,
                           int permissions, GString *fileID,
                           GString *ownerPassword, GString *userPassword,
-                          Guchar *fileKey, GBool *ownerPasswordOk) {
-  Guchar test[32];
+                          Guchar *fileKey, GBool encryptMetadata,
+                          GBool *ownerPasswordOk) {
+  Guchar test[32], test2[32];
   GString *userPassword2;
   Guchar fState[256];
+  Guchar tmpKey[16];
   Guchar fx, fy;
-  int len, i;
+  int len, i, j;
 
   // try using the supplied owner password to generate the user password
+  *ownerPasswordOk = gFalse;
   if (ownerPassword) {
     len = ownerPassword->getLength();
     if (len < 32) {
@@ -80,39 +85,53 @@ GBool Decrypt::makeFileKey(int encVersion, int encRevision, int keyLength,
     } else {
       memcpy(test, ownerPassword->getCString(), 32);
     }
-  } else {
-    memcpy(test, passwordPad, 32);
-  }
-  md5(test, 32, test);
-  if (encRevision == 3) {
-    for (i = 0; i < 50; ++i) {
-      md5(test, 16, test);
+    md5(test, 32, test);
+    if (encRevision == 3) {
+      for (i = 0; i < 50; ++i) {
+       md5(test, 16, test);
+      }
+    }
+    if (encRevision == 2) {
+      rc4InitKey(test, keyLength, fState);
+      fx = fy = 0;
+      for (i = 0; i < 32; ++i) {
+       test2[i] = rc4DecryptByte(fState, &fx, &fy, ownerKey->getChar(i));
+      }
+    } else {
+      memcpy(test2, ownerKey->getCString(), 32);
+      for (i = 19; i >= 0; --i) {
+       for (j = 0; j < keyLength; ++j) {
+         tmpKey[j] = test[j] ^ i;
+       }
+       rc4InitKey(tmpKey, keyLength, fState);
+       fx = fy = 0;
+       for (j = 0; j < 32; ++j) {
+         test2[j] = rc4DecryptByte(fState, &fx, &fy, test2[j]);
+       }
+      }
+    }
+    userPassword2 = new GString((char *)test2, 32);
+    if (makeFileKey2(encVersion, encRevision, keyLength, ownerKey, userKey,
+                    permissions, fileID, userPassword2, fileKey,
+                    encryptMetadata)) {
+      *ownerPasswordOk = gTrue;
+      delete userPassword2;
+      return gTrue;
     }
-  }
-  rc4InitKey(test, keyLength, fState);
-  fx = fy = 0;
-  for (i = 0; i < 32; ++i) {
-    test[i] = rc4DecryptByte(fState, &fx, &fy, ownerKey->getChar(i));
-  }
-  userPassword2 = new GString((char *)test, 32);
-  if (makeFileKey2(encVersion, encRevision, keyLength, ownerKey, userKey,
-                  permissions, fileID, userPassword2, fileKey)) {
-    *ownerPasswordOk = gTrue;
     delete userPassword2;
-    return gTrue;
   }
-  *ownerPasswordOk = gFalse;
-  delete userPassword2;
 
   // try using the supplied user password
   return makeFileKey2(encVersion, encRevision, keyLength, ownerKey, userKey,
-                     permissions, fileID, userPassword, fileKey);
+                     permissions, fileID, userPassword, fileKey,
+                     encryptMetadata);
 }
 
 GBool Decrypt::makeFileKey2(int encVersion, int encRevision, int keyLength,
                            GString *ownerKey, GString *userKey,
                            int permissions, GString *fileID,
-                           GString *userPassword, Guchar *fileKey) {
+                           GString *userPassword, Guchar *fileKey,
+                           GBool encryptMetadata) {
   Guchar *buf;
   Guchar test[32];
   Guchar fState[256];
@@ -122,7 +141,7 @@ GBool Decrypt::makeFileKey2(int encVersion, int encRevision, int keyLength,
   GBool ok;
 
   // generate file key
-  buf = (Guchar *)gmalloc(68 + fileID->getLength());
+  buf = (Guchar *)gmalloc(72 + fileID->getLength());
   if (userPassword) {
     len = userPassword->getLength();
     if (len < 32) {
@@ -140,10 +159,17 @@ GBool Decrypt::makeFileKey2(int encVersion, int encRevision, int keyLength,
   buf[66] = (permissions >> 16) & 0xff;
   buf[67] = (permissions >> 24) & 0xff;
   memcpy(buf + 68, fileID->getCString(), fileID->getLength());
-  md5(buf, 68 + fileID->getLength(), fileKey);
+  len = 68 + fileID->getLength();
+  if (!encryptMetadata) {
+    buf[len++] = 0xff;
+    buf[len++] = 0xff;
+    buf[len++] = 0xff;
+    buf[len++] = 0xff;
+  }
+  md5(buf, len, fileKey);
   if (encRevision == 3) {
     for (i = 0; i < 50; ++i) {
-      md5(fileKey, 16, fileKey);
+      md5(fileKey, keyLength, fileKey);
     }
   }
 
@@ -366,20 +392,20 @@ static void md5(Guchar *msg, int msgLen, Guchar *digest) {
   }
 
   // break digest into bytes
-  digest[0] = a & 0xff;
-  digest[1] = (a >>= 8) & 0xff;
-  digest[2] = (a >>= 8) & 0xff;
-  digest[3] = (a >>= 8) & 0xff;
-  digest[4] = b & 0xff;
-  digest[5] = (b >>= 8) & 0xff;
-  digest[6] = (b >>= 8) & 0xff;
-  digest[7] = (b >>= 8) & 0xff;
-  digest[8] = c & 0xff;
-  digest[9] = (c >>= 8) & 0xff;
-  digest[10] = (c >>= 8) & 0xff;
-  digest[11] = (c >>= 8) & 0xff;
-  digest[12] = d & 0xff;
-  digest[13] = (d >>= 8) & 0xff;
-  digest[14] = (d >>= 8) & 0xff;
-  digest[15] = (d >>= 8) & 0xff;
+  digest[0] = (Guchar)(a & 0xff);
+  digest[1] = (Guchar)((a >>= 8) & 0xff);
+  digest[2] = (Guchar)((a >>= 8) & 0xff);
+  digest[3] = (Guchar)((a >>= 8) & 0xff);
+  digest[4] = (Guchar)(b & 0xff);
+  digest[5] = (Guchar)((b >>= 8) & 0xff);
+  digest[6] = (Guchar)((b >>= 8) & 0xff);
+  digest[7] = (Guchar)((b >>= 8) & 0xff);
+  digest[8] = (Guchar)(c & 0xff);
+  digest[9] = (Guchar)((c >>= 8) & 0xff);
+  digest[10] = (Guchar)((c >>= 8) & 0xff);
+  digest[11] = (Guchar)((c >>= 8) & 0xff);
+  digest[12] = (Guchar)(d & 0xff);
+  digest[13] = (Guchar)((d >>= 8) & 0xff);
+  digest[14] = (Guchar)((d >>= 8) & 0xff);
+  digest[15] = (Guchar)((d >>= 8) & 0xff);
 }