Document BN_mod_mul_montgomery bug;

make disabled code slightly more correct (this does not solve
the problem though).

CHANGES

@@ -4,6 +4,10 @@
 
  Changes between 0.9.5a and 0.9.6  [xx XXX 2000]
 
+  *) Disable optimized squaring variant in BN_mod_mul_montgomery,
+     it can return incorrect results.
+     [Bodo Moeller]
+
   *) Disable the check for content being present when verifying detached
      signatures in pk7_smime.c. Some versions of Netscape (wrongly)
      include zero length content when signing messages.

crypto/bn/bn_mont.c

@@ -85,7 +85,8 @@ int BN_mod_mul_montgomery(BIGNUM *r, BIGNUM *a, BIGNUM *b,
 
 	if (a == b)
 		{
-#if 0 /* buggy -- try squaring  g  in the following parameters
+#if 0 /* buggy -- try squaring  g  (after converting it to Montgomery
+         representation) in the following parameters
          (but note that squaring 2 or 4 works):
 Diffie-Hellman-Parameters: (1024 bit)
     prime:
@@ -109,7 +110,7 @@ Sgh5jjQE3e+VGbPNOkMbMCsKbfJfFDdP4TVtbVHCReSFtXZiXn7G9ExC6aY37WsL
 		bn_wexpand(tmp2,a->top*4);
 		bn_sqr_recursive(tmp->d,a->d,a->top,tmp2->d);
 		tmp->top=a->top*2;
-		if (tmp->top > 0 && tmp->d[tmp->top-1] == 0)
+		while (tmp->top > 0 && tmp->d[tmp->top-1] == 0)
 			tmp->top--;
 #else
 		if (!BN_sqr(tmp,a,ctx)) goto err;