Add support for the modified SGC key format used in IIS.

CHANGES

@@ -4,6 +4,18 @@
 
  Changes between 0.9.5a and 0.9.6  [xx XXX 2000]
 
+  *) When some versions of IIS use the 'NET' form of private key the
+     key derivation algorithm is different. Normally MD5(password) is
+     used as a 128 bit RC4 key. In the modified case
+     MD5(MD5(password) + "SGCKEYSALT")  is used insted. Added some
+     new functions i2d_RSA_NET(), d2i_RSA_NET() etc which are the same
+     as the old Netscape_RSA functions except they have an additional
+     'sgckey' parameter which uses the modified algorithm. Also added
+     an -sgckey command line option to the rsa utility. Thanks to 
+     Adrian Peck <bertie@ncipher.com> for posting details of the modified
+     algorithm to openssl-dev.
+     [Steve Henson]
+
   *) The evp_local.h macros were using 'c.##kname' which resulted in
      invalid expansion on some systems (SCO 5.0.5 for example).
      Corrected to 'c.kname'.

apps/rsa.c

@@ -92,7 +92,7 @@ int MAIN(int argc, char **argv)
 	{
 	int ret=1;
 	RSA *rsa=NULL;
-	int i,badops=0;
+	int i,badops=0, sgckey=0;
 	const EVP_CIPHER *enc=NULL;
 	BIO *in=NULL,*out=NULL;
 	int informat,outformat,text=0,check=0,noout=0;
@@ -148,6 +148,8 @@ int MAIN(int argc, char **argv)
 			if (--argc < 1) goto bad;
 			passargout= *(++argv);
 			}
+		else if (strcmp(*argv,"-sgckey") == 0)
+			sgckey=1;
 		else if (strcmp(*argv,"-pubin") == 0)
 			pubin=1;
 		else if (strcmp(*argv,"-pubout") == 0)
@@ -178,6 +180,7 @@ bad:
 		BIO_printf(bio_err," -inform arg     input format - one of DER NET PEM\n");
 		BIO_printf(bio_err," -outform arg    output format - one of DER NET PEM\n");
 		BIO_printf(bio_err," -in arg         input file\n");
+		BIO_printf(bio_err," -sgckey         Use IIS SGC key format\n");
 		BIO_printf(bio_err," -passin arg     input file pass phrase source\n");
 		BIO_printf(bio_err," -out arg        output file\n");
 		BIO_printf(bio_err," -passout arg    output file pass phrase source\n");
@@ -254,7 +257,7 @@ bad:
 				}
 			}
 		p=(unsigned char *)buf->data;
-		rsa=d2i_Netscape_RSA(NULL,&p,(long)size,NULL);
+		rsa=d2i_RSA_NET(NULL,&p,(long)size,NULL, sgckey);
 		BUF_MEM_free(buf);
 		}
 #endif
@@ -344,14 +347,14 @@ bad:
 		int size;
 
 		i=1;
-		size=i2d_Netscape_RSA(rsa,NULL,NULL);
+		size=i2d_RSA_NET(rsa,NULL,NULL, sgckey);
 		if ((p=(unsigned char *)OPENSSL_malloc(size)) == NULL)
 			{
 			BIO_printf(bio_err,"Memory allocation failure\n");
 			goto end;
 			}
 		pp=p;
-		i2d_Netscape_RSA(rsa,&p,NULL);
+		i2d_RSA_NET(rsa,&p,NULL, sgckey);
 		BIO_write(out,(char *)pp,size);
 		OPENSSL_free(pp);
 		}

crypto/asn1/n_pkey.c

@@ -81,6 +81,11 @@ static NETSCAPE_PKEY *NETSCAPE_PKEY_new(void);
 static void NETSCAPE_PKEY_free(NETSCAPE_PKEY *);
 
 int i2d_Netscape_RSA(RSA *a, unsigned char **pp, int (*cb)())
+{
+	return i2d_RSA_NET(a, pp, cb, 0);
+}
+
+int i2d_RSA_NET(RSA *a, unsigned char **pp, int (*cb)(), int sgckey)
 	{
 	int i,j,l[6];
 	NETSCAPE_PKEY *pkey;
@@ -164,8 +169,18 @@ int i2d_Netscape_RSA(RSA *a, unsigned char **pp, int (*cb)())
 		ASN1err(ASN1_F_I2D_NETSCAPE_RSA,ASN1_R_BAD_PASSWORD_READ);
 		goto err;
 		}
-	EVP_BytesToKey(EVP_rc4(),EVP_md5(),NULL,buf,
-		strlen((char *)buf),1,key,NULL);
+	i = strlen((char *)buf);
+	/* If the key is used for SGC the algorithm is modified a little. */
+	if(sgckey){
+		EVP_MD_CTX mctx;
+		EVP_DigestInit(&mctx, EVP_md5());
+		EVP_DigestUpdate(&mctx, buf, i);
+		EVP_DigestFinal(&mctx, buf, NULL);
+		memcpy(buf + 16, "SGCKEYSALT", 10);
+		i = 26;
+	}
+		
+	EVP_BytesToKey(EVP_rc4(),EVP_md5(),NULL,buf,i,1,key,NULL);
 	memset(buf,0,256);
 
 	EVP_CIPHER_CTX_init(&ctx);
@@ -189,7 +204,13 @@ err:
 	return(ret);
 	}
 
+
 RSA *d2i_Netscape_RSA(RSA **a, unsigned char **pp, long length, int (*cb)())
+{
+	return d2i_RSA_NET(a, pp, length, cb, 0);
+}
+
+RSA *d2i_RSA_NET(RSA **a, unsigned char **pp, long length, int (*cb)(), int sgckey)
 	{
 	RSA *ret=NULL;
 	ASN1_OCTET_STRING *os=NULL;
@@ -210,7 +231,7 @@ RSA *d2i_Netscape_RSA(RSA **a, unsigned char **pp, long length, int (*cb)())
 		}
 	M_ASN1_BIT_STRING_free(os);
 	c.q=c.p;
-	if ((ret=d2i_Netscape_RSA_2(a,&c.p,c.slen,cb)) == NULL) goto err;
+	if ((ret=d2i_RSA_NET_2(a,&c.p,c.slen,cb, sgckey)) == NULL) goto err;
 	/* Note: some versions of IIS key files use length values that are
 	 * too small for the surrounding SEQUENCEs. This following line
 	 * effectively disable length checking.
@@ -222,6 +243,12 @@ RSA *d2i_Netscape_RSA(RSA **a, unsigned char **pp, long length, int (*cb)())
 
 RSA *d2i_Netscape_RSA_2(RSA **a, unsigned char **pp, long length,
 	     int (*cb)())
+{
+	return d2i_RSA_NET_2(a, pp, length, cb, 0);
+}
+
+RSA *d2i_RSA_NET_2(RSA **a, unsigned char **pp, long length,
+	     int (*cb)(), int sgckey)
 	{
 	NETSCAPE_PKEY *pkey=NULL;
 	RSA *ret=NULL;
@@ -254,8 +281,17 @@ RSA *d2i_Netscape_RSA_2(RSA **a, unsigned char **pp, long length,
 		goto err;
 		}
 
-	EVP_BytesToKey(EVP_rc4(),EVP_md5(),NULL,buf,
-		strlen((char *)buf),1,key,NULL);
+	i = strlen((char *)buf);
+	if(sgckey){
+		EVP_MD_CTX mctx;
+		EVP_DigestInit(&mctx, EVP_md5());
+		EVP_DigestUpdate(&mctx, buf, i);
+		EVP_DigestFinal(&mctx, buf, NULL);
+		memcpy(buf + 16, "SGCKEYSALT", 10);
+		i = 26;
+	}
+		
+	EVP_BytesToKey(EVP_rc4(),EVP_md5(),NULL,buf,i,1,key,NULL);
 	memset(buf,0,256);
 
 	EVP_CIPHER_CTX_init(&ctx);

crypto/rsa/rsa.h

@@ -216,6 +216,10 @@ int	RSA_print_fp(FILE *fp, RSA *r,int offset);
 int	RSA_print(BIO *bp, RSA *r,int offset);
 #endif
 
+int i2d_RSA_NET(RSA *a, unsigned char **pp, int (*cb)(), int sgckey);
+RSA *d2i_RSA_NET(RSA **a, unsigned char **pp, long length, int (*cb)(), int sgckey);
+RSA *d2i_RSA_NET_2(RSA **a, unsigned char **pp, long length, int (*cb)(), int sgckey);
+
 int i2d_Netscape_RSA(RSA *a, unsigned char **pp, int (*cb)());
 RSA *d2i_Netscape_RSA(RSA **a, unsigned char **pp, long length, int (*cb)());
 /* Naughty internal function required elsewhere, to handle a MS structure

doc/apps/rsa.pod

@@ -14,6 +14,7 @@ B<openssl> B<rsa>
 [B<-passin arg>]
 [B<-out filename>]
 [B<-passout arg>]
+[B<-sgckey>]
 [B<-des>]
 [B<-des3>]
 [B<-idea>]
@@ -42,9 +43,8 @@ This specifies the input format. The B<DER> option uses an ASN1 DER encoded
 form compatible with the PKCS#1 RSAPrivateKey or SubjectPublicKeyInfo format.
 The B<PEM> form is the default format: it consists of the B<DER> format base64
 encoded with additional header and footer lines. On input PKCS#8 format private
-keys are also accepted. The B<NET> form is a format compatible with older Netscape
-servers and MS IIS, this uses unsalted RC4 for its encryption. It is not very
-secure and so should only be used when necessary.
+keys are also accepted. The B<NET> form is a format is described in the B<NOTES>
+section.
 
 =item B<-outform DER|NET|PEM>
 
@@ -74,6 +74,11 @@ filename.
 the output file password source. For more information about the format of B<arg>
 see the B<PASS PHRASE ARGUMENTS> section in L<openssl(1)|openssl(1)>.
 
+=item B<-sgckey>
+
+use the modified NET algorithm used with some versions of Microsoft IIS and SGC
+keys.
+
 =item B<-des|-des3|-idea>
 
 These options encrypt the private key with the DES, triple DES, or the 
@@ -126,6 +131,18 @@ The PEM public key format uses the header and footer lines:
  -----BEGIN PUBLIC KEY-----
  -----END PUBLIC KEY-----
 
+The B<NET> form is a format compatible with older Netscape servers
+and Microsoft IIS .key files, this uses unsalted RC4 for its encryption.
+It is not very secure and so should only be used when necessary.
+
+Some newer version of IIS have additional data in the exported .key
+files. To use thse with the utility view the file with a binary editor
+and look for the string "private-key", then trace back to the byte
+sequence 0x30, 0x82 (this is an ASN1 SEQUENCE). Copy all the data
+from this point onwards to another file and use that as the input
+to the B<rsa> utility with the B<-inform NET> option. If you get
+an error after entering the password try the B<-sgckey> option.
+
 =head1 EXAMPLES
 
 To remove the pass phrase on an RSA private key:
@@ -148,6 +165,14 @@ To just output the public part of a private key:
 
  openssl rsa -in key.pem -pubout -out pubkey.pem
 
+=head1 BUGS
+
+The command line password arguments don't currently work with
+B<NET> format.
+
+There should be an option that automatically handles .key files,
+without having to manually edit them.
+
 =head1 SEE ALSO
 
 L<pkcs8(1)|pkcs8(1)>, L<dsa(1)|dsa(1)>, L<genrsa(1)|genrsa(1)>,

util/libeay.num

@@ -1806,3 +1806,6 @@ RAND_egd_bytes                          2402
 X509_REQ_get1_email                     2403
 X509_get1_email                         2404
 X509_email_free                         2405
+i2d_RSA_NET                             2406
+d2i_RSA_NET_2                           2407
+d2i_RSA_NET                             2408