Add a bunch of SSL_xxx() functions for configuring the temporary RSA and DH

private keys and/or callback functions which directly correspond to their
SSL_CTX_xxx() counterparts but work on a per-connection basis. This is needed
for applications which have to configure certificates on a per-connection
basis (e.g. Apache+mod_ssl) instead of a per-context basis (e.g.
s_server).

For the RSA certificate situation is makes no difference, but for the DSA
certificate situation this fixes the "no shared cipher" problem where the
OpenSSL cipher selection procedure failed because the temporary keys were not
overtaken from the context and the API provided no way to reconfigure them.

The new functions now let applications reconfigure the stuff and they are in
detail: SSL_need_tmp_RSA, SSL_set_tmp_rsa, SSL_set_tmp_dh,
SSL_set_tmp_rsa_callback and SSL_set_tmp_dh_callback.  Additionally a new
non-public-API function ssl_cert_instantiate() is used as a helper function
and also to reduce code redundancy inside ssl_rsa.c.

Submitted by: Ralf S. Engelschall
Reviewed by: Ben Laurie

CHANGES

@@ -5,6 +5,24 @@
 
  Changes between 0.9.1c and 0.9.2
 
+  *) Add a bunch of SSL_xxx() functions for configuring the temporary RSA and
+     DH private keys and/or callback functions which directly correspond to
+     their SSL_CTX_xxx() counterparts but work on a per-connection basis. This
+     is needed for applications which have to configure certificates on a
+     per-connection basis (e.g. Apache+mod_ssl) instead of a per-context basis
+     (e.g. s_server). 
+        For the RSA certificate situation is makes no difference, but
+     for the DSA certificate situation this fixes the "no shared cipher"
+     problem where the OpenSSL cipher selection procedure failed because the
+     temporary keys were not overtaken from the context and the API provided
+     no way to reconfigure them. 
+        The new functions now let applications reconfigure the stuff and they
+     are in detail: SSL_need_tmp_RSA, SSL_set_tmp_rsa, SSL_set_tmp_dh,
+     SSL_set_tmp_rsa_callback and SSL_set_tmp_dh_callback.  Additionally a new
+     non-public-API function ssl_cert_instantiate() is used as a helper
+     function and also to reduce code redundancy inside ssl_rsa.c.
+     [Ralf S. Engelschall]
+
   *) Move s_server -dcert and -dkey options out of the undocumented feature
      area because they are useful for the DSA situation and should be
      recognized by the users.

ssl/s3_lib.c

@@ -546,6 +546,26 @@ char *parg;
 	{
 	int ret=0;
 
+#if !defined(NO_DSA) || !defined(NO_RSA)
+	if (
+#ifndef NO_RSA
+	    cmd == SSL_CTRL_SET_TMP_RSA ||
+	    cmd == SSL_CTRL_SET_TMP_RSA_CB ||
+#endif
+#ifndef NO_DSA
+	    cmd == SSL_CTRL_SET_TMP_DH ||
+	    cmd == SSL_CTRL_SET_TMP_DH_CB ||
+#endif
+		0)
+		{
+		if (!ssl_cert_instantiate(&s->cert, s->ctx->default_cert)) 
+		    	{
+			SSLerr(SSL_F_SSL3_CTRL, ERR_R_MALLOC_FAILURE);
+			return(0);
+			}
+		}
+#endif
+
 	switch (cmd)
 		{
 	case SSL_CTRL_GET_SESSION_REUSED:
@@ -566,6 +586,69 @@ char *parg;
 	case SSL_CTRL_GET_FLAGS:
 		ret=(int)(s->s3->flags);
 		break;
+#ifndef NO_RSA
+	case SSL_CTRL_NEED_TMP_RSA:
+		if ((s->cert != NULL) && (s->cert->rsa_tmp == NULL) &&
+		    ((s->cert->pkeys[SSL_PKEY_RSA_ENC].privatekey == NULL) ||
+		     (EVP_PKEY_size(s->cert->pkeys[SSL_PKEY_RSA_ENC].privatekey) > (512/8))))
+			ret = 1;
+		break;
+	case SSL_CTRL_SET_TMP_RSA:
+		{
+			RSA *rsa = (RSA *)parg;
+			if (rsa == NULL) {
+				SSLerr(SSL_F_SSL3_CTRL, ERR_R_PASSED_NULL_PARAMETER);
+				return(ret);
+			}
+			if ((rsa = RSAPrivateKey_dup(rsa)) == NULL) {
+				SSLerr(SSL_F_SSL3_CTRL, ERR_R_RSA_LIB);
+				return(ret);
+			}
+			if (s->cert->rsa_tmp != NULL)
+				RSA_free(s->cert->rsa_tmp);
+			s->cert->rsa_tmp = rsa;
+			ret = 1;
+		}
+		break;
+	case SSL_CTRL_SET_TMP_RSA_CB:
+#ifndef NOPROTO
+		s->cert->rsa_tmp_cb = (RSA *(*)(SSL *, int, int))parg;
+#else
+		s->cert->rsa_tmp_cb = (RSA *(*)())parg;
+#endif
+		break;
+#endif
+#ifndef NO_DH
+	case SSL_CTRL_SET_TMP_DH:
+		{
+			DH *dh = (DH *)parg;
+			if (dh == NULL) {
+				SSLerr(SSL_F_SSL3_CTRL, ERR_R_PASSED_NULL_PARAMETER);
+				return(ret);
+			}
+			if ((dh = DHparams_dup(dh)) == NULL) {
+				SSLerr(SSL_F_SSL3_CTRL, ERR_R_DH_LIB);
+				return(ret);
+			}
+			if (!DH_generate_key(dh)) {
+				DH_free(dh);
+				SSLerr(SSL_F_SSL3_CTRL, ERR_R_DH_LIB);
+				return(ret);
+			}
+			if (s->cert->dh_tmp != NULL)
+				DH_free(s->cert->dh_tmp);
+			s->cert->dh_tmp = dh;
+			ret = 1;
+		}
+		break;
+	case SSL_CTRL_SET_TMP_DH_CB:
+#ifndef NOPROTO
+		s->cert->dh_tmp_cb = (DH *(*)(SSL *, int, int))parg;
+#else
+		s->cert->dh_tmp_cb = (DH *(*)())parg;
+#endif
+		break;
+#endif
 	default:
 		break;
 		}

ssl/ssl.err

@@ -113,6 +113,8 @@
 #define SSL_F_TLS1_ENC					 210
 #define SSL_F_TLS1_SETUP_KEY_BLOCK			 211
 #define SSL_F_WRITE_PENDING				 212
+#define SSL_F_SSL3_CTRL					 213
+#define SSL_F_SSL_CERT_INSTANTIATE			 214
 
 /* Reason codes. */
 #define SSL_R_APP_DATA_IN_HANDSHAKE			 100

ssl/ssl.h

@@ -784,6 +784,13 @@ struct ssl_st
 #define SSL_CTX_set_tmp_dh(ctx,dh) \
 	SSL_CTX_ctrl(ctx,SSL_CTRL_SET_TMP_DH,0,(char *)dh)
 
+#define SSL_need_tmp_RSA(ssl) \
+	SSL_ctrl(ssl,SSL_CTRL_NEED_TMP_RSA,0,NULL)
+#define SSL_set_tmp_rsa(ssl,rsa) \
+	SSL_ctrl(ssl,SSL_CTRL_SET_TMP_RSA,0,(char *)rsa)
+#define SSL_set_tmp_dh(ssl,dh) \
+	SSL_ctrl(ssl,SSL_CTRL_SET_TMP_DH,0,(char *)dh)
+
 #define SSL_CTX_add_extra_chain_cert(ctx,x509) \
 	SSL_CTX_ctrl(ctx,SSL_CTRL_EXTRA_CHAIN_CERT,0,(char *)x509)
 
@@ -1029,6 +1036,12 @@ void SSL_CTX_set_tmp_rsa_callback(SSL_CTX *ctx,
 void SSL_CTX_set_tmp_dh_callback(SSL_CTX *ctx,
 				 DH *(*dh)(SSL *ssl,int export,int keylength));
 
+void SSL_set_tmp_rsa_callback(SSL *ssl,
+				  RSA *(*cb)(SSL *ssl,int export,
+					     int keylength));
+void SSL_set_tmp_dh_callback(SSL *ssl,
+				 DH *(*dh)(SSL *ssl,int export,int keylength));
+
 #ifdef HEADER_COMP_H
 int SSL_COMP_add_compression_method(int id,COMP_METHOD *cm);
 #else
@@ -1258,6 +1271,9 @@ int SSL_COMP_add_compression_method();
 void SSL_CTX_set_tmp_rsa_callback();
 void SSL_CTX_set_tmp_dh_callback();
 
+void SSL_set_tmp_rsa_callback();
+void SSL_set_tmp_dh_callback();
+
 /* #endif */
 
 #endif
@@ -1378,6 +1394,8 @@ void SSL_CTX_set_tmp_dh_callback();
 #define SSL_F_TLS1_ENC					 210
 #define SSL_F_TLS1_SETUP_KEY_BLOCK			 211
 #define SSL_F_WRITE_PENDING				 212
+#define SSL_F_SSL3_CTRL					 213
+#define SSL_F_SSL_CERT_INSTANTIATE			 214
 
 /* Reason codes. */
 #define SSL_R_APP_DATA_IN_HANDSHAKE			 100

ssl/ssl_cert.c

@@ -144,6 +144,27 @@ CERT *c;
 	Free(c);
 	}
 
+int ssl_cert_instantiate(CERT **o, CERT *d)
+	{
+	CERT *n;
+	if (o == NULL) 
+		{
+		SSLerr(SSL_F_SSL_CERT_INSTANTIATE, ERR_R_PASSED_NULL_PARAMETER);
+		return(0);
+		}
+	if (*o != NULL && d != NULL && *o != d)
+	    return(1);
+	if ((n = ssl_cert_new()) == NULL) 
+		{
+		SSLerr(SSL_F_SSL_CERT_INSTANTIATE, ERR_R_MALLOC_FAILURE);
+		return(0);
+		}
+	if (*o != NULL) 
+		ssl_cert_free(*o);
+	*o = n;
+	return(1);
+	}
+
 int ssl_set_cert_type(c, type)
 CERT *c;
 int type;

ssl/ssl_err.c

@@ -175,6 +175,8 @@ static ERR_STRING_DATA SSL_str_functs[]=
 {ERR_PACK(0,SSL_F_TLS1_ENC,0),	"TLS1_ENC"},
 {ERR_PACK(0,SSL_F_TLS1_SETUP_KEY_BLOCK,0),	"TLS1_SETUP_KEY_BLOCK"},
 {ERR_PACK(0,SSL_F_WRITE_PENDING,0),	"WRITE_PENDING"},
+{ERR_PACK(0,SSL_F_SSL3_CTRL,0),	"SSL3_CTRL"},
+{ERR_PACK(0,SSL_F_SSL_CERT_INSTANTIATE,0),	"SSL_CERT_INSTANTIATE"},
 {0,NULL},
 	};
 

ssl/ssl_lib.c

@@ -1899,6 +1899,14 @@ void SSL_CTX_set_tmp_dh_callback(SSL_CTX *ctx,DH *(*dh)(SSL *ssl,int export,
 							int keylength))
     { SSL_CTX_ctrl(ctx,SSL_CTRL_SET_TMP_DH_CB,0,(char *)dh); }
 
+void SSL_set_tmp_rsa_callback(SSL *ssl,RSA *(*cb)(SSL *ssl,int export,
+							  int keylength))
+    { SSL_ctrl(ssl,SSL_CTRL_SET_TMP_RSA_CB,0,(char *)cb); }
+
+void SSL_set_tmp_dh_callback(SSL *ssl,DH *(*dh)(SSL *ssl,int export,
+							int keylength))
+    { SSL_ctrl(ssl,SSL_CTRL_SET_TMP_DH_CB,0,(char *)dh); }
+
 #if defined(_WINDLL) && defined(WIN16)
 #include "../crypto/bio/bss_file.c"
 #endif

ssl/ssl_locl.h

@@ -348,6 +348,7 @@ SSL_METHOD *sslv3_base_method(void);
 void ssl_clear_cipher_ctx(SSL *s);
 int ssl_clear_bad_session(SSL *s);
 CERT *ssl_cert_new(void);
+int ssl_cert_instantiate(CERT **o, CERT *d);
 void ssl_cert_free(CERT *c);
 int ssl_set_cert_type(CERT *c, int type);
 int ssl_get_new_session(SSL *s, int session);
@@ -483,6 +484,7 @@ SSL_METHOD *sslv3_base_method();
 void ssl_clear_cipher_ctx();
 int ssl_clear_bad_session();
 CERT *ssl_cert_new();
+int ssl_cert_instantiate();
 void ssl_cert_free();
 int ssl_set_cert_type();
 int ssl_get_new_session();

ssl/ssl_rsa.c

@@ -76,27 +76,17 @@ int SSL_use_certificate(ssl, x)
 SSL *ssl;
 X509 *x;
 	{
-	CERT *c;
-
 	if (x == NULL)
 		{
 		SSLerr(SSL_F_SSL_USE_CERTIFICATE,ERR_R_PASSED_NULL_PARAMETER);
 		return(0);
 		}
-	if ((ssl->cert == NULL) || (ssl->cert == ssl->ctx->default_cert))
+	if (!ssl_cert_instantiate(&ssl->cert, ssl->ctx->default_cert)) 
 		{
-		c=ssl_cert_new();
-		if (c == NULL)
-			{
-			SSLerr(SSL_F_SSL_USE_CERTIFICATE,ERR_R_MALLOC_FAILURE);
-			return(0);
-			}
-		if (ssl->cert != NULL) ssl_cert_free(ssl->cert);
-		ssl->cert=c;
+		SSLerr(SSL_F_SSL_USE_CERTIFICATE,ERR_R_MALLOC_FAILURE);
+		return(0);
 		}
-	c=ssl->cert;
-
-	return(ssl_set_cert(c,x));
+	return(ssl_set_cert(ssl->cert,x));
 	}
 
 #ifndef NO_STDIO
@@ -177,7 +167,6 @@ int SSL_use_RSAPrivateKey(ssl, rsa)
 SSL *ssl;
 RSA *rsa;
 	{
-	CERT *c;
 	EVP_PKEY *pkey;
 	int ret;
 
@@ -186,19 +175,11 @@ RSA *rsa;
 		SSLerr(SSL_F_SSL_USE_RSAPRIVATEKEY,ERR_R_PASSED_NULL_PARAMETER);
 		return(0);
 		}
-
-        if ((ssl->cert == NULL) || (ssl->cert == ssl->ctx->default_cert))
-                {
-                c=ssl_cert_new();
-		if (c == NULL)
-			{
-			SSLerr(SSL_F_SSL_USE_RSAPRIVATEKEY,ERR_R_MALLOC_FAILURE);
-			return(0);
-			}
-                if (ssl->cert != NULL) ssl_cert_free(ssl->cert);
-		ssl->cert=c;
+	if (!ssl_cert_instantiate(&ssl->cert, ssl->ctx->default_cert)) 
+		{
+		SSLerr(SSL_F_SSL_USE_RSAPRIVATEKEY,ERR_R_MALLOC_FAILURE);
+		return(0);
 		}
-	c=ssl->cert;
 	if ((pkey=EVP_PKEY_new()) == NULL)
 		{
 		SSLerr(SSL_F_SSL_USE_RSAPRIVATEKEY,ERR_R_EVP_LIB);
@@ -208,7 +189,7 @@ RSA *rsa;
 	CRYPTO_add(&rsa->references,1,CRYPTO_LOCK_RSA);
 	EVP_PKEY_assign_RSA(pkey,rsa);
 
-	ret=ssl_set_pkey(c,pkey);
+	ret=ssl_set_pkey(ssl->cert,pkey);
 	EVP_PKEY_free(pkey);
 	return(ret);
 	}
@@ -366,7 +347,6 @@ int SSL_use_PrivateKey(ssl, pkey)
 SSL *ssl;
 EVP_PKEY *pkey;
 	{
-	CERT *c;
 	int ret;
 
 	if (pkey == NULL)
@@ -374,21 +354,12 @@ EVP_PKEY *pkey;
 		SSLerr(SSL_F_SSL_USE_PRIVATEKEY,ERR_R_PASSED_NULL_PARAMETER);
 		return(0);
 		}
-
-        if ((ssl->cert == NULL) || (ssl->cert == ssl->ctx->default_cert))
-                {
-                c=ssl_cert_new();
-		if (c == NULL)
-			{
-			SSLerr(SSL_F_SSL_USE_PRIVATEKEY,ERR_R_MALLOC_FAILURE);
-			return(0);
-			}
-                if (ssl->cert != NULL) ssl_cert_free(ssl->cert);
-		ssl->cert=c;
+	if (!ssl_cert_instantiate(&ssl->cert, ssl->ctx->default_cert)) 
+		{
+		SSLerr(SSL_F_SSL_USE_PRIVATEKEY,ERR_R_MALLOC_FAILURE);
+		return(0);
 		}
-	c=ssl->cert;
-
-	ret=ssl_set_pkey(c,pkey);
+	ret=ssl_set_pkey(ssl->cert,pkey);
 	return(ret);
 	}
 
@@ -464,27 +435,17 @@ int SSL_CTX_use_certificate(ctx, x)
 SSL_CTX *ctx;
 X509 *x;
 	{
-	CERT *c;
-
 	if (x == NULL)
 		{
 		SSLerr(SSL_F_SSL_CTX_USE_CERTIFICATE,ERR_R_PASSED_NULL_PARAMETER);
 		return(0);
 		}
-
-	if (ctx->default_cert == NULL)
+	if (!ssl_cert_instantiate(&ctx->default_cert, NULL))
 		{
-		c=ssl_cert_new();
-		if (c == NULL)
-			{
-			SSLerr(SSL_F_SSL_CTX_USE_CERTIFICATE,ERR_R_MALLOC_FAILURE);
-			return(0);
-			}
-		ctx->default_cert=c;
+		SSLerr(SSL_F_SSL_CTX_USE_CERTIFICATE,ERR_R_MALLOC_FAILURE);
+		return(0);
 		}
-	c=ctx->default_cert;
-
-	return(ssl_set_cert(c,x));
+	return(ssl_set_cert(ctx->default_cert,x));
 	}
 
 static int ssl_set_cert(c,x)
@@ -648,7 +609,6 @@ SSL_CTX *ctx;
 RSA *rsa;
 	{
 	int ret;
-	CERT *c;
 	EVP_PKEY *pkey;
 
 	if (rsa == NULL)
@@ -656,18 +616,11 @@ RSA *rsa;
 		SSLerr(SSL_F_SSL_CTX_USE_RSAPRIVATEKEY,ERR_R_PASSED_NULL_PARAMETER);
 		return(0);
 		}
-	if (ctx->default_cert == NULL)
+	if (!ssl_cert_instantiate(&ctx->default_cert, NULL))
 		{
-		c=ssl_cert_new();
-		if (c == NULL)
-			{
-			SSLerr(SSL_F_SSL_CTX_USE_RSAPRIVATEKEY,ERR_R_MALLOC_FAILURE);
-			return(0);
-			}
-		ctx->default_cert=c;
+		SSLerr(SSL_F_SSL_CTX_USE_RSAPRIVATEKEY,ERR_R_MALLOC_FAILURE);
+		return(0);
 		}
-	c=ctx->default_cert;
-
 	if ((pkey=EVP_PKEY_new()) == NULL)
 		{
 		SSLerr(SSL_F_SSL_CTX_USE_RSAPRIVATEKEY,ERR_R_EVP_LIB);
@@ -677,7 +630,7 @@ RSA *rsa;
 	CRYPTO_add(&rsa->references,1,CRYPTO_LOCK_RSA);
 	EVP_PKEY_assign_RSA(pkey,rsa);
 
-	ret=ssl_set_pkey(c,pkey);
+	ret=ssl_set_pkey(ctx->default_cert,pkey);
 	EVP_PKEY_free(pkey);
 	return(ret);
 	}
@@ -759,27 +712,17 @@ int SSL_CTX_use_PrivateKey(ctx, pkey)
 SSL_CTX *ctx;
 EVP_PKEY *pkey;
 	{
-	CERT *c;
-
 	if (pkey == NULL)
 		{
 		SSLerr(SSL_F_SSL_CTX_USE_PRIVATEKEY,ERR_R_PASSED_NULL_PARAMETER);
 		return(0);
 		}
-		
-	if (ctx->default_cert == NULL)
+	if (!ssl_cert_instantiate(&ctx->default_cert, NULL))
 		{
-		c=ssl_cert_new();
-		if (c == NULL)
-			{
-			SSLerr(SSL_F_SSL_CTX_USE_PRIVATEKEY,ERR_R_MALLOC_FAILURE);
-			return(0);
-			}
-		ctx->default_cert=c;
+		SSLerr(SSL_F_SSL_CTX_USE_PRIVATEKEY,ERR_R_MALLOC_FAILURE);
+		return(0);
 		}
-	c=ctx->default_cert;
-
-	return(ssl_set_pkey(c,pkey));
+	return(ssl_set_pkey(ctx->default_cert,pkey));
 	}
 
 #ifndef NO_STDIO