Support pmeth->digest_custom

Reviewed-by: NTim Hudson <tjh@openssl.org>
Reviewed-by: NMatt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/7113)

crypto/evp/m_sigver.c

@@ -75,6 +75,14 @@ static int do_sigver_init(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx,
         return 1;
     if (!EVP_DigestInit_ex(ctx, type, e))
         return 0;
+    if (ctx->pctx->pmeth->flags & EVP_PKEY_FLAG_DIGEST_CUSTOM) {
+        /*
+         * This indicates the current algorithm requires
+         * special treatment before hashing the tbs-message.
+         */
+        if (ctx->pctx->pmeth->digest_custom)
+            return ctx->pctx->pmeth->digest_custom(ctx->pctx, ctx);
+    }
     return 1;
 }
 

crypto/evp/pmeth_lib.c

@@ -655,6 +655,13 @@ void EVP_PKEY_meth_set_param_check(EVP_PKEY_METHOD *pmeth,
     pmeth->param_check = check;
 }
 
+void EVP_PKEY_meth_set_digest_custom(EVP_PKEY_METHOD *pmeth,
+                                     int (*digest_custom) (EVP_PKEY_CTX *ctx,
+                                                           EVP_MD_CTX *mctx))
+{
+    pmeth->digest_custom = digest_custom;
+}
+
 void EVP_PKEY_meth_get_init(const EVP_PKEY_METHOD *pmeth,
                             int (**pinit) (EVP_PKEY_CTX *ctx))
 {
@@ -842,3 +849,11 @@ void EVP_PKEY_meth_get_param_check(const EVP_PKEY_METHOD *pmeth,
     if (*pcheck)
         *pcheck = pmeth->param_check;
 }
+
+void EVP_PKEY_meth_get_digest_custom(EVP_PKEY_METHOD *pmeth,
+                                     int (**pdigest_custom) (EVP_PKEY_CTX *ctx,
+                                                             EVP_MD_CTX *mctx))
+{
+    if (*pdigest_custom)
+        *pdigest_custom = pmeth->digest_custom;
+}

crypto/include/internal/evp_int.h

@@ -79,6 +79,8 @@ struct evp_pkey_method_st {
     int (*check) (EVP_PKEY *pkey);
     int (*public_check) (EVP_PKEY *pkey);
     int (*param_check) (EVP_PKEY *pkey);
+
+    int (*digest_custom) (EVP_PKEY_CTX *ctx, EVP_MD_CTX *mctx);
 } /* EVP_PKEY_METHOD */ ;
 
 DEFINE_STACK_OF_CONST(EVP_PKEY_METHOD)

include/openssl/evp.h

@@ -1325,6 +1325,8 @@ void EVP_PKEY_asn1_set_security_bits(EVP_PKEY_ASN1_METHOD *ameth,
  * Method handles all operations: don't assume any digest related defaults.
  */
 # define EVP_PKEY_FLAG_SIGCTX_CUSTOM     4
+/* Do a customized hashing process */
+# define EVP_PKEY_FLAG_DIGEST_CUSTOM     8
 
 const EVP_PKEY_METHOD *EVP_PKEY_meth_find(int type);
 EVP_PKEY_METHOD *EVP_PKEY_meth_new(int id, int flags);
@@ -1523,6 +1525,10 @@ void EVP_PKEY_meth_set_public_check(EVP_PKEY_METHOD *pmeth,
 void EVP_PKEY_meth_set_param_check(EVP_PKEY_METHOD *pmeth,
                                    int (*check) (EVP_PKEY *pkey));
 
+void EVP_PKEY_meth_set_digest_custom(EVP_PKEY_METHOD *pmeth,
+                                     int (*digest_custom) (EVP_PKEY_CTX *ctx,
+                                                           EVP_MD_CTX *mctx));
+
 void EVP_PKEY_meth_get_init(const EVP_PKEY_METHOD *pmeth,
                             int (**pinit) (EVP_PKEY_CTX *ctx));
 
@@ -1624,6 +1630,9 @@ void EVP_PKEY_meth_get_public_check(const EVP_PKEY_METHOD *pmeth,
 void EVP_PKEY_meth_get_param_check(const EVP_PKEY_METHOD *pmeth,
                                    int (**pcheck) (EVP_PKEY *pkey));
 
+void EVP_PKEY_meth_get_digest_custom(EVP_PKEY_METHOD *pmeth,
+                                     int (**pdigest_custom) (EVP_PKEY_CTX *ctx,
+                                                             EVP_MD_CTX *mctx));
 void EVP_add_alg_module(void);
 
 

util/libcrypto.num

@@ -4575,3 +4575,5 @@ EC_GROUP_get_curve                      4528	1_1_1	EXIST::FUNCTION:EC
 OCSP_resp_get0_tbs_sigalg               4529	1_1_0j	EXIST::FUNCTION:OCSP
 OCSP_resp_get0_respdata                 4530	1_1_0j	EXIST::FUNCTION:OCSP
 EVP_MD_CTX_set_pkey_ctx                 4531	1_1_1	EXIST::FUNCTION:
+EVP_PKEY_meth_set_digest_custom         4532	1_1_1	EXIST::FUNCTION:
+EVP_PKEY_meth_get_digest_custom         4533	1_1_1	EXIST::FUNCTION: