Add a test for small subgroup attacks on DH/DHE

Following on from the previous commit, add a test to ensure that
DH_compute_key correctly fails if passed a bad y such that:

y^q (mod p) != 1
Reviewed-by: NViktor Dukhovni <viktor@openssl.org>

test/dhtest.c

@@ -454,6 +454,31 @@ static const unsigned char dhtest_2048_256_Z[] = {
     0xC2, 0x6C, 0x5D, 0x7C
 };
 
+static const unsigned char dhtest_rfc5114_2048_224_bad_y[] = {
+    0x45, 0x32, 0x5F, 0x51, 0x07, 0xE5, 0xDF, 0x1C, 0xD6, 0x02, 0x82, 0xB3,
+    0x32, 0x8F, 0xA4, 0x0F, 0x87, 0xB8, 0x41, 0xFE, 0xB9, 0x35, 0xDE, 0xAD,
+    0xC6, 0x26, 0x85, 0xB4, 0xFF, 0x94, 0x8C, 0x12, 0x4C, 0xBF, 0x5B, 0x20,
+    0xC4, 0x46, 0xA3, 0x26, 0xEB, 0xA4, 0x25, 0xB7, 0x68, 0x8E, 0xCC, 0x67,
+    0xBA, 0xEA, 0x58, 0xD0, 0xF2, 0xE9, 0xD2, 0x24, 0x72, 0x60, 0xDA, 0x88,
+    0x18, 0x9C, 0xE0, 0x31, 0x6A, 0xAD, 0x50, 0x6D, 0x94, 0x35, 0x8B, 0x83,
+    0x4A, 0x6E, 0xFA, 0x48, 0x73, 0x0F, 0x83, 0x87, 0xFF, 0x6B, 0x66, 0x1F,
+    0xA8, 0x82, 0xC6, 0x01, 0xE5, 0x80, 0xB5, 0xB0, 0x52, 0xD0, 0xE9, 0xD8,
+    0x72, 0xF9, 0x7D, 0x5B, 0x8B, 0xA5, 0x4C, 0xA5, 0x25, 0x95, 0x74, 0xE2,
+    0x7A, 0x61, 0x4E, 0xA7, 0x8F, 0x12, 0xE2, 0xD2, 0x9D, 0x8C, 0x02, 0x70,
+    0x34, 0x44, 0x32, 0xC7, 0xB2, 0xF3, 0xB9, 0xFE, 0x17, 0x2B, 0xD6, 0x1F,
+    0x8B, 0x7E, 0x4A, 0xFA, 0xA3, 0xB5, 0x3E, 0x7A, 0x81, 0x9A, 0x33, 0x66,
+    0x62, 0xA4, 0x50, 0x18, 0x3E, 0xA2, 0x5F, 0x00, 0x07, 0xD8, 0x9B, 0x22,
+    0xE4, 0xEC, 0x84, 0xD5, 0xEB, 0x5A, 0xF3, 0x2A, 0x31, 0x23, 0xD8, 0x44,
+    0x22, 0x2A, 0x8B, 0x37, 0x44, 0xCC, 0xC6, 0x87, 0x4B, 0xBE, 0x50, 0x9D,
+    0x4A, 0xC4, 0x8E, 0x45, 0xCF, 0x72, 0x4D, 0xC0, 0x89, 0xB3, 0x72, 0xED,
+    0x33, 0x2C, 0xBC, 0x7F, 0x16, 0x39, 0x3B, 0xEB, 0xD2, 0xDD, 0xA8, 0x01,
+    0x73, 0x84, 0x62, 0xB9, 0x29, 0xD2, 0xC9, 0x51, 0x32, 0x9E, 0x7A, 0x6A,
+    0xCF, 0xC1, 0x0A, 0xDB, 0x0E, 0xE0, 0x62, 0x77, 0x6F, 0x59, 0x62, 0x72,
+    0x5A, 0x69, 0xA6, 0x5B, 0x70, 0xCA, 0x65, 0xC4, 0x95, 0x6F, 0x9A, 0xC2,
+    0xDF, 0x72, 0x6D, 0xB1, 0x1E, 0x54, 0x7B, 0x51, 0xB4, 0xEF, 0x7F, 0x89,
+    0x93, 0x74, 0x89, 0x59
+};
+
 typedef struct {
     DH *(*get_param) (void);
     const unsigned char *xA;
@@ -491,12 +516,9 @@ static int run_rfc5114_tests(void)
     unsigned char *Z1 = NULL;
     unsigned char *Z2 = NULL;
     const rfc5114_td *td = NULL;
+    BIGNUM *bady = NULL;
 
     for (i = 0; i < (int)OSSL_NELEM(rfctd); i++) {
-        dhA = NULL;
-        dhB = NULL;
-        Z1 = NULL;
-        Z2 = NULL;
         td = rfctd + i;
         /* Set up DH structures setting key components */
         dhA = td->get_param();
@@ -542,10 +564,48 @@ static int run_rfc5114_tests(void)
         DH_free(dhB);
         OPENSSL_free(Z1);
         OPENSSL_free(Z2);
+        dhA = NULL;
+        dhB = NULL;
+        Z1 = NULL;
+        Z2 = NULL;
+    }
 
+    /* Now i == OSSL_NELEM(rfctd) */
+    /* RFC5114 uses unsafe primes, so now test an invalid y value */
+    dhA = DH_get_2048_224();
+    if (dhA == NULL)
+        goto bad_err;
+    Z1 = OPENSSL_malloc(DH_size(dhA));
+    if (Z1 == NULL)
+        goto bad_err;
+
+    bady = BN_bin2bn(dhtest_rfc5114_2048_224_bad_y,
+                     sizeof(dhtest_rfc5114_2048_224_bad_y), NULL);
+    if (bady == NULL)
+        goto bad_err;
+
+    if (!DH_generate_key(dhA))
+        goto bad_err;
+
+    if (DH_compute_key(Z1, bady, dhA) != -1) {
+        /*
+         * DH_compute_key should fail with -1. If we get here we unexpectedly
+         * allowed an invalid y value
+         */
+        goto err;
     }
+    /* We'll have a stale error on the queue from the above test so clear it */
+    ERR_clear_error();
+
+    printf("RFC5114 parameter test %d OK\n", i + 1);
+
+    BN_free(bady);
+    DH_free(dhA);
+    OPENSSL_free(Z1);
+
     return 1;
  bad_err:
+    BN_free(bady);
     DH_free(dhA);
     DH_free(dhB);
     OPENSSL_free(Z1);
@@ -555,6 +615,7 @@ static int run_rfc5114_tests(void)
     ERR_print_errors_fp(stderr);
     return 0;
  err:
+    BN_free(bady);
     DH_free(dhA);
     DH_free(dhB);
     OPENSSL_free(Z1);