Extract compression form in EC_KEY_oct2key().

Extract compression form in EC_KEY_oct2key() instead of manually in the
ASN.1 code. For custom curves do not assume the initial octet is the
compression form: it isn't for X25519 et al.
Reviewed-by: NRich Salz <rsalz@openssl.org>
Reviewed-by: NEmilia Käsper <emilia@openssl.org>

crypto/ec/ec_asn1.c

@@ -1039,17 +1039,7 @@ EC_KEY *d2i_ECPrivateKey(EC_KEY **a, const unsigned char **in, long len)
 
         pub_oct = ASN1_STRING_data(priv_key->publicKey);
         pub_oct_len = ASN1_STRING_length(priv_key->publicKey);
-        /*
-         * The first byte - point conversion form - must be present.
-         */
-        if (pub_oct_len <= 0) {
-            ECerr(EC_F_D2I_ECPRIVATEKEY, EC_R_BUFFER_TOO_SMALL);
-            goto err;
-        }
-        /* Save the point conversion form. */
-        ret->conv_form = (point_conversion_form_t) (pub_oct[0] & ~0x01);
-        if (!EC_POINT_oct2point(ret->group, ret->pub_key,
-                                pub_oct, (size_t)(pub_oct_len), NULL)) {
+        if (!EC_KEY_oct2key(ret, pub_oct, pub_oct_len, NULL)) {
             ECerr(EC_F_D2I_ECPRIVATEKEY, ERR_R_EC_LIB);
             goto err;
         }
@@ -1201,17 +1191,10 @@ EC_KEY *o2i_ECPublicKey(EC_KEY **a, const unsigned char **in, long len)
         return 0;
     }
     ret = *a;
-    if (ret->pub_key == NULL &&
-        (ret->pub_key = EC_POINT_new(ret->group)) == NULL) {
-        ECerr(EC_F_O2I_ECPUBLICKEY, ERR_R_MALLOC_FAILURE);
-        return 0;
-    }
-    if (!EC_POINT_oct2point(ret->group, ret->pub_key, *in, len, NULL)) {
+    if (!EC_KEY_oct2key(ret, *in, len, NULL)) {
         ECerr(EC_F_O2I_ECPUBLICKEY, ERR_R_EC_LIB);
         return 0;
     }
-    /* save the point conversion form */
-    ret->conv_form = (point_conversion_form_t) (*in[0] & ~0x01);
     *in += len;
     return ret;
 }

crypto/ec/ec_key.c

@@ -552,7 +552,18 @@ int EC_KEY_oct2key(EC_KEY *key, const unsigned char *buf, size_t len,
         key->pub_key = EC_POINT_new(key->group);
     if (key->pub_key == NULL)
         return 0;
-    return EC_POINT_oct2point(key->group, key->pub_key, buf, len, ctx);
+    if (EC_POINT_oct2point(key->group, key->pub_key, buf, len, ctx) == 0)
+        return 0;
+    /*
+     * Save the point conversion form.
+     * For non-custom curves the first octet of the buffer (excluding
+     * the last significant bit) contains the point conversion form.
+     * EC_POINT_oct2point() has already performed sanity checking of
+     * the buffer so we know it is valid.
+     */
+    if ((key->group->meth->flags & EC_FLAGS_CUSTOM_CURVE) == 0)
+        key->conv_form = (point_conversion_form_t)(buf[0] & ~0x01);
+    return 1;
 }
 
 size_t EC_KEY_priv2oct(const EC_KEY *eckey, unsigned char *buf, size_t len)