Update smime utility to support streaming for -encrypt and -sign -nodetach

options. Add new streaming i2d (though strictly speaking it is BER format
when streaming) and PEM functions.

These all process content on the fly without storing it all in memory.

CHANGES

@@ -4,6 +4,11 @@
 
  Changes between 0.9.8f and 0.9.9  [xx XXX xxxx]
 
+  *) Add option -stream to use PKCS#7 streaming in smime utility. New
+     function i2d_PKCS7_bio_stream() and PEM_write_PKCS7_bio_stream()
+     to output in BER and PEM format.
+     [Steve Henson]
+
   *) Experimental support for use of HMAC via EVP_PKEY interface. This
      allows HMAC to be handled via the EVP_DigestSign*() interface. The
      EVP_PKEY "key" in this case is the HMAC key, potentially allowing

apps/smime.c

@@ -109,6 +109,7 @@ int MAIN(int argc, char **argv)
 	char *passargin = NULL, *passin = NULL;
 	char *inrand = NULL;
 	int need_rand = 0;
+	int indef = 0;
 	const EVP_MD *sign_md = NULL;
 	int informat = FORMAT_SMIME, outformat = FORMAT_SMIME;
         int keyform = FORMAT_PEM;
@@ -196,6 +197,12 @@ int MAIN(int argc, char **argv)
 				flags |= PKCS7_BINARY;
 		else if (!strcmp (*args, "-nosigs"))
 				flags |= PKCS7_NOSIGS;
+		else if (!strcmp (*args, "-stream"))
+				indef = 1;
+		else if (!strcmp (*args, "-indef"))
+				indef = 1;
+		else if (!strcmp (*args, "-noindef"))
+				indef = 0;
 		else if (!strcmp (*args, "-nooldmime"))
 				flags |= PKCS7_NOOLDMIMETYPE;
 		else if (!strcmp (*args, "-crlfeol"))
@@ -666,7 +673,11 @@ int MAIN(int argc, char **argv)
 	ret = 3;
 
 	if (operation == SMIME_ENCRYPT)
+		{
+		if (indef)
+			flags |= PKCS7_STREAM;
 		p7 = PKCS7_encrypt(encerts, in, cipher, flags);
+		}
 	else if (operation & SMIME_SIGNERS)
 		{
 		int i;
@@ -675,8 +686,7 @@ int MAIN(int argc, char **argv)
 		 */
 		if (operation == SMIME_SIGN)
 			{
-			if ((flags & PKCS7_DETACHED)
-				&& (outformat == FORMAT_SMIME))
+			if (indef || (flags & PKCS7_DETACHED))
 				flags |= PKCS7_STREAM;
 			flags |= PKCS7_PARTIAL;
 			p7 = PKCS7_sign(NULL, NULL, other, in, flags);
@@ -764,9 +774,9 @@ int MAIN(int argc, char **argv)
 				SMIME_write_PKCS7(out, p7, in, flags);
 			}
 		else if (outformat == FORMAT_PEM) 
-			PEM_write_bio_PKCS7(out,p7);
+			PEM_write_bio_PKCS7_stream(out, p7, in, flags);
 		else if (outformat == FORMAT_ASN1) 
-			i2d_PKCS7_bio(out,p7);
+			i2d_PKCS7_bio_stream(out,p7, in, flags);
 		else
 			{
 			BIO_printf(bio_err, "Bad output format for PKCS#7 file\n");

crypto/asn1/asn1_lib.c

@@ -427,7 +427,8 @@ ASN1_STRING *ASN1_STRING_type_new(int type)
 void ASN1_STRING_free(ASN1_STRING *a)
 	{
 	if (a == NULL) return;
-	if (a->data != NULL) OPENSSL_free(a->data);
+	if (a->data && !(a->flags & ASN1_STRING_FLAG_NDEF))
+		OPENSSL_free(a->data);
 	OPENSSL_free(a);
 	}
 

crypto/pkcs7/bio_pk7.c

@@ -97,8 +97,9 @@ typedef struct pkcs7_aux_st
 	} PKCS7_SUPPORT;
 
 static int pkcs7_prefix(BIO *b, unsigned char **pbuf, int *plen, void *parg);
-static int pkcs7_psfix_free(BIO *b, unsigned char **pbuf, int *plen, void *parg);
+static int pkcs7_prefix_free(BIO *b, unsigned char **pbuf, int *plen, void *parg);
 static int pkcs7_suffix(BIO *b, unsigned char **pbuf, int *plen, void *parg);
+static int pkcs7_suffix_free(BIO *b, unsigned char **pbuf, int *plen, void *parg);
 
 BIO *BIO_new_PKCS7(BIO *out, PKCS7 *p7) 
 	{
@@ -113,8 +114,8 @@ BIO *BIO_new_PKCS7(BIO *out, PKCS7 *p7)
 
 	out = BIO_push(asn_bio, out);
 
-	BIO_asn1_set_prefix(asn_bio, pkcs7_prefix, pkcs7_psfix_free);
-	BIO_asn1_set_suffix(asn_bio, pkcs7_suffix, pkcs7_psfix_free);
+	BIO_asn1_set_prefix(asn_bio, pkcs7_prefix, pkcs7_prefix_free);
+	BIO_asn1_set_suffix(asn_bio, pkcs7_suffix, pkcs7_suffix_free);
 
 	/* Now initialize BIO for PKCS#7 output */
 
@@ -132,7 +133,6 @@ BIO *BIO_new_PKCS7(BIO *out, PKCS7 *p7)
 
 	}
 
-
 static int pkcs7_prefix(BIO *b, unsigned char **pbuf, int *plen, void *parg)
 	{
 	PKCS7_SUPPORT *p7aux;
@@ -150,12 +150,15 @@ static int pkcs7_prefix(BIO *b, unsigned char **pbuf, int *plen, void *parg)
 	*pbuf = p;
 	i2d_PKCS7_NDEF(p7aux->p7, &p);
 
+	if (!*p7aux->boundary)
+		return 0;
+
 	*plen = *p7aux->boundary - *pbuf;
 
 	return 1;
 	}
 
-static int pkcs7_psfix_free(BIO *b, unsigned char **pbuf, int *plen, void *parg)
+static int pkcs7_prefix_free(BIO *b, unsigned char **pbuf, int *plen, void *parg)
 	{
 	PKCS7_SUPPORT *p7aux;
 
@@ -173,6 +176,16 @@ static int pkcs7_psfix_free(BIO *b, unsigned char **pbuf, int *plen, void *parg)
 	return 1;
 	}
 
+static int pkcs7_suffix_free(BIO *b, unsigned char **pbuf, int *plen, void *parg)
+	{
+	PKCS7_SUPPORT **pp7aux = (PKCS7_SUPPORT **)parg;
+	if (!pkcs7_prefix_free(b, pbuf, plen, parg))
+		return 0;
+	OPENSSL_free(*pp7aux);
+	*pp7aux = NULL;
+	return 1;
+	}
+
 static int pkcs7_suffix(BIO *b, unsigned char **pbuf, int *plen, void *parg)
 	{
 	PKCS7_SUPPORT *p7aux;
@@ -191,6 +204,8 @@ static int pkcs7_suffix(BIO *b, unsigned char **pbuf, int *plen, void *parg)
 	p = OPENSSL_malloc(derlen);
 	p7aux->derbuf = p;
 	i2d_PKCS7_NDEF(p7aux->p7, &p);
+	if (!*p7aux->boundary)
+		return 0;
 	*pbuf = *p7aux->boundary;
 	*plen = derlen - (*p7aux->boundary - p7aux->derbuf);
 

crypto/pkcs7/pk7_asn1.c

@@ -163,7 +163,7 @@ IMPLEMENT_ASN1_FUNCTIONS(PKCS7_RECIP_INFO)
 ASN1_NDEF_SEQUENCE(PKCS7_ENC_CONTENT) = {
 	ASN1_SIMPLE(PKCS7_ENC_CONTENT, content_type, ASN1_OBJECT),
 	ASN1_SIMPLE(PKCS7_ENC_CONTENT, algorithm, X509_ALGOR),
-	ASN1_IMP_OPT(PKCS7_ENC_CONTENT, enc_data, ASN1_OCTET_STRING, 0)
+	ASN1_IMP_OPT(PKCS7_ENC_CONTENT, enc_data, ASN1_OCTET_STRING_NDEF, 0)
 } ASN1_NDEF_SEQUENCE_END(PKCS7_ENC_CONTENT)
 
 IMPLEMENT_ASN1_FUNCTIONS(PKCS7_ENC_CONTENT)

crypto/pkcs7/pk7_mime.c

@@ -87,7 +87,7 @@ DECLARE_STACK_OF(MIME_HEADER)
 IMPLEMENT_STACK_OF(MIME_HEADER)
 
 static int pkcs7_output_data(BIO *bio, BIO *data, PKCS7 *p7, int flags);
-static int B64_write_PKCS7(BIO *bio, PKCS7 *p7);
+static int B64_write_PKCS7(BIO *bio, PKCS7 *p7, BIO *in, int flags);
 static PKCS7 *B64_read_PKCS7(BIO *bio);
 static char * strip_ends(char *name);
 static char * strip_start(char *name);
@@ -110,22 +110,58 @@ static void mime_hdr_free(MIME_HEADER *hdr);
 #define MAX_SMLEN 1024
 #define mime_debug(x) /* x */
 
+/* Output a PKCS#7 structure in BER format streaming if necessary */
+
+int i2d_PKCS7_bio_stream(BIO *out, PKCS7 *p7, BIO *in, int flags)
+	{
+	/* If streaming create stream BIO and copy all content through it */
+	if (flags & PKCS7_STREAM)
+		{
+		BIO *bio, *tbio;
+		bio = BIO_new_PKCS7(out, p7);
+		if (!bio)
+			{
+			PKCS7err(PKCS7_F_B64_WRITE_PKCS7,ERR_R_MALLOC_FAILURE);
+			return 0;
+			}
+		SMIME_crlf_copy(in, bio, flags);
+		BIO_flush(bio);
+		/* Free up successive BIOs until we hit the old output BIO */
+		do
+			{
+			tbio = BIO_pop(bio);
+			BIO_free(bio);
+			bio = tbio;
+			} while (bio != out);
+		}
+	/* else just write out PKCS7 structure which will have all content
+	 * stored internally
+	 */
+	else
+		i2d_PKCS7_bio(out, p7);
+	return 1;
+	}
+
 /* Base 64 read and write of PKCS#7 structure */
 
-static int B64_write_PKCS7(BIO *bio, PKCS7 *p7)
-{
+static int B64_write_PKCS7(BIO *out, PKCS7 *p7, BIO *in, int flags)
+	{
 	BIO *b64;
-	if(!(b64 = BIO_new(BIO_f_base64()))) {
+	int r;
+	b64 = BIO_new(BIO_f_base64());
+	if(!b64)
+		{
 		PKCS7err(PKCS7_F_B64_WRITE_PKCS7,ERR_R_MALLOC_FAILURE);
 		return 0;
-	}
-	bio = BIO_push(b64, bio);
-	i2d_PKCS7_bio(bio, p7);
-	BIO_flush(bio);
-	bio = BIO_pop(bio);
+		}
+	/* prepend the b64 BIO so all data is base64 encoded.
+	 */
+	out = BIO_push(b64, out);
+	r = i2d_PKCS7_bio_stream(out, p7, in, flags);
+	BIO_pop(out);
 	BIO_free(b64);
-	return 1;
-}
+	return r;
+	}
 
 static PKCS7 *B64_read_PKCS7(BIO *bio)
 {
@@ -144,6 +180,17 @@ static PKCS7 *B64_read_PKCS7(BIO *bio)
 	return p7;
 }
 
+/* Streaming PKCS#7 PEM write */
+
+int PEM_write_bio_PKCS7_stream(BIO *out, PKCS7 *p7, BIO *in, int flags)
+	{
+	int r;
+	BIO_puts(out, "-----BEGIN PKCS7-----\n");
+	r = B64_write_PKCS7(out, p7, in, flags);
+	BIO_puts(out, "-----END PKCS7-----\n");
+	return r;
+	}
+
 /* Generate the MIME "micalg" parameter from RFC3851, RFC4490 */
 
 static int pk7_write_micalg(BIO *out, PKCS7 *p7)
@@ -275,7 +322,7 @@ int SMIME_write_PKCS7(BIO *bio, PKCS7 *p7, BIO *data, int flags)
 		BIO_printf(bio, "Content-Disposition: attachment;");
 		BIO_printf(bio, " filename=\"smime.p7s\"%s%s",
 							mime_eol, mime_eol);
-		B64_write_PKCS7(bio, p7);
+		B64_write_PKCS7(bio, p7, NULL, 0);
 		BIO_printf(bio,"%s------%s--%s%s", mime_eol, bound,
 							mime_eol, mime_eol);
 		return 1;
@@ -297,6 +344,8 @@ int SMIME_write_PKCS7(BIO *bio, PKCS7 *p7, BIO *data, int flags)
 		else
 			msg_type = "certs-only";
 		}
+	else
+		flags &= ~PKCS7_STREAM;
 	/* MIME headers */
 	BIO_printf(bio, "MIME-Version: 1.0%s", mime_eol);
 	BIO_printf(bio, "Content-Disposition: attachment;");
@@ -307,7 +356,7 @@ int SMIME_write_PKCS7(BIO *bio, PKCS7 *p7, BIO *data, int flags)
 	BIO_printf(bio, " name=\"smime.p7m\"%s", mime_eol);
 	BIO_printf(bio, "Content-Transfer-Encoding: base64%s%s",
 						mime_eol, mime_eol);
-	B64_write_PKCS7(bio, p7);
+	B64_write_PKCS7(bio, p7, data, flags);
 	BIO_printf(bio, "%s", mime_eol);
 	return 1;
 }
@@ -463,22 +512,38 @@ PKCS7 *SMIME_read_PKCS7(BIO *bio, BIO **bcont)
 /* Copy text from one BIO to another making the output CRLF at EOL */
 int SMIME_crlf_copy(BIO *in, BIO *out, int flags)
 {
+	BIO *bf;
 	char eol;
 	int len;
 	char linebuf[MAX_SMLEN];
-	if(flags & PKCS7_BINARY) {
+	/* Buffer output so we don't write one line at a time. This is
+	 * useful when streaming as we don't end up with one OCTET STRING
+	 * per line.
+	 */
+	bf = BIO_new(BIO_f_buffer());
+	if (!bf)
+		return 0;
+	out = BIO_push(bf, out);
+	if(flags & PKCS7_BINARY)
+		{
 		while((len = BIO_read(in, linebuf, MAX_SMLEN)) > 0)
 						BIO_write(out, linebuf, len);
-		return 1;
-	}
-	if(flags & PKCS7_TEXT)
-		BIO_printf(out, "Content-Type: text/plain\r\n\r\n");
-	while ((len = BIO_gets(in, linebuf, MAX_SMLEN)) > 0) {
-		eol = strip_eol(linebuf, &len);
-		if (len)
-			BIO_write(out, linebuf, len);
-		if(eol) BIO_write(out, "\r\n", 2);
-	}
+		}
+	else
+		{
+		if(flags & PKCS7_TEXT)
+			BIO_printf(out, "Content-Type: text/plain\r\n\r\n");
+		while ((len = BIO_gets(in, linebuf, MAX_SMLEN)) > 0)
+			{
+			eol = strip_eol(linebuf, &len);
+			if (len)
+				BIO_write(out, linebuf, len);
+			if(eol) BIO_write(out, "\r\n", 2);
+			}
+		}
+	BIO_flush(out);
+	BIO_pop(out);
+	BIO_free(bf);
 	return 1;
 }
 

crypto/pkcs7/pkcs7.h

@@ -287,6 +287,8 @@ int i2d_PKCS7_fp(FILE *fp,PKCS7 *p7);
 PKCS7 *PKCS7_dup(PKCS7 *p7);
 PKCS7 *d2i_PKCS7_bio(BIO *bp,PKCS7 **p7);
 int i2d_PKCS7_bio(BIO *bp,PKCS7 *p7);
+int i2d_PKCS7_bio_stream(BIO *out, PKCS7 *p7, BIO *in, int flags);
+int PEM_write_bio_PKCS7_stream(BIO *out, PKCS7 *p7, BIO *in, int flags);
 
 DECLARE_ASN1_FUNCTIONS(PKCS7_SIGNER_INFO)
 DECLARE_ASN1_FUNCTIONS(PKCS7_RECIP_INFO)