issue-8493: Fix for filenames with newlines using openssl dgst

The output format now matches coreutils *dgst tools.

[ edited to remove trailing white space ]
Reviewed-by: NRichard Levitte <levitte@openssl.org>
Reviewed-by: NPaul Dale <paul.dale@oracle.com>

(cherry picked from commit f3448f5481a8d1f6fbf5fd05caaca229af0b87f7)
Reviewed-by: NTomas Mraz <tmraz@fedoraproject.org>
Reviewed-by: NMatt Caswell <matt@openssl.org>
Reviewed-by: NMatthias St. Pierre <Matthias.St.Pierre@ncp-e.com>
(Merged from https://github.com/openssl/openssl/pull/10094)

CHANGES

@@ -9,6 +9,11 @@
 
  Changes between 1.1.1d and 1.1.1e [xx XXX xxxx]
 
+  *) Added newline escaping functionality to a filename when using openssl dgst.
+     This output format is to replicate the output format found in the '*sum'
+     checksum programs. This aims to preserve backward compatibility.
+     [Matt Eaton, Richard Levitte, and Paul Dale]
+
   *) Print all values for a PKCS#12 attribute with 'openssl pkcs12', not just
      the first value.
      [Jon Spillett]

apps/dgst.c

@@ -413,13 +413,52 @@ int dgst_main(int argc, char **argv)
     return ret;
 }
 
+/*
+ * The newline_escape_filename function performs newline escaping for any
+ * filename that contains a newline.  This function also takes a pointer
+ * to backslash. The backslash pointer is a flag to indicating whether a newline
+ * is present in the filename.  If a newline is present, the backslash flag is
+ * set and the output format will contain a backslash at the beginning of the
+ * digest output. This output format is to replicate the output format found
+ * in the '*sum' checksum programs. This aims to preserve backward
+ * compatibility.
+ */
+static const char *newline_escape_filename(const char *file, int * backslash)
+{
+    size_t i, e = 0, length = strlen(file), newline_count = 0, mem_len = 0;
+    char *file_cpy = NULL;
+
+    for (i = 0; i < length; i++)
+        if (file[i] == '\n')
+            newline_count++;
+
+    mem_len = length + newline_count + 1;
+    file_cpy = app_malloc(mem_len, file);
+    i = 0;
+
+    while(e < length) {
+        const char c = file[e];
+        if (c == '\n') {
+            file_cpy[i++] = '\\';
+            file_cpy[i++] = 'n';
+            *backslash = 1;
+        } else {
+            file_cpy[i++] = c;
+        }
+        e++;
+    }
+    file_cpy[i] = '\0';
+    return (const char*)file_cpy;
+}
+
+
 int do_fp(BIO *out, unsigned char *buf, BIO *bp, int sep, int binout,
           EVP_PKEY *key, unsigned char *sigin, int siglen,
           const char *sig_name, const char *md_name,
           const char *file)
 {
     size_t len;
-    int i;
+    int i, backslash = 0;
 
     while (BIO_pending(bp) || !BIO_eof(bp)) {
         i = BIO_read(bp, (char *)buf, BUFSIZE);
@@ -467,9 +506,16 @@ int do_fp(BIO *out, unsigned char *buf, BIO *bp, int sep, int binout,
     if (binout) {
         BIO_write(out, buf, len);
     } else if (sep == 2) {
+        file = newline_escape_filename(file, &backslash);
+
+        if (backslash == 1)
+            BIO_puts(out, "\\");
+
         for (i = 0; i < (int)len; i++)
             BIO_printf(out, "%02x", buf[i]);
+
         BIO_printf(out, " *%s\n", file);
+        OPENSSL_free((char *)file);
     } else {
         if (sig_name != NULL) {
             BIO_puts(out, sig_name);

doc/man1/dgst.pod

@@ -79,7 +79,8 @@ Output the digest or signature in binary form.
 
 =item B<-r>
 
-Output the digest in the "coreutils" format used by programs like B<sha1sum>.
+Output the digest in the "coreutils" format, including newlines.
+Used by programs like B<sha1sum>.
 
 =item B<-out filename>
 

test/README

@@ -114,7 +114,7 @@ Generic form of C test executables
         int observed;
 
         observed = function();              /* Call the code under test     */
-        if (!TEST_int_equal(observed, 2))   /* Check the result is correct  */
+        if (!TEST_int_eq(observed, 2))      /* Check the result is correct  */
             goto end;                       /* Exit on failure - optional   */
 
         testresult = 1;                     /* Mark the test case a success */