Fix memory checking.

CHANGES

@@ -4,6 +4,20 @@
 
  Changes between 0.9.3a and 0.9.4
 
+  *) Memory leak checking had some problems.  The interface is as follows:
+     Applications can use
+         CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON) aka MemCheck_start(),
+         CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_OFF) aka MemCheck_stop();
+     "off" is now the default.
+     The library internally uses
+         CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_DISABLE) aka MemCheck_off(),
+         CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ENABLE) aka MemCheck_on()
+     to disable memory-checking temporarily.
+
+     Some inconsistent states that previously were possible (and were
+     even the default) are now avoided.
+     [Bodo Moeller]
+
   *) Introduce "mode" for SSL structures (with defaults in SSL_CTX),
      which largely parallels "options", but is for changing API behaviour,
      whereas "options" are about protocol behaviour.

apps/openssl.c

@@ -136,7 +136,7 @@ int main(int Argc, char *Argv[])
 		if ((bio_err=BIO_new(BIO_s_file())) != NULL)
 			BIO_set_fp(bio_err,stderr,BIO_NOCLOSE|BIO_FP_TEXT);
 
-	CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ENABLE);
+	CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
 
 	ERR_load_crypto_strings();
 

crypto/dsa/dsatest.c

@@ -134,7 +134,7 @@ int main(int argc, char **argv)
 	if (bio_err == NULL)
 		bio_err=BIO_new_fp(stderr,BIO_NOCLOSE);
 
-	CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ENABLE);
+	CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
 
 	BIO_printf(bio_err,"test generation of DSA parameters\n");
 	BIO_printf(bio_err,"expect '.*' followed by 5 lines of '.'s and '+'s\n");

crypto/mem.c

@@ -63,11 +63,21 @@
 #include <openssl/lhash.h>
 #include "cryptlib.h"
 
-#ifdef CRYPTO_MDEBUG
-static int mh_mode=CRYPTO_MEM_CHECK_ON;
-#else
+/* #ifdef CRYPTO_MDEBUG */
+/* static int mh_mode=CRYPTO_MEM_CHECK_ON; */
+/* #else */
 static int mh_mode=CRYPTO_MEM_CHECK_OFF;
-#endif
+/* #endif */
+/* State CRYPTO_MEM_CHECK_ON exists only temporarily when the library
+ * thinks that certain allocations should not be checked (e.g. the data
+ * structures used for memory checking).  It is not suitable as an initial
+ * state: the library will unexpectedly enable memory checking when it
+ * executes one of those sections that want to disable checking
+ * temporarily.
+ *
+ * State CRYPTO_MEM_CHECK_ENABLE without ..._ON makes no sense whatsoever.
+ */
+
 static unsigned long order=0;
 
 static LHASH *mh=NULL;
@@ -88,19 +98,23 @@ int CRYPTO_mem_ctrl(int mode)
 	CRYPTO_w_lock(CRYPTO_LOCK_MALLOC);
 	switch (mode)
 		{
-	case CRYPTO_MEM_CHECK_ON:
-		mh_mode|=CRYPTO_MEM_CHECK_ON;
+	/* for applications: */
+	case CRYPTO_MEM_CHECK_ON: /* aka MemCheck_start() */
+		mh_mode = CRYPTO_MEM_CHECK_ON|CRYPTO_MEM_CHECK_ENABLE;
 		break;
-	case CRYPTO_MEM_CHECK_OFF:
-		mh_mode&= ~CRYPTO_MEM_CHECK_ON;
+	case CRYPTO_MEM_CHECK_OFF: /* aka MemCheck_stop() */
+		mh_mode = 0;
 		break;
-	case CRYPTO_MEM_CHECK_DISABLE:
+
+	/* switch off temporarily (for library-internal use): */
+	case CRYPTO_MEM_CHECK_DISABLE: /* aka MemCheck_off() */
 		mh_mode&= ~CRYPTO_MEM_CHECK_ENABLE;
 		break;
-	case CRYPTO_MEM_CHECK_ENABLE:
+	case CRYPTO_MEM_CHECK_ENABLE: /* aka MemCheck_on() */
 		if (mh_mode&CRYPTO_MEM_CHECK_ON)
 			mh_mode|=CRYPTO_MEM_CHECK_ENABLE;
 		break;
+
 	default:
 		break;
 		}

crypto/rsa/rsa_oaep_test.c

@@ -216,7 +216,7 @@ int main()
     int clen = 0;
     int num;
 
-    CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ENABLE);
+    CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
 	
     plen = sizeof(ptext_ex) - 1;
 

ssl/ssltest.c

@@ -163,7 +163,7 @@ int main(int argc, char *argv[])
 	bio_err=BIO_new_fp(stderr,BIO_NOCLOSE);
 	bio_stdout=BIO_new_fp(stdout,BIO_NOCLOSE);
 
-	CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ENABLE);
+	CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
 
 	argc--;
 	argv++;