20260729 Coverity fixes

src/pk_rsa.c

@@ -651,6 +651,12 @@ static int wolfssl_read_der_bio(WOLFSSL_BIO* bio, unsigned char** out)
         WOLFSSL_ERROR_MSG("DER SEQUENCE decode failed");
         err = 1;
     }
+    /* Cap at 8x the maximum modulus size, leaves headroom for the full
+     * private key encoding. */
+    if ((!err) && (derLen > RSA_MAX_SIZE)) {
+        WOLFSSL_ERROR_MSG("DER length too large");
+        err = 1;
+    }
     /* Allocate a buffer to read DER data into. */
     if ((!err) && ((der = (unsigned char*)XMALLOC((size_t)derLen, bio->heap,
             DYNAMIC_TYPE_TMP_BUFFER)) == NULL)) {

src/ssl_sess.c

@@ -2165,13 +2165,37 @@ void AddSession(WOLFSSL* ssl)
         if (ssl->rng != NULL)
             rng = ssl->rng;
 #if defined(HAVE_GLOBAL_RNG) && defined(OPENSSL_EXTRA)
-        else if (initGlobalRNG == 1 || wolfSSL_RAND_Init() == WOLFSSL_SUCCESS) {
+        else if (initGlobalRNG == 1 ||
+                 wolfSSL_RAND_Init() == WOLFSSL_SUCCESS) {
             rng = &globalRNG;
         }
+        if (rng == &globalRNG) {
+            if (wc_LockMutex(&globalRNGMutex) != 0) {
+                WOLFSSL_MSG("Bad Lock Mutex rng");
+                return;
+            }
+            /* The above access requires initGlobalRNG recheck now
+             * that we have the lock. */
+            if (initGlobalRNG == 0) {
+                wc_UnLockMutex(&globalRNGMutex);
+                return;
+            }
+        }
 #endif
         if (wc_RNG_GenerateBlock(rng, ssl->session->altSessionID,
-                ID_LEN) != 0)
+                ID_LEN) != 0) {
+#if defined(HAVE_GLOBAL_RNG) && defined(OPENSSL_EXTRA)
+            if (rng == &globalRNG) {
+                wc_UnLockMutex(&globalRNGMutex);
+            }
+#endif
             return;
+        }
+#if defined(HAVE_GLOBAL_RNG) && defined(OPENSSL_EXTRA)
+        if (rng == &globalRNG) {
+            wc_UnLockMutex(&globalRNGMutex);
+        }
+#endif
         ssl->session->haveAltSessionID = 1;
         id = ssl->session->altSessionID;
         idSz = ID_LEN;

tests/api.c

@@ -20491,6 +20491,83 @@ static int test_wolfSSL_d2i_PrivateKeys_bio(void)
 }
 #endif /* OPENSSL_ALL || (WOLFSSL_ASIO && !NO_RSA) */
 
+/* Build gate for the hook + test so they can't drift: OPENSSL_EXTRA (compat
+ * BIO/RSA) + a d2i_RSAPrivateKey_bio profile + RSA/key-gen + swappable alloc.
+ * DEBUG_MEMORY excluded on purpose: the hook is single-arg only. */
+#if defined(OPENSSL_EXTRA) &&                                                 \
+    (defined(OPENSSL_ALL) || defined(WOLFSSL_ASIO) ||                         \
+     defined(WOLFSSL_HAPROXY) || defined(WOLFSSL_NGINX)) &&                   \
+    !defined(NO_RSA) && defined(WOLFSSL_KEY_GEN) &&                           \
+    defined(USE_WOLFSSL_MEMORY) && !defined(WOLFSSL_NO_MALLOC) &&             \
+    !defined(WOLFSSL_STATIC_MEMORY) && !defined(WOLFSSL_DEBUG_MEMORY)
+    #define TEST_DER_CAP_MALLOC_HOOK
+#endif
+
+#ifdef TEST_DER_CAP_MALLOC_HOOK
+/* Refuses allocations >= der_cap_threshold; forwards the rest to the installed
+ * allocator (native if none) so the alloc triple stays consistent. */
+static wolfSSL_Malloc_cb der_cap_prev_malloc = NULL;
+static size_t der_cap_threshold = 0; /* 0 = disabled */
+static int    der_cap_attempts  = 0;
+
+static void* der_cap_malloc_cb(size_t size)
+{
+    if (der_cap_threshold != 0 && size >= der_cap_threshold) {
+        der_cap_attempts++;
+        return NULL; /* refuse; records the attempt */
+    }
+    if (der_cap_prev_malloc != NULL)
+        return der_cap_prev_malloc(size);
+    return malloc(size);
+}
+#endif /* TEST_DER_CAP_MALLOC_HOOK */
+
+/* Oversized-DER cap must reject before allocating. NULL alone doesn't prove it
+ * (uncapped returns NULL too), so assert no large alloc was attempted. */
+static int test_wolfSSL_d2i_RSAPrivateKey_bio_oversized(void)
+{
+    EXPECT_DECLS;
+#ifdef TEST_DER_CAP_MALLOC_HOOK
+    /* SEQUENCE, canonical 4-byte length 0x01000000 (16 MB), over the cap.
+     * Must be canonical -- 0x00FFFFFF is rejected by the parser first. */
+    static const unsigned char hugeSeq[] =
+        { 0x30, 0x84, 0x01, 0x00, 0x00, 0x00 };
+    BIO* bio = NULL;
+    RSA* rsa = NULL;
+    wolfSSL_Malloc_cb  prev_mc = NULL;
+    wolfSSL_Free_cb    prev_fc = NULL;
+    wolfSSL_Realloc_cb prev_rc = NULL;
+    int allocators_set = 0;
+
+    ExpectIntEQ(wolfSSL_GetAllocators(&prev_mc, &prev_fc, &prev_rc), 0);
+    der_cap_prev_malloc = prev_mc;
+    /* Override only malloc; keep the installed free/realloc. */
+    ExpectIntEQ(wolfSSL_SetAllocators(der_cap_malloc_cb, prev_fc, prev_rc), 0);
+    if (EXPECT_SUCCESS())
+        allocators_set = 1;
+
+    ExpectNotNull(bio = BIO_new(BIO_s_mem()));
+    ExpectIntGT(BIO_write(bio, hugeSeq, (int)sizeof(hugeSeq)), 0);
+
+    der_cap_attempts = 0;
+    der_cap_threshold = 0x10000;
+    ExpectNull(d2i_RSAPrivateKey_bio(bio, &rsa));
+    der_cap_threshold = 0;
+
+    ExpectIntEQ(der_cap_attempts, 0);
+
+    BIO_free(bio);
+    RSA_free(rsa);
+
+    if (allocators_set)
+        (void)wolfSSL_SetAllocators(prev_mc, prev_fc, prev_rc);
+    der_cap_prev_malloc = NULL;
+#endif
+    return EXPECT_RESULT();
+}
+
+#undef TEST_DER_CAP_MALLOC_HOOK
+
 #endif /* !NO_BIO */
 
 
@@ -38233,6 +38310,9 @@ TEST_CASE testCases[] = {
     TEST_DECL(test_wolfSSL_d2i_PrivateKeys_bio),
 #endif /* !NO_BIO */
 #endif
+#ifndef NO_BIO
+    TEST_DECL(test_wolfSSL_d2i_RSAPrivateKey_bio_oversized),
+#endif /* !NO_BIO */
 
 
 #if !defined(NO_WOLFSSL_CLIENT) && !defined(NO_WOLFSSL_SERVER)

tests/api/test_coding.c

@@ -330,18 +330,22 @@ int test_wc_Base64_EncodeDecisionCoverage(void)
         byte enc[128];
         word32 i;
         int nlCount = 0;
+
+        XMEMSET(enc, 0, sizeof(enc));
         for (i = 0; i < (word32)sizeof(in48); i++)
             in48[i] = (byte)(i + 1);
         outLen = (word32)sizeof(enc);
         ExpectIntEQ(Base64_Encode(in48, (word32)sizeof(in48), enc, &outLen),
                     0);
-        for (i = 0; i < outLen; i++) {
-            if (enc[i] == '\n')
-                nlCount++;
+        if (EXPECT_SUCCESS()) {
+            for (i = 0; i < outLen; i++) {
+                if (enc[i] == '\n')
+                    nlCount++;
+            }
+            /* exactly one (trailing) newline -- none inserted mid-stream */
+            ExpectIntEQ(nlCount, 1);
+            ExpectIntEQ(enc[outLen - 1], '\n');
         }
-        /* exactly one (trailing) newline -- none inserted mid-stream */
-        ExpectIntEQ(nlCount, 1);
-        ExpectIntEQ(enc[outLen - 1], '\n');
     }
 
     /* --- force a BUFFER_E from CEscape() inside the *main* while loop