Issue #29157: Prefer getrandom() over getentropy()

Copy and then adapt Python/random.c from default branch. Difference between 3.5
and default branches:

* Python 3.5 only uses getrandom() in non-blocking mode: flags=GRND_NONBLOCK
* If getrandom() fails with EAGAIN: py_getrandom() immediately fails and
  remembers that getrandom() doesn't work.
* Python 3.5 has no _PyOS_URandomNonblock() function: _PyOS_URandom()
  works in non-blocking mode on Python 3.5

Python/random.c

@@ -1,6 +1,9 @@
 #include "Python.h"
 #ifdef MS_WINDOWS
 #  include <windows.h>
+/* All sample MSDN wincrypt programs include the header below. It is at least
+ * required with Min GW. */
+#  include <wincrypt.h>
 #else
 #  include <fcntl.h>
 #  ifdef HAVE_SYS_STAT_H
@@ -37,10 +40,9 @@ win32_urandom_init(int raise)
     return 0;
 
 error:
-    if (raise)
+    if (raise) {
         PyErr_SetFromWindowsErr(0);
-    else
-        Py_FatalError("Failed to initialize Windows random API (CryptoGen)");
+    }
     return -1;
 }
 
@@ -53,8 +55,9 @@ win32_urandom(unsigned char *buffer, Py_ssize_t size, int raise)
 
     if (hCryptProv == 0)
     {
-        if (win32_urandom_init(raise) == -1)
+        if (win32_urandom_init(raise) == -1) {
             return -1;
+        }
     }
 
     while (size > 0)
@@ -63,11 +66,9 @@ win32_urandom(unsigned char *buffer, Py_ssize_t size, int raise)
         if (!CryptGenRandom(hCryptProv, (DWORD)chunk, buffer))
         {
             /* CryptGenRandom() failed */
-            if (raise)
+            if (raise) {
                 PyErr_SetFromWindowsErr(0);
-            else
-                Py_FatalError("Failed to initialized the randomized hash "
-                        "secret using CryptoGen)");
+            }
             return -1;
         }
         buffer += chunk;
@@ -76,58 +77,23 @@ win32_urandom(unsigned char *buffer, Py_ssize_t size, int raise)
     return 0;
 }
 
-/* Issue #25003: Don't use getentropy() on Solaris (available since
- * Solaris 11.3), it is blocking whereas os.urandom() should not block. */
-#elif defined(HAVE_GETENTROPY) && !defined(sun)
-#define PY_GETENTROPY 1
-
-/* Fill buffer with size pseudo-random bytes generated by getentropy().
-   Return 0 on success, or raise an exception and return -1 on error.
-
-   If fatal is nonzero, call Py_FatalError() instead of raising an exception
-   on error. */
-static int
-py_getentropy(unsigned char *buffer, Py_ssize_t size, int fatal)
-{
-    while (size > 0) {
-        Py_ssize_t len = Py_MIN(size, 256);
-        int res;
-
-        if (!fatal) {
-            Py_BEGIN_ALLOW_THREADS
-            res = getentropy(buffer, len);
-            Py_END_ALLOW_THREADS
-
-            if (res < 0) {
-                PyErr_SetFromErrno(PyExc_OSError);
-                return -1;
-            }
-        }
-        else {
-            res = getentropy(buffer, len);
-            if (res < 0)
-                Py_FatalError("getentropy() failed");
-        }
-
-        buffer += len;
-        size -= len;
-    }
-    return 0;
-}
-
-#else
+#else /* !MS_WINDOWS */
 
 #if defined(HAVE_GETRANDOM) || defined(HAVE_GETRANDOM_SYSCALL)
 #define PY_GETRANDOM 1
 
-/* Call getrandom()
+/* Call getrandom() to get random bytes:
+
    - Return 1 on success
-   - Return 0 if getrandom() syscall is not available (failed with ENOSYS or
-     EPERM) or if getrandom(GRND_NONBLOCK) failed with EAGAIN (system urandom
-     not initialized yet) and raise=0.
+   - Return 0 if getrandom() is not available (failed with ENOSYS or EPERM),
+     or if getrandom(GRND_NONBLOCK) failed with EAGAIN (system urandom not
+     initialized yet).
    - Raise an exception (if raise is non-zero) and return -1 on error:
-     getrandom() failed with EINTR and the Python signal handler raised an
-     exception, or getrandom() failed with a different error. */
+     if getrandom() failed with EINTR, raise is non-zero and the Python signal
+     handler raised an exception, or if getrandom() failed with a different
+     error.
+
+   getrandom() is retried if it failed with EINTR: interrupted by a signal. */
 static int
 py_getrandom(void *buffer, Py_ssize_t size, int raise)
 {
@@ -142,16 +108,19 @@ py_getrandom(void *buffer, Py_ssize_t size, int raise)
      * see https://bugs.python.org/issue26839. To avoid this, use the
      * GRND_NONBLOCK flag. */
     const int flags = GRND_NONBLOCK;
+    char *dest;
     long n;
 
     if (!getrandom_works) {
         return 0;
     }
 
+    dest = buffer;
     while (0 < size) {
 #ifdef sun
         /* Issue #26735: On Solaris, getrandom() is limited to returning up
-           to 1024 bytes */
+           to 1024 bytes. Call it multiple times if more bytes are
+           requested. */
         n = Py_MIN(size, 1024);
 #else
         n = Py_MIN(size, LONG_MAX);
@@ -161,34 +130,35 @@ py_getrandom(void *buffer, Py_ssize_t size, int raise)
 #ifdef HAVE_GETRANDOM
         if (raise) {
             Py_BEGIN_ALLOW_THREADS
-            n = getrandom(buffer, n, flags);
+            n = getrandom(dest, n, flags);
             Py_END_ALLOW_THREADS
         }
         else {
-            n = getrandom(buffer, n, flags);
+            n = getrandom(dest, n, flags);
         }
 #else
         /* On Linux, use the syscall() function because the GNU libc doesn't
-         * expose the Linux getrandom() syscall yet. See:
-         * https://sourceware.org/bugzilla/show_bug.cgi?id=17252 */
+           expose the Linux getrandom() syscall yet. See:
+           https://sourceware.org/bugzilla/show_bug.cgi?id=17252 */
         if (raise) {
             Py_BEGIN_ALLOW_THREADS
-            n = syscall(SYS_getrandom, buffer, n, flags);
+            n = syscall(SYS_getrandom, dest, n, flags);
             Py_END_ALLOW_THREADS
         }
         else {
-            n = syscall(SYS_getrandom, buffer, n, flags);
+            n = syscall(SYS_getrandom, dest, n, flags);
         }
 #endif
 
         if (n < 0) {
-            /* ENOSYS: getrandom() syscall not supported by the kernel (but
-             * maybe supported by the host which built Python). EPERM:
-             * getrandom() syscall blocked by SECCOMP or something else. */
+            /* ENOSYS: the syscall is not supported by the kernel.
+               EPERM: the syscall is blocked by a security policy (ex: SECCOMP)
+               or something else. */
             if (errno == ENOSYS || errno == EPERM) {
                 getrandom_works = 0;
                 return 0;
             }
+
             if (errno == EAGAIN) {
                 /* getrandom(GRND_NONBLOCK) fails with EAGAIN if the system
                    urandom is not initialiazed yet. In this case, fall back on
@@ -202,32 +172,101 @@ py_getrandom(void *buffer, Py_ssize_t size, int raise)
             }
 
             if (errno == EINTR) {
-                if (PyErr_CheckSignals()) {
-                    if (!raise) {
-                        Py_FatalError("getrandom() interrupted by a signal");
+                if (raise) {
+                    if (PyErr_CheckSignals()) {
+                        return -1;
                     }
-                    return -1;
                 }
 
-                /* retry getrandom() */
+                /* retry getrandom() if it was interrupted by a signal */
                 continue;
             }
 
             if (raise) {
                 PyErr_SetFromErrno(PyExc_OSError);
             }
-            else {
-                Py_FatalError("getrandom() failed");
-            }
             return -1;
         }
 
-        buffer += n;
+        dest += n;
         size -= n;
     }
     return 1;
 }
-#endif
+
+#elif defined(HAVE_GETENTROPY)
+#define PY_GETENTROPY 1
+
+/* Fill buffer with size pseudo-random bytes generated by getentropy():
+
+   - Return 1 on success
+   - Return 0 if getentropy() syscall is not available (failed with ENOSYS or
+     EPERM).
+   - Raise an exception (if raise is non-zero) and return -1 on error:
+     if getentropy() failed with EINTR, raise is non-zero and the Python signal
+     handler raised an exception, or if getentropy() failed with a different
+     error.
+
+   getentropy() is retried if it failed with EINTR: interrupted by a signal. */
+static int
+py_getentropy(char *buffer, Py_ssize_t size, int raise)
+{
+    /* Is getentropy() supported by the running kernel? Set to 0 if
+       getentropy() failed with ENOSYS or EPERM. */
+    static int getentropy_works = 1;
+
+    if (!getentropy_works) {
+        return 0;
+    }
+
+    while (size > 0) {
+        /* getentropy() is limited to returning up to 256 bytes. Call it
+           multiple times if more bytes are requested. */
+        Py_ssize_t len = Py_MIN(size, 256);
+        int res;
+
+        if (raise) {
+            Py_BEGIN_ALLOW_THREADS
+            res = getentropy(buffer, len);
+            Py_END_ALLOW_THREADS
+        }
+        else {
+            res = getentropy(buffer, len);
+        }
+
+        if (res < 0) {
+            /* ENOSYS: the syscall is not supported by the running kernel.
+               EPERM: the syscall is blocked by a security policy (ex: SECCOMP)
+               or something else. */
+            if (errno == ENOSYS || errno == EPERM) {
+                getentropy_works = 0;
+                return 0;
+            }
+
+            if (errno == EINTR) {
+                if (raise) {
+                    if (PyErr_CheckSignals()) {
+                        return -1;
+                    }
+                }
+
+                /* retry getentropy() if it was interrupted by a signal */
+                continue;
+            }
+
+            if (raise) {
+                PyErr_SetFromErrno(PyExc_OSError);
+            }
+            return -1;
+        }
+
+        buffer += len;
+        size -= len;
+    }
+    return 1;
+}
+#endif /* defined(HAVE_GETENTROPY) && !defined(sun) */
+
 
 static struct {
     int fd;
@@ -235,136 +274,123 @@ static struct {
     ino_t st_ino;
 } urandom_cache = { -1 };
 
+/* Read random bytes from the /dev/urandom device:
 
-/* Read 'size' random bytes from py_getrandom(). Fall back on reading from
-   /dev/urandom if getrandom() is not available.
+   - Return 0 on success
+   - Raise an exception (if raise is non-zero) and return -1 on error
 
-   Call Py_FatalError() on error. */
-static void
-dev_urandom_noraise(unsigned char *buffer, Py_ssize_t size)
-{
-    int fd;
-    Py_ssize_t n;
+   Possible causes of errors:
 
-    assert (0 < size);
+   - open() failed with ENOENT, ENXIO, ENODEV, EACCES: the /dev/urandom device
+     was not found. For example, it was removed manually or not exposed in a
+     chroot or container.
+   - open() failed with a different error
+   - fstat() failed
+   - read() failed or returned 0
 
-#ifdef PY_GETRANDOM
-    if (py_getrandom(buffer, size, 0) == 1) {
-        return;
-    }
-    /* getrandom() failed with ENOSYS or EPERM,
-       fall back on reading /dev/urandom */
-#endif
+   read() is retried if it failed with EINTR: interrupted by a signal.
 
-    fd = _Py_open_noraise("/dev/urandom", O_RDONLY);
-    if (fd < 0) {
-        Py_FatalError("Failed to open /dev/urandom");
-    }
+   The file descriptor of the device is kept open between calls to avoid using
+   many file descriptors when run in parallel from multiple threads:
+   see the issue #18756.
 
-    while (0 < size)
-    {
-        do {
-            n = read(fd, buffer, (size_t)size);
-        } while (n < 0 && errno == EINTR);
+   st_dev and st_ino fields of the file descriptor (from fstat()) are cached to
+   check if the file descriptor was replaced by a different file (which is
+   likely a bug in the application): see the issue #21207.
 
-        if (n <= 0) {
-            /* read() failed or returned 0 bytes */
-            Py_FatalError("Failed to read bytes from /dev/urandom");
-            break;
-        }
-        buffer += n;
-        size -= n;
-    }
-    close(fd);
-}
-
-/* Read 'size' random bytes from py_getrandom(). Fall back on reading from
-   /dev/urandom if getrandom() is not available.
-
-   Return 0 on success. Raise an exception and return -1 on error. */
+   If the file descriptor was closed or replaced, open a new file descriptor
+   but don't close the old file descriptor: it probably points to something
+   important for some third-party code. */
 static int
-dev_urandom_python(char *buffer, Py_ssize_t size)
+dev_urandom(char *buffer, Py_ssize_t size, int raise)
 {
     int fd;
     Py_ssize_t n;
-    struct _Py_stat_struct st;
-#ifdef PY_GETRANDOM
-    int res;
-#endif
 
-    if (size <= 0)
-        return 0;
+    if (raise) {
+        struct _Py_stat_struct st;
 
-#ifdef PY_GETRANDOM
-    res = py_getrandom(buffer, size, 1);
-    if (res < 0) {
-        return -1;
-    }
-    if (res == 1) {
-        return 0;
-    }
-    /* getrandom() failed with ENOSYS or EPERM,
-       fall back on reading /dev/urandom */
-#endif
-
-    if (urandom_cache.fd >= 0) {
-        /* Does the fd point to the same thing as before? (issue #21207) */
-        if (_Py_fstat_noraise(urandom_cache.fd, &st)
-            || st.st_dev != urandom_cache.st_dev
-            || st.st_ino != urandom_cache.st_ino) {
-            /* Something changed: forget the cached fd (but don't close it,
-               since it probably points to something important for some
-               third-party code). */
-            urandom_cache.fd = -1;
+        if (urandom_cache.fd >= 0) {
+            /* Does the fd point to the same thing as before? (issue #21207) */
+            if (_Py_fstat_noraise(urandom_cache.fd, &st)
+                || st.st_dev != urandom_cache.st_dev
+                || st.st_ino != urandom_cache.st_ino) {
+                /* Something changed: forget the cached fd (but don't close it,
+                   since it probably points to something important for some
+                   third-party code). */
+                urandom_cache.fd = -1;
+            }
         }
+        if (urandom_cache.fd >= 0)
+            fd = urandom_cache.fd;
+        else {
+            fd = _Py_open("/dev/urandom", O_RDONLY);
+            if (fd < 0) {
+                if (errno == ENOENT || errno == ENXIO ||
+                    errno == ENODEV || errno == EACCES) {
+                    PyErr_SetString(PyExc_NotImplementedError,
+                                    "/dev/urandom (or equivalent) not found");
+                }
+                /* otherwise, keep the OSError exception raised by _Py_open() */
+                return -1;
+            }
+            if (urandom_cache.fd >= 0) {
+                /* urandom_fd was initialized by another thread while we were
+                   not holding the GIL, keep it. */
+                close(fd);
+                fd = urandom_cache.fd;
+            }
+            else {
+                if (_Py_fstat(fd, &st)) {
+                    close(fd);
+                    return -1;
+                }
+                else {
+                    urandom_cache.fd = fd;
+                    urandom_cache.st_dev = st.st_dev;
+                    urandom_cache.st_ino = st.st_ino;
+                }
+            }
+        }
+
+        do {
+            n = _Py_read(fd, buffer, (size_t)size);
+            if (n == -1)
+                return -1;
+            if (n == 0) {
+                PyErr_Format(PyExc_RuntimeError,
+                        "Failed to read %zi bytes from /dev/urandom",
+                        size);
+                return -1;
+            }
+
+            buffer += n;
+            size -= n;
+        } while (0 < size);
     }
-    if (urandom_cache.fd >= 0)
-        fd = urandom_cache.fd;
     else {
-        fd = _Py_open("/dev/urandom", O_RDONLY);
+        fd = _Py_open_noraise("/dev/urandom", O_RDONLY);
         if (fd < 0) {
-            if (errno == ENOENT || errno == ENXIO ||
-                errno == ENODEV || errno == EACCES)
-                PyErr_SetString(PyExc_NotImplementedError,
-                                "/dev/urandom (or equivalent) not found");
-            /* otherwise, keep the OSError exception raised by _Py_open() */
             return -1;
         }
-        if (urandom_cache.fd >= 0) {
-            /* urandom_fd was initialized by another thread while we were
-               not holding the GIL, keep it. */
-            close(fd);
-            fd = urandom_cache.fd;
-        }
-        else {
-            if (_Py_fstat(fd, &st)) {
+
+        while (0 < size)
+        {
+            do {
+                n = read(fd, buffer, (size_t)size);
+            } while (n < 0 && errno == EINTR);
+
+            if (n <= 0) {
+                /* stop on error or if read(size) returned 0 */
                 close(fd);
                 return -1;
             }
-            else {
-                urandom_cache.fd = fd;
-                urandom_cache.st_dev = st.st_dev;
-                urandom_cache.st_ino = st.st_ino;
-            }
+
+            buffer += n;
+            size -= n;
         }
+        close(fd);
     }
-
-    do {
-        n = _Py_read(fd, buffer, (size_t)size);
-        if (n == -1) {
-            return -1;
-        }
-        if (n == 0) {
-            PyErr_Format(PyExc_RuntimeError,
-                    "Failed to read %zi bytes from /dev/urandom",
-                    size);
-            return -1;
-        }
-
-        buffer += n;
-        size -= n;
-    } while (0 < size);
-
     return 0;
 }
 
@@ -376,8 +402,8 @@ dev_urandom_close(void)
         urandom_cache.fd = -1;
     }
 }
+#endif /* !MS_WINDOWS */
 
-#endif
 
 /* Fill buffer with pseudo-random bytes generated by a linear congruent
    generator (LCG):
@@ -400,29 +426,98 @@ lcg_urandom(unsigned int x0, unsigned char *buffer, size_t size)
     }
 }
 
+/* Read random bytes:
+
+   - Return 0 on success
+   - Raise an exception (if raise is non-zero) and return -1 on error
+
+   Used sources of entropy ordered by preference, preferred source first:
+
+   - CryptGenRandom() on Windows
+   - getrandom() function (ex: Linux and Solaris): call py_getrandom()
+   - getentropy() function (ex: OpenBSD): call py_getentropy()
+   - /dev/urandom device
+
+   Read from the /dev/urandom device if getrandom() or getentropy() function
+   is not available or does not work.
+
+   Prefer getrandom() over getentropy() because getrandom() supports blocking
+   and non-blocking mode and Python requires non-blocking RNG at startup to
+   initialize its hash secret: see the PEP 524.
+
+   Prefer getrandom() and getentropy() over reading directly /dev/urandom
+   because these functions don't need file descriptors and so avoid ENFILE or
+   EMFILE errors (too many open files): see the issue #18756.
+
+   Only use RNG running in the kernel. They are more secure because it is
+   harder to get the internal state of a RNG running in the kernel land than a
+   RNG running in the user land. The kernel has a direct access to the hardware
+   and has access to hardware RNG, they are used as entropy sources.
+
+   Note: the OpenSSL RAND_pseudo_bytes() function does not automatically reseed
+   its RNG on fork(), two child processes (with the same pid) generate the same
+   random numbers: see issue #18747. Kernel RNGs don't have this issue,
+   they have access to good quality entropy sources.
+
+   If raise is zero:
+
+   - Don't raise an exception on error
+   - Don't call the Python signal handler (don't call PyErr_CheckSignals()) if
+     a function fails with EINTR: retry directly the interrupted function
+   - Don't release the GIL to call functions.
+*/
+static int
+pyurandom(void *buffer, Py_ssize_t size, int raise)
+{
+#if defined(PY_GETRANDOM) || defined(PY_GETENTROPY)
+    int res;
+#endif
+
+    if (size < 0) {
+        if (raise) {
+            PyErr_Format(PyExc_ValueError,
+                         "negative argument not allowed");
+        }
+        return -1;
+    }
+
+    if (size == 0) {
+        return 0;
+    }
+
+#ifdef MS_WINDOWS
+    return win32_urandom((unsigned char *)buffer, size, raise);
+#else
+
+#if defined(PY_GETRANDOM) || defined(PY_GETENTROPY)
+#ifdef PY_GETRANDOM
+    res = py_getrandom(buffer, size, raise);
+#else
+    res = py_getentropy(buffer, size, raise);
+#endif
+    if (res < 0) {
+        return -1;
+    }
+    if (res == 1) {
+        return 0;
+    }
+    /* getrandom() or getentropy() function is not available: failed with
+       ENOSYS, EPERM or EAGAIN. Fall back on reading from /dev/urandom. */
+#endif
+
+    return dev_urandom(buffer, size, raise);
+#endif
+}
+
 /* Fill buffer with size pseudo-random bytes from the operating system random
    number generator (RNG). It is suitable for most cryptographic purposes
    except long living private keys for asymmetric encryption.
 
-   Return 0 on success, raise an exception and return -1 on error. */
+   Return 0 on success. Raise an exception and return -1 on error. */
 int
 _PyOS_URandom(void *buffer, Py_ssize_t size)
 {
-    if (size < 0) {
-        PyErr_Format(PyExc_ValueError,
-                     "negative argument not allowed");
-        return -1;
-    }
-    if (size == 0)
-        return 0;
-
-#ifdef MS_WINDOWS
-    return win32_urandom((unsigned char *)buffer, size, 1);
-#elif defined(PY_GETENTROPY)
-    return py_getentropy(buffer, size, 0);
-#else
-    return dev_urandom_python((char*)buffer, size);
-#endif
+    return pyurandom(buffer, size, 1);
 }
 
 void
@@ -463,13 +558,14 @@ _PyRandom_Init(void)
         }
     }
     else {
-#ifdef MS_WINDOWS
-        (void)win32_urandom(secret, secret_size, 0);
-#elif defined(PY_GETENTROPY)
-        (void)py_getentropy(secret, secret_size, 1);
-#else
-        dev_urandom_noraise(secret, secret_size);
-#endif
+        int res;
+
+        /* _PyRandom_Init() is called very early in the Python initialization
+           and so exceptions cannot be used (use raise=0). */
+        res = pyurandom(secret, secret_size, 0);
+        if (res < 0) {
+            Py_FatalError("failed to get random numbers to initialize Python");
+        }
     }
 }
 
@@ -481,8 +577,6 @@ _PyRandom_Fini(void)
         CryptReleaseContext(hCryptProv, 0);
         hCryptProv = 0;
     }
-#elif defined(PY_GETENTROPY)
-    /* nothing to clean */
 #else
     dev_urandom_close();
 #endif