diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index 9bddfe4dce2..865a9e5d2bf 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -259,6 +259,7 @@ Create, Finish, Discard If *size* is greater than zero, allocate *size* bytes, and set the writer size to *size*. The caller is responsible to write *size* bytes using :c:func:`PyBytesWriter_GetData`. + This function does not overallocate. On error, set an exception and return ``NULL``. @@ -349,6 +350,8 @@ Low-level API Resize the writer to *size* bytes. It can be used to enlarge or to shrink the writer. + This function typically overallocates to achieve amortized performance when + resizing multiple times. Newly allocated bytes are left uninitialized. @@ -360,6 +363,8 @@ Low-level API .. c:function:: int PyBytesWriter_Grow(PyBytesWriter *writer, Py_ssize_t grow) Resize the writer by adding *grow* bytes to the current writer size. + This function typically overallocates to achieve amortized performance when + resizing multiple times. Newly allocated bytes are left uninitialized. diff --git a/Doc/deprecations/pending-removal-in-3.20.rst b/Doc/deprecations/pending-removal-in-3.20.rst index 8bc863b185d..c86979c8ff9 100644 --- a/Doc/deprecations/pending-removal-in-3.20.rst +++ b/Doc/deprecations/pending-removal-in-3.20.rst @@ -19,5 +19,6 @@ Pending removal in Python 3.20 - :mod:`tabnanny` - :mod:`tkinter.font` - :mod:`tkinter.ttk` + - :mod:`zlib` - (Contributed by Hugo van Kemenade in :gh:`76007`.) + (Contributed by Hugo van Kemenade and Stan Ulbrych in :gh:`76007`.) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index a5843f3fc0d..1ac87b32bad 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -3383,6 +3383,214 @@ features: Added the :attr:`st_birthtime` member on Windows. +.. function:: statx(path, mask, *, flags=0, dir_fd=None, follow_symlinks=True) + + Get the status of a file or file descriptor by performing a :c:func:`!statx` + system call on the given path. + + *path* is a :term:`path-like object` or an open file descriptor. *mask* is a + combination of the module-level :const:`STATX_* ` constants + specifying the information to retrieve. *flags* is a combination of the + module-level :const:`AT_STATX_* ` constants and/or + :const:`AT_NO_AUTOMOUNT`. Returns a :class:`statx_result` object whose + :attr:`~os.statx_result.stx_mask` attribute specifies the information + actually retrieved (which may differ from *mask*). + + This function supports :ref:`specifying a file descriptor `, + :ref:`paths relative to directory descriptors `, and + :ref:`not following symlinks `. + + .. seealso:: The :manpage:`statx(2)` man page. + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: next + + +.. class:: statx_result + + Information about a file returned by :func:`os.statx`. + + :class:`!statx_result` has all the attributes that :class:`~stat_result` has + on Linux, making it :term:`duck-typing` compatible, but + :class:`!statx_result` is not a subclass of :class:`~stat_result` and cannot + be used as a tuple. + + :class:`!statx_result` has the following additional attributes: + + .. attribute:: stx_mask + + Bitmask of :const:`STATX_* ` constants specifying the + information retrieved, which may differ from what was requested. + + .. attribute:: stx_attributes_mask + + Bitmask of :const:`STATX_ATTR_* ` constants + specifying the attributes bits supported for this file. + + .. attribute:: stx_attributes + + Bitmask of :const:`STATX_ATTR_* ` constants + specifying the attributes of this file. + + .. attribute:: stx_dev_major + + Major number of the device on which this file resides. + + .. attribute:: stx_dev_minor + + Minor number of the device on which this file resides. + + .. attribute:: stx_rdev_major + + Major number of the device this file represents. + + .. attribute:: stx_rdev_minor + + Minor number of the device this file represents. + + .. attribute:: stx_mnt_id + + Mount ID. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 5.8. + + .. attribute:: stx_dio_mem_align + + Direct I/O memory buffer alignment requirement. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.1. + + .. attribute:: stx_dio_offset_align + + Direct I/O file offset alignment requirement. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.1. + + .. attribute:: stx_subvol + + Subvolume ID. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.10. + + .. attribute:: stx_atomic_write_unit_min + + Minimum size for direct I/O with torn-write protection. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.11. + + .. attribute:: stx_atomic_write_unit_max + + Maximum size for direct I/O with torn-write protection. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.11. + + .. attribute:: stx_atomic_write_unit_max_opt + + Maximum optimized size for direct I/O with torn-write protection. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.11. + + .. attribute:: stx_atomic_write_segments_max + + Maximum iovecs for direct I/O with torn-write protection. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.11. + + .. attribute:: stx_dio_read_offset_align + + Direct I/O file offset alignment requirement for reads. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.14. + + .. seealso:: The :manpage:`statx(2)` man page. + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: next + + +.. data:: STATX_TYPE + STATX_MODE + STATX_NLINK + STATX_UID + STATX_GID + STATX_ATIME + STATX_MTIME + STATX_CTIME + STATX_INO + STATX_SIZE + STATX_BLOCKS + STATX_BASIC_STATS + STATX_BTIME + STATX_MNT_ID + STATX_DIOALIGN + STATX_MNT_ID_UNIQUE + STATX_SUBVOL + STATX_WRITE_ATOMIC + STATX_DIO_READ_ALIGN + + Bitflags for use in the *mask* parameter to :func:`os.statx`. Flags + including and after :const:`!STATX_MNT_ID` are only available when their + corresponding members in :class:`statx_result` are available. + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: next + +.. data:: AT_STATX_FORCE_SYNC + + A flag for the :func:`os.statx` function. Requests that the kernel return + up-to-date information even when doing so is expensive (for example, + requiring a round trip to the server for a file on a network filesystem). + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: next + +.. data:: AT_STATX_DONT_SYNC + + A flag for the :func:`os.statx` function. Requests that the kernel return + cached information if possible. + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: next + +.. data:: AT_STATX_SYNC_AS_STAT + + A flag for the :func:`os.statx` function. This flag is defined as ``0``, so + it has no effect, but it can be used to explicitly indicate neither + :data:`AT_STATX_FORCE_SYNC` nor :data:`AT_STATX_DONT_SYNC` is being passed. + In the absence of the other two flags, the kernel will generally return + information as fresh as :func:`os.stat` would return. + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: next + + +.. data:: AT_NO_AUTOMOUNT + + If the final component of a path is an automount point, operate on the + automount point instead of performing the automount. On Linux, + :func:`os.stat`, :func:`os.fstat` and :func:`os.lstat` always behave this + way. + + .. availability:: Linux. + + .. versionadded:: next + + .. function:: statvfs(path) Perform a :c:func:`!statvfs` system call on the given path. The return value is diff --git a/Doc/library/stat.rst b/Doc/library/stat.rst index 8434b2e8c75..1cbec3ab847 100644 --- a/Doc/library/stat.rst +++ b/Doc/library/stat.rst @@ -493,3 +493,22 @@ constants, but are not an exhaustive list. IO_REPARSE_TAG_APPEXECLINK .. versionadded:: 3.8 + +On Linux, the following file attribute constants are available for use when +testing bits in the :attr:`~os.statx_result.stx_attributes` and +:attr:`~os.statx_result.stx_attributes_mask` members returned by +:func:`os.statx`. See the :manpage:`statx(2)` man page for more detail on the +meaning of these constants. + +.. data:: STATX_ATTR_COMPRESSED + STATX_ATTR_IMMUTABLE + STATX_ATTR_APPEND + STATX_ATTR_NODUMP + STATX_ATTR_ENCRYPTED + STATX_ATTR_AUTOMOUNT + STATX_ATTR_MOUNT_ROOT + STATX_ATTR_VERITY + STATX_ATTR_DAX + STATX_ATTR_WRITE_ATOMIC + + .. versionadded:: next diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index c5321ee9983..56028a92aa2 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -433,6 +433,14 @@ mmap (Contributed by Serhiy Storchaka in :gh:`78502`.) +os +-- + +* Add :func:`os.statx` on Linux kernel versions 4.11 and later with + glibc versions 2.28 and later. + (Contributed by Jeffrey Bosboom in :gh:`83714`.) + + os.path ------- @@ -828,8 +836,9 @@ New deprecations - :mod:`tabnanny` - :mod:`tkinter.font` - :mod:`tkinter.ttk` + - :mod:`zlib` - (Contributed by Hugo van Kemenade in :gh:`76007`.) + (Contributed by Hugo van Kemenade and Stan Ulbrych in :gh:`76007`.) .. Add deprecations above alphabetically, not here at the end. diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 1f6b27b14d0..f7416c5ffc5 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1867,6 +1867,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(loop)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(manual_reset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mapping)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mask)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(match)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(max_length)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxdigits)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 6959343947c..ca71c12836d 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -590,6 +590,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(loop) STRUCT_FOR_ID(manual_reset) STRUCT_FOR_ID(mapping) + STRUCT_FOR_ID(mask) STRUCT_FOR_ID(match) STRUCT_FOR_ID(max_length) STRUCT_FOR_ID(maxdigits) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index be4eae42b5d..72996db9f71 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1865,6 +1865,7 @@ extern "C" { INIT_ID(loop), \ INIT_ID(manual_reset), \ INIT_ID(mapping), \ + INIT_ID(mask), \ INIT_ID(match), \ INIT_ID(max_length), \ INIT_ID(maxdigits), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 45b00a20a07..c4cf56ad7f1 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -2148,6 +2148,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(mask); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(match); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Lib/os.py b/Lib/os.py index 328d13c303b..6e6db96b307 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -136,6 +136,8 @@ def _add(str, fn): _add("HAVE_UNLINKAT", "unlink") _add("HAVE_UNLINKAT", "rmdir") _add("HAVE_UTIMENSAT", "utime") + if _exists("statx"): + _set.add(statx) supports_dir_fd = _set _set = set() @@ -157,6 +159,8 @@ def _add(str, fn): _add("HAVE_FPATHCONF", "pathconf") if _exists("statvfs") and _exists("fstatvfs"): # mac os x10.3 _add("HAVE_FSTATVFS", "statvfs") + if _exists("statx"): + _set.add(statx) supports_fd = _set _set = set() @@ -195,6 +199,8 @@ def _add(str, fn): _add("HAVE_FSTATAT", "stat") _add("HAVE_UTIMENSAT", "utime") _add("MS_WINDOWS", "stat") + if _exists("statx"): + _set.add(statx) supports_follow_symlinks = _set del _set diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index e0d4583f0a1..7a0f739a542 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -642,9 +642,14 @@ def sample( if output_format == "pstats" and not filename: stats = pstats.SampledStats(collector).strip_dirs() - print_sampled_stats( - stats, sort, limit, show_summary, sample_interval_usec - ) + if not stats.stats: + print("No samples were collected.") + if mode == PROFILING_MODE_CPU: + print("This can happen in CPU mode when all threads are idle.") + else: + print_sampled_stats( + stats, sort, limit, show_summary, sample_interval_usec + ) else: collector.export(filename) diff --git a/Lib/pstats.py b/Lib/pstats.py index 079abd2c1b8..07ecda07796 100644 --- a/Lib/pstats.py +++ b/Lib/pstats.py @@ -154,6 +154,7 @@ def load_stats(self, arg): arg.create_stats() self.stats = arg.stats arg.stats = {} + return if not self.stats: raise TypeError("Cannot create or construct a %r object from %r" % (self.__class__, arg)) diff --git a/Lib/stat.py b/Lib/stat.py index 1b4ed1ebc94..ab1b25b9d63 100644 --- a/Lib/stat.py +++ b/Lib/stat.py @@ -200,6 +200,21 @@ def filemode(mode): FILE_ATTRIBUTE_VIRTUAL = 65536 +# Linux STATX_ATTR constants for interpreting os.statx()'s +# "stx_attributes" and "stx_attributes_mask" members + +STATX_ATTR_COMPRESSED = 0x00000004 +STATX_ATTR_IMMUTABLE = 0x00000010 +STATX_ATTR_APPEND = 0x00000020 +STATX_ATTR_NODUMP = 0x00000040 +STATX_ATTR_ENCRYPTED = 0x00000800 +STATX_ATTR_AUTOMOUNT = 0x00001000 +STATX_ATTR_MOUNT_ROOT = 0x00002000 +STATX_ATTR_VERITY = 0x00100000 +STATX_ATTR_DAX = 0x00200000 +STATX_ATTR_WRITE_ATOMIC = 0x00400000 + + # If available, use C implementation try: from _stat import * diff --git a/Lib/test/test_atexit.py b/Lib/test/test_atexit.py index 66142a108d5..8256ff183f2 100644 --- a/Lib/test/test_atexit.py +++ b/Lib/test/test_atexit.py @@ -1,9 +1,11 @@ import atexit import os +import subprocess import textwrap import unittest from test import support -from test.support import script_helper +from test.support import SuppressCrashReport, script_helper +from test.support import os_helper from test.support import threading_helper class GeneralTest(unittest.TestCase): @@ -189,6 +191,37 @@ def callback(): self.assertEqual(os.read(r, len(expected)), expected) os.close(r) + # Python built with Py_TRACE_REFS fail with a fatal error in + # _PyRefchain_Trace() on memory allocation error. + @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') + def test_atexit_with_low_memory(self): + # gh-140080: Test that setting low memory after registering an atexit + # callback doesn't cause an infinite loop during finalization. + code = textwrap.dedent(""" + import atexit + import _testcapi + + def callback(): + print("hello") + + atexit.register(callback) + # Simulate low memory condition + _testcapi.set_nomemory(0) + """) + + with os_helper.temp_dir() as temp_dir: + script = script_helper.make_script(temp_dir, 'test_atexit_script', code) + with SuppressCrashReport(): + with script_helper.spawn_python(script, + stderr=subprocess.PIPE) as proc: + proc.wait() + stdout = proc.stdout.read() + stderr = proc.stderr.read() + + self.assertIn(proc.returncode, (0, 1)) + self.assertNotIn(b"hello", stdout) + self.assertIn(b"MemoryError", stderr) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_os/test_os.py b/Lib/test/test_os/test_os.py index 86880a6d281..dd6f89e81aa 100644 --- a/Lib/test/test_os/test_os.py +++ b/Lib/test/test_os/test_os.py @@ -630,6 +630,14 @@ def setUp(self): self.addCleanup(os_helper.unlink, self.fname) create_file(self.fname, b"ABC") + def check_timestamp_agreement(self, result, names): + # Make sure that the st_?time and st_?time_ns fields roughly agree + # (they should always agree up to around tens-of-microseconds) + for name in names: + floaty = int(getattr(result, name) * 100_000) + nanosecondy = getattr(result, name + "_ns") // 10_000 + self.assertAlmostEqual(floaty, nanosecondy, delta=2, msg=name) + def check_stat_attributes(self, fname): result = os.stat(fname) @@ -650,21 +658,15 @@ def trunc(x): return x result[getattr(stat, name)]) self.assertIn(attr, members) - # Make sure that the st_?time and st_?time_ns fields roughly agree - # (they should always agree up to around tens-of-microseconds) - for name in 'st_atime st_mtime st_ctime'.split(): - floaty = int(getattr(result, name) * 100000) - nanosecondy = getattr(result, name + "_ns") // 10000 - self.assertAlmostEqual(floaty, nanosecondy, delta=2) - - # Ensure both birthtime and birthtime_ns roughly agree, if present + time_attributes = ['st_atime', 'st_mtime', 'st_ctime'] try: - floaty = int(result.st_birthtime * 100000) - nanosecondy = result.st_birthtime_ns // 10000 + result.st_birthtime + result.st_birthtime_ns except AttributeError: pass else: - self.assertAlmostEqual(floaty, nanosecondy, delta=2) + time_attributes.append('st_birthtime') + self.check_timestamp_agreement(result, time_attributes) try: result[200] @@ -725,6 +727,88 @@ def test_stat_result_pickle(self): unpickled = pickle.loads(p) self.assertEqual(result, unpickled) + def check_statx_attributes(self, fname): + maximal_mask = 0 + for name in dir(os): + if name.startswith('STATX_'): + maximal_mask |= getattr(os, name) + result = os.statx(self.fname, maximal_mask) + + time_attributes = ('st_atime', 'st_mtime', 'st_ctime', 'st_birthtime') + self.check_timestamp_agreement(result, time_attributes) + + # Check that valid attributes match os.stat. + requirements = ( + ('st_mode', os.STATX_TYPE | os.STATX_MODE), + ('st_nlink', os.STATX_NLINK), + ('st_uid', os.STATX_UID), + ('st_gid', os.STATX_GID), + ('st_atime', os.STATX_ATIME), + ('st_atime_ns', os.STATX_ATIME), + ('st_mtime', os.STATX_MTIME), + ('st_mtime_ns', os.STATX_MTIME), + ('st_ctime', os.STATX_CTIME), + ('st_ctime_ns', os.STATX_CTIME), + ('st_ino', os.STATX_INO), + ('st_size', os.STATX_SIZE), + ('st_blocks', os.STATX_BLOCKS), + ('st_birthtime', os.STATX_BTIME), + ('st_birthtime_ns', os.STATX_BTIME), + # unconditionally valid members + ('st_blksize', 0), + ('st_rdev', 0), + ('st_dev', 0), + ) + basic_result = os.stat(self.fname) + for name, bits in requirements: + if result.stx_mask & bits == bits and hasattr(basic_result, name): + x = getattr(result, name) + b = getattr(basic_result, name) + self.assertEqual(type(x), type(b)) + if isinstance(x, float): + self.assertAlmostEqual(x, b, msg=name) + else: + self.assertEqual(x, b, msg=name) + + self.assertEqual(result.stx_rdev_major, os.major(result.st_rdev)) + self.assertEqual(result.stx_rdev_minor, os.minor(result.st_rdev)) + self.assertEqual(result.stx_dev_major, os.major(result.st_dev)) + self.assertEqual(result.stx_dev_minor, os.minor(result.st_dev)) + + members = [name for name in dir(result) + if name.startswith('st_') or name.startswith('stx_')] + for name in members: + try: + setattr(result, name, 1) + self.fail("No exception raised") + except AttributeError: + pass + + self.assertEqual(result.stx_attributes & result.stx_attributes_mask, + result.stx_attributes) + + # statx_result is not a tuple or tuple-like object. + with self.assertRaisesRegex(TypeError, 'not subscriptable'): + result[0] + with self.assertRaisesRegex(TypeError, 'cannot unpack'): + _, _ = result + + @unittest.skipUnless(hasattr(os, 'statx'), 'test needs os.statx()') + def test_statx_attributes(self): + self.check_statx_attributes(self.fname) + + @unittest.skipUnless(hasattr(os, 'statx'), 'test needs os.statx()') + def test_statx_attributes_bytes(self): + try: + fname = self.fname.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + self.skipTest("cannot encode %a for the filesystem" % self.fname) + self.check_statx_attributes(fname) + + @unittest.skipUnless(hasattr(os, 'statx'), 'test needs os.statx()') + def test_statx_attributes_pathlike(self): + self.check_statx_attributes(FakePath(self.fname)) + @unittest.skipUnless(hasattr(os, 'statvfs'), 'test needs os.statvfs()') def test_statvfs_attributes(self): result = os.statvfs(self.fname) diff --git a/Lib/test/test_os/test_posix.py b/Lib/test/test_os/test_posix.py index ab3d128d08a..905f0201253 100644 --- a/Lib/test/test_os/test_posix.py +++ b/Lib/test/test_os/test_posix.py @@ -668,22 +668,65 @@ def test_fstat(self): finally: fp.close() - def test_stat(self): - self.assertTrue(posix.stat(os_helper.TESTFN)) - self.assertTrue(posix.stat(os.fsencode(os_helper.TESTFN))) + def check_statlike_path(self, func): + self.assertTrue(func(os_helper.TESTFN)) + self.assertTrue(func(os.fsencode(os_helper.TESTFN))) + self.assertTrue(func(os_helper.FakePath(os_helper.TESTFN))) self.assertRaisesRegex(TypeError, 'should be string, bytes, os.PathLike or integer, not', - posix.stat, bytearray(os.fsencode(os_helper.TESTFN))) + func, bytearray(os.fsencode(os_helper.TESTFN))) self.assertRaisesRegex(TypeError, 'should be string, bytes, os.PathLike or integer, not', - posix.stat, None) + func, None) self.assertRaisesRegex(TypeError, 'should be string, bytes, os.PathLike or integer, not', - posix.stat, list(os_helper.TESTFN)) + func, list(os_helper.TESTFN)) self.assertRaisesRegex(TypeError, 'should be string, bytes, os.PathLike or integer, not', - posix.stat, list(os.fsencode(os_helper.TESTFN))) + func, list(os.fsencode(os_helper.TESTFN))) + + def test_stat(self): + self.check_statlike_path(posix.stat) + + @unittest.skipUnless(hasattr(posix, 'statx'), 'test needs posix.statx()') + def test_statx(self): + def func(path, **kwargs): + return posix.statx(path, posix.STATX_BASIC_STATS, **kwargs) + self.check_statlike_path(func) + + @unittest.skipUnless(hasattr(posix, 'statx'), 'test needs posix.statx()') + def test_statx_flags(self): + # glibc's fallback implementation of statx via the stat family fails + # with EINVAL on the (nonzero) sync flags. If you see this failure, + # update your kernel and/or seccomp syscall filter. + valid_flag_names = ('AT_NO_AUTOMOUNT', 'AT_STATX_SYNC_AS_STAT', + 'AT_STATX_FORCE_SYNC', 'AT_STATX_DONT_SYNC') + for flag_name in valid_flag_names: + flag = getattr(posix, flag_name) + with self.subTest(msg=flag_name, flags=flag): + posix.statx(os_helper.TESTFN, posix.STATX_BASIC_STATS, + flags=flag) + + # These flags are not exposed to Python because their functionality is + # implemented via kwargs instead. + kwarg_equivalent_flags = ( + (0x0100, 'AT_SYMLINK_NOFOLLOW', 'follow_symlinks'), + (0x0400, 'AT_SYMLINK_FOLLOW', 'follow_symlinks'), + (0x1000, 'AT_EMPTY_PATH', 'dir_fd'), + ) + for flag, flag_name, kwarg_name in kwarg_equivalent_flags: + with self.subTest(msg=flag_name, flags=flag): + with self.assertRaisesRegex(ValueError, kwarg_name): + posix.statx(os_helper.TESTFN, posix.STATX_BASIC_STATS, + flags=flag) + + with self.subTest(msg="AT_STATX_FORCE_SYNC | AT_STATX_DONT_SYNC"): + with self.assertRaises(OSError) as ctx: + flags = posix.AT_STATX_FORCE_SYNC | posix.AT_STATX_DONT_SYNC + posix.statx(os_helper.TESTFN, posix.STATX_BASIC_STATS, + flags=flags) + self.assertEqual(ctx.exception.errno, errno.EINVAL) @unittest.skipUnless(hasattr(posix, 'mkfifo'), "don't have mkfifo()") def test_mkfifo(self): @@ -1629,33 +1672,42 @@ def test_chown_dir_fd(self): with self.prepare_file() as (dir_fd, name, fullname): posix.chown(name, os.getuid(), os.getgid(), dir_fd=dir_fd) - @unittest.skipUnless(os.stat in os.supports_dir_fd, "test needs dir_fd support in os.stat()") - def test_stat_dir_fd(self): + def check_statlike_dir_fd(self, func): with self.prepare() as (dir_fd, name, fullname): with open(fullname, 'w') as outfile: outfile.write("testline\n") self.addCleanup(posix.unlink, fullname) - s1 = posix.stat(fullname) - s2 = posix.stat(name, dir_fd=dir_fd) - self.assertEqual(s1, s2) - s2 = posix.stat(fullname, dir_fd=None) - self.assertEqual(s1, s2) + s1 = func(fullname) + s2 = func(name, dir_fd=dir_fd) + self.assertEqual((s1.st_dev, s1.st_ino), (s2.st_dev, s2.st_ino)) + s2 = func(fullname, dir_fd=None) + self.assertEqual((s1.st_dev, s1.st_ino), (s2.st_dev, s2.st_ino)) self.assertRaisesRegex(TypeError, 'should be integer or None, not', - posix.stat, name, dir_fd=posix.getcwd()) + func, name, dir_fd=posix.getcwd()) self.assertRaisesRegex(TypeError, 'should be integer or None, not', - posix.stat, name, dir_fd=float(dir_fd)) + func, name, dir_fd=float(dir_fd)) self.assertRaises(OverflowError, - posix.stat, name, dir_fd=10**20) + func, name, dir_fd=10**20) for fd in False, True: with self.assertWarnsRegex(RuntimeWarning, 'bool is used as a file descriptor') as cm: with self.assertRaises(OSError): - posix.stat('nonexisting', dir_fd=fd) + func('nonexisting', dir_fd=fd) self.assertEqual(cm.filename, __file__) + @unittest.skipUnless(os.stat in os.supports_dir_fd, "test needs dir_fd support in os.stat()") + def test_stat_dir_fd(self): + self.check_statlike_dir_fd(posix.stat) + + @unittest.skipUnless(hasattr(posix, 'statx'), "test needs os.statx()") + def test_statx_dir_fd(self): + def func(path, **kwargs): + return posix.statx(path, os.STATX_INO, **kwargs) + self.check_statlike_dir_fd(func) + @unittest.skipUnless(os.utime in os.supports_dir_fd, "test needs dir_fd support in os.utime()") def test_utime_dir_fd(self): with self.prepare_file() as (dir_fd, name, fullname): diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py index a1342cafff1..59bc18b9bcf 100644 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler.py @@ -11,6 +11,7 @@ import sys import tempfile import unittest +from collections import namedtuple from unittest import mock from profiling.sampling.pstats_collector import PstatsCollector @@ -84,6 +85,8 @@ def __repr__(self): "Test only runs on Linux, Windows and MacOS", ) +SubprocessInfo = namedtuple('SubprocessInfo', ['process', 'socket']) + @contextlib.contextmanager def test_subprocess(script): @@ -123,7 +126,7 @@ def test_subprocess(script): if response != b"ready": raise RuntimeError(f"Unexpected response from subprocess: {response}") - yield proc + yield SubprocessInfo(proc, client_socket) finally: if client_socket is not None: client_socket.close() @@ -1752,13 +1755,13 @@ def main_loop(): def test_sampling_basic_functionality(self): with ( - test_subprocess(self.test_script) as proc, + test_subprocess(self.test_script) as subproc, io.StringIO() as captured_output, mock.patch("sys.stdout", captured_output), ): try: profiling.sampling.sample.sample( - proc.pid, + subproc.process.pid, duration_sec=2, sample_interval_usec=1000, # 1ms show_summary=False, @@ -1782,7 +1785,7 @@ def test_sampling_with_pstats_export(self): ) self.addCleanup(close_and_unlink, pstats_out) - with test_subprocess(self.test_script) as proc: + with test_subprocess(self.test_script) as subproc: # Suppress profiler output when testing file export with ( io.StringIO() as captured_output, @@ -1790,7 +1793,7 @@ def test_sampling_with_pstats_export(self): ): try: profiling.sampling.sample.sample( - proc.pid, + subproc.process.pid, duration_sec=1, filename=pstats_out.name, sample_interval_usec=10000, @@ -1826,7 +1829,7 @@ def test_sampling_with_collapsed_export(self): self.addCleanup(close_and_unlink, collapsed_file) with ( - test_subprocess(self.test_script) as proc, + test_subprocess(self.test_script) as subproc, ): # Suppress profiler output when testing file export with ( @@ -1835,7 +1838,7 @@ def test_sampling_with_collapsed_export(self): ): try: profiling.sampling.sample.sample( - proc.pid, + subproc.process.pid, duration_sec=1, filename=collapsed_file.name, output_format="collapsed", @@ -1876,14 +1879,14 @@ def test_sampling_with_collapsed_export(self): def test_sampling_all_threads(self): with ( - test_subprocess(self.test_script) as proc, + test_subprocess(self.test_script) as subproc, # Suppress profiler output io.StringIO() as captured_output, mock.patch("sys.stdout", captured_output), ): try: profiling.sampling.sample.sample( - proc.pid, + subproc.process.pid, duration_sec=1, all_threads=True, sample_interval_usec=10000, @@ -1969,14 +1972,14 @@ def test_invalid_pid(self): profiling.sampling.sample.sample(-1, duration_sec=1) def test_process_dies_during_sampling(self): - with test_subprocess("import time; time.sleep(0.5); exit()") as proc: + with test_subprocess("import time; time.sleep(0.5); exit()") as subproc: with ( io.StringIO() as captured_output, mock.patch("sys.stdout", captured_output), ): try: profiling.sampling.sample.sample( - proc.pid, + subproc.process.pid, duration_sec=2, # Longer than process lifetime sample_interval_usec=50000, ) @@ -2018,17 +2021,17 @@ def test_invalid_output_format_with_mocked_profiler(self): ) def test_is_process_running(self): - with test_subprocess("import time; time.sleep(1000)") as proc: + with test_subprocess("import time; time.sleep(1000)") as subproc: try: - profiler = SampleProfiler(pid=proc.pid, sample_interval_usec=1000, all_threads=False) + profiler = SampleProfiler(pid=subproc.process.pid, sample_interval_usec=1000, all_threads=False) except PermissionError: self.skipTest( "Insufficient permissions to read the stack trace" ) self.assertTrue(profiler._is_process_running()) self.assertIsNotNone(profiler.unwinder.get_stack_trace()) - proc.kill() - proc.wait() + subproc.process.kill() + subproc.process.wait() self.assertRaises(ProcessLookupError, profiler.unwinder.get_stack_trace) # Exit the context manager to ensure the process is terminated @@ -2037,9 +2040,9 @@ def test_is_process_running(self): @unittest.skipUnless(sys.platform == "linux", "Only valid on Linux") def test_esrch_signal_handling(self): - with test_subprocess("import time; time.sleep(1000)") as proc: + with test_subprocess("import time; time.sleep(1000)") as subproc: try: - unwinder = _remote_debugging.RemoteUnwinder(proc.pid) + unwinder = _remote_debugging.RemoteUnwinder(subproc.process.pid) except PermissionError: self.skipTest( "Insufficient permissions to read the stack trace" @@ -2047,10 +2050,10 @@ def test_esrch_signal_handling(self): initial_trace = unwinder.get_stack_trace() self.assertIsNotNone(initial_trace) - proc.kill() + subproc.process.kill() # Wait for the process to die and try to get another trace - proc.wait() + subproc.process.wait() with self.assertRaises(ProcessLookupError): unwinder.get_stack_trace() @@ -2644,35 +2647,47 @@ def test_cpu_mode_integration_filtering(self): import time import threading +cpu_ready = threading.Event() + def idle_worker(): time.sleep(999999) def cpu_active_worker(): + cpu_ready.set() x = 1 while True: x += 1 def main(): -# Start both threads + # Start both threads idle_thread = threading.Thread(target=idle_worker) cpu_thread = threading.Thread(target=cpu_active_worker) idle_thread.start() cpu_thread.start() + + # Wait for CPU thread to be running, then signal test + cpu_ready.wait() + _test_sock.sendall(b"threads_ready") + idle_thread.join() cpu_thread.join() main() ''' - with test_subprocess(cpu_vs_idle_script) as proc: + with test_subprocess(cpu_vs_idle_script) as subproc: + # Wait for signal that threads are running + response = subproc.socket.recv(1024) + self.assertEqual(response, b"threads_ready") + with ( io.StringIO() as captured_output, mock.patch("sys.stdout", captured_output), ): try: profiling.sampling.sample.sample( - proc.pid, - duration_sec=0.5, + subproc.process.pid, + duration_sec=2.0, sample_interval_usec=5000, mode=1, # CPU mode show_summary=False, @@ -2690,8 +2705,8 @@ def main(): ): try: profiling.sampling.sample.sample( - proc.pid, - duration_sec=0.5, + subproc.process.pid, + duration_sec=2.0, sample_interval_usec=5000, mode=0, # Wall-clock mode show_summary=False, @@ -2716,6 +2731,37 @@ def main(): self.assertIn("cpu_active_worker", wall_mode_output) self.assertIn("idle_worker", wall_mode_output) + def test_cpu_mode_with_no_samples(self): + """Test that CPU mode handles no samples gracefully when no samples are collected.""" + # Mock a collector that returns empty stats + mock_collector = mock.MagicMock() + mock_collector.stats = {} + mock_collector.create_stats = mock.MagicMock() + + with ( + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + mock.patch("profiling.sampling.sample.PstatsCollector", return_value=mock_collector), + mock.patch("profiling.sampling.sample.SampleProfiler") as mock_profiler_class, + ): + mock_profiler = mock.MagicMock() + mock_profiler_class.return_value = mock_profiler + + profiling.sampling.sample.sample( + 12345, # dummy PID + duration_sec=0.5, + sample_interval_usec=5000, + mode=1, # CPU mode + show_summary=False, + all_threads=True, + ) + + output = captured_output.getvalue() + + # Should see the "No samples were collected" message + self.assertIn("No samples were collected", output) + self.assertIn("CPU mode", output) + class TestGilModeFiltering(unittest.TestCase): """Test GIL mode filtering functionality (--mode=gil).""" @@ -2852,34 +2898,46 @@ def test_gil_mode_integration_behavior(self): import time import threading +gil_ready = threading.Event() + def gil_releasing_work(): time.sleep(999999) def gil_holding_work(): + gil_ready.set() x = 1 while True: x += 1 def main(): -# Start both threads + # Start both threads idle_thread = threading.Thread(target=gil_releasing_work) cpu_thread = threading.Thread(target=gil_holding_work) idle_thread.start() cpu_thread.start() + + # Wait for GIL-holding thread to be running, then signal test + gil_ready.wait() + _test_sock.sendall(b"threads_ready") + idle_thread.join() cpu_thread.join() main() ''' - with test_subprocess(gil_test_script) as proc: + with test_subprocess(gil_test_script) as subproc: + # Wait for signal that threads are running + response = subproc.socket.recv(1024) + self.assertEqual(response, b"threads_ready") + with ( io.StringIO() as captured_output, mock.patch("sys.stdout", captured_output), ): try: profiling.sampling.sample.sample( - proc.pid, - duration_sec=0.5, + subproc.process.pid, + duration_sec=2.0, sample_interval_usec=5000, mode=2, # GIL mode show_summary=False, @@ -2897,7 +2955,7 @@ def main(): ): try: profiling.sampling.sample.sample( - proc.pid, + subproc.process.pid, duration_sec=0.5, sample_interval_usec=5000, mode=0, # Wall-clock mode diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index c57ab51eca1..ed9d8540815 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -1222,5 +1222,15 @@ def __index__(self): return 100 +class TestModule(unittest.TestCase): + def test_deprecated__version__(self): + with self.assertWarnsRegex( + DeprecationWarning, + "'__version__' is deprecated and slated for removal in Python 3.20", + ) as cm: + getattr(zlib, "__version__") + self.assertEqual(cm.filename, __file__) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/ACKS b/Misc/ACKS index 0812b229e0a..2dc513829a2 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -210,6 +210,7 @@ Médéric Boquien Matias Bordese Jonas Borgström Jurjen Bos +Jeffrey Bosboom Peter Bosch Dan Boswell Eric Bouck diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-20-18-31.gh-issue-140080.8ROjxW.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-20-18-31.gh-issue-140080.8ROjxW.rst new file mode 100644 index 00000000000..0ddcea57f9d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-20-18-31.gh-issue-140080.8ROjxW.rst @@ -0,0 +1 @@ +Fix hang during finalization when attempting to call :mod:`atexit` handlers under no memory. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-00-21-40.gh-issue-140061.J0XeDV.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-00-21-40.gh-issue-140061.J0XeDV.rst new file mode 100644 index 00000000000..7c3924195eb --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-00-21-40.gh-issue-140061.J0XeDV.rst @@ -0,0 +1,2 @@ +Fixing the checking of whether an object is uniquely referenced to ensure +free-threaded compatibility. Patch by Sergey Miryanov. diff --git a/Misc/NEWS.d/next/Library/2025-09-18-21-25-41.gh-issue-83714.TQjDWZ.rst b/Misc/NEWS.d/next/Library/2025-09-18-21-25-41.gh-issue-83714.TQjDWZ.rst new file mode 100644 index 00000000000..7229a361147 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-18-21-25-41.gh-issue-83714.TQjDWZ.rst @@ -0,0 +1,2 @@ +Implement :func:`os.statx` on Linux kernel versions 4.11 and later with +glibc versions 2.28 and later. Contributed by Jeffrey Bosboom. diff --git a/Misc/NEWS.d/next/Library/2025-10-14-20-27-06.gh-issue-76007.2NcUbo.rst b/Misc/NEWS.d/next/Library/2025-10-14-20-27-06.gh-issue-76007.2NcUbo.rst new file mode 100644 index 00000000000..567fb5ef904 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-14-20-27-06.gh-issue-76007.2NcUbo.rst @@ -0,0 +1,2 @@ +:mod:`zlib`: Deprecate ``__version__`` and schedule for removal in Python +3.20. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 9263f14b57f..3173b52afb3 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -912,7 +912,7 @@ deepcopy(elementtreestate *st, PyObject *object, PyObject *memo) return Py_NewRef(object); } - if (Py_REFCNT(object) == 1) { + if (_PyObject_IsUniquelyReferenced(object)) { if (PyDict_CheckExact(object)) { PyObject *key, *value; Py_ssize_t pos = 0; @@ -2794,8 +2794,9 @@ treebuilder_handle_data(TreeBuilderObject* self, PyObject* data) self->data = Py_NewRef(data); } else { /* more than one item; use a list to collect items */ - if (PyBytes_CheckExact(self->data) && Py_REFCNT(self->data) == 1 && - PyBytes_CheckExact(data) && PyBytes_GET_SIZE(data) == 1) { + if (PyBytes_CheckExact(self->data) + && _PyObject_IsUniquelyReferenced(self->data) + && PyBytes_CheckExact(data) && PyBytes_GET_SIZE(data) == 1) { /* XXX this code path unused in Python 3? */ /* expat often generates single character data sections; handle the most common case by resizing the existing string... */ diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 257d5c6d536..44b1a302d49 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -291,7 +291,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (kw == NULL) { pto->kw = PyDict_New(); } - else if (Py_REFCNT(kw) == 1) { + else if (_PyObject_IsUniquelyReferenced(kw)) { pto->kw = Py_NewRef(kw); } else { @@ -1076,7 +1076,7 @@ _functools_reduce_impl(PyObject *module, PyObject *func, PyObject *seq, for (;;) { PyObject *op2; - if (Py_REFCNT(args) > 1) { + if (!_PyObject_IsUniquelyReferenced(args)) { Py_DECREF(args); if ((args = PyTuple_New(2)) == NULL) goto Fail; diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 7f1fa26c3ba..4a213f34888 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -118,7 +118,9 @@ static void blob_seterror(pysqlite_Blob *self, int rc) { assert(self->connection != NULL); - set_error_from_db(self->connection->state, self->connection->db); + assert(rc != SQLITE_OK); + set_error_from_code(self->connection->state, rc); + assert(PyErr_Occurred()); } static PyObject * diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index cb0f9adcc45..4611c9e5e3e 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -58,6 +58,41 @@ check_cursor_locked(pysqlite_Cursor *cur) return 1; } +static pysqlite_state * +get_module_state_by_cursor(pysqlite_Cursor *cursor) +{ + if (cursor->connection != NULL && cursor->connection->state != NULL) { + return cursor->connection->state; + } + return pysqlite_get_state_by_type(Py_TYPE(cursor)); +} + +static void +cursor_sqlite3_internal_error(pysqlite_Cursor *cursor, + const char *error_message, + int chain_exceptions) +{ + pysqlite_state *state = get_module_state_by_cursor(cursor); + if (chain_exceptions) { + assert(PyErr_Occurred()); + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(state->InternalError, error_message); + _PyErr_ChainExceptions1(exc); + } + else { + assert(!PyErr_Occurred()); + PyErr_SetString(state->InternalError, error_message); + } +} + +static void +cursor_cannot_reset_stmt_error(pysqlite_Cursor *cursor, int chain_exceptions) +{ + cursor_sqlite3_internal_error(cursor, + "cannot reset statement", + chain_exceptions); +} + /*[clinic input] module _sqlite3 class _sqlite3.Cursor "pysqlite_Cursor *" "clinic_state()->CursorType" @@ -173,8 +208,12 @@ cursor_clear(PyObject *op) Py_CLEAR(self->row_factory); if (self->statement) { /* Reset the statement if the user has not closed the cursor */ - stmt_reset(self->statement); + int rc = stmt_reset(self->statement); Py_CLEAR(self->statement); + if (rc != SQLITE_OK) { + cursor_cannot_reset_stmt_error(self, 0); + PyErr_FormatUnraisable("Exception ignored in cursor_clear()"); + } } return 0; @@ -837,7 +876,9 @@ _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* operation if (self->statement) { // Reset pending statements on this cursor. - (void)stmt_reset(self->statement); + if (stmt_reset(self->statement) != SQLITE_OK) { + goto reset_failure; + } } PyObject *stmt = get_statement_from_cache(self, operation); @@ -861,7 +902,9 @@ _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* operation } } - (void)stmt_reset(self->statement); + if (stmt_reset(self->statement) != SQLITE_OK) { + goto reset_failure; + } self->rowcount = self->statement->is_dml ? 0L : -1L; /* We start a transaction implicitly before a DML statement. @@ -943,7 +986,9 @@ _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* operation if (self->statement->is_dml) { self->rowcount += (long)sqlite3_changes(self->connection->db); } - stmt_reset(self->statement); + if (stmt_reset(self->statement) != SQLITE_OK) { + goto reset_failure; + } } Py_XDECREF(parameters); } @@ -968,8 +1013,15 @@ _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* operation if (PyErr_Occurred()) { if (self->statement) { - (void)stmt_reset(self->statement); + sqlite3 *db = sqlite3_db_handle(self->statement->st); + int sqlite3_state = sqlite3_errcode(db); + // stmt_reset() may return a previously set exception, + // either triggered because of Python or sqlite3. + rc = stmt_reset(self->statement); Py_CLEAR(self->statement); + if (sqlite3_state == SQLITE_OK && rc != SQLITE_OK) { + cursor_cannot_reset_stmt_error(self, 1); + } } self->rowcount = -1L; return NULL; @@ -978,6 +1030,20 @@ _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* operation Py_CLEAR(self->statement); } return Py_NewRef((PyObject *)self); + +reset_failure: + /* suite to execute when stmt_reset() failed and no exception is set */ + assert(!PyErr_Occurred()); + + Py_XDECREF(parameters); + Py_XDECREF(parameters_iter); + Py_XDECREF(parameters_list); + + self->locked = 0; + self->rowcount = -1L; + Py_CLEAR(self->statement); + cursor_cannot_reset_stmt_error(self, 0); + return NULL; } /*[clinic input] @@ -1120,14 +1186,20 @@ pysqlite_cursor_iternext(PyObject *op) if (self->statement->is_dml) { self->rowcount = (long)sqlite3_changes(self->connection->db); } - (void)stmt_reset(self->statement); + rc = stmt_reset(self->statement); Py_CLEAR(self->statement); + if (rc != SQLITE_OK) { + goto reset_failure; + } } else if (rc != SQLITE_ROW) { - set_error_from_db(self->connection->state, self->connection->db); - (void)stmt_reset(self->statement); + rc = set_error_from_db(self->connection->state, self->connection->db); + int reset_rc = stmt_reset(self->statement); Py_CLEAR(self->statement); Py_DECREF(row); + if (rc == SQLITE_OK && reset_rc != SQLITE_OK) { + goto reset_failure; + } return NULL; } if (!Py_IsNone(self->row_factory)) { @@ -1137,6 +1209,10 @@ pysqlite_cursor_iternext(PyObject *op) Py_SETREF(row, new_row); } return row; + +reset_failure: + cursor_cannot_reset_stmt_error(self, 0); + return NULL; } /*[clinic input] @@ -1291,8 +1367,15 @@ pysqlite_cursor_close_impl(pysqlite_Cursor *self) } if (self->statement) { - (void)stmt_reset(self->statement); + int rc = stmt_reset(self->statement); + // Force self->statement to be NULL even if stmt_reset() may have + // failed to avoid a possible double-free if someone calls close() + // twice as a leak here would be better than a double-free. Py_CLEAR(self->statement); + if (rc != SQLITE_OK) { + cursor_cannot_reset_stmt_error(self, 0); + return NULL; + } } self->closed = 1; diff --git a/Modules/_sqlite/statement.c b/Modules/_sqlite/statement.c index c551ca59b33..f31c699482f 100644 --- a/Modules/_sqlite/statement.c +++ b/Modules/_sqlite/statement.c @@ -103,7 +103,12 @@ pysqlite_statement_create(pysqlite_Connection *connection, PyObject *sql) return self; error: - (void)sqlite3_finalize(stmt); + assert(PyErr_Occurred()); + if (sqlite3_finalize(stmt) != SQLITE_OK) { + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(connection->InternalError, "cannot finalize statement"); + _PyErr_ChainExceptions1(exc); + } return NULL; } @@ -114,10 +119,16 @@ stmt_dealloc(PyObject *op) PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(op); if (self->st) { + int rc; Py_BEGIN_ALLOW_THREADS - sqlite3_finalize(self->st); + rc = sqlite3_finalize(self->st); Py_END_ALLOW_THREADS - self->st = 0; + self->st = NULL; + if (rc != SQLITE_OK) { + pysqlite_state *state = PyType_GetModuleState(Py_TYPE(op)); + PyErr_SetString(state->InternalError, "cannot finalize statement"); + PyErr_FormatUnraisable("Exception ignored in stmt_dealloc()"); + } } tp->tp_free(self); Py_DECREF(tp); diff --git a/Modules/_sqlite/util.c b/Modules/_sqlite/util.c index 103248ff55a..177e0f668a0 100644 --- a/Modules/_sqlite/util.c +++ b/Modules/_sqlite/util.c @@ -135,14 +135,14 @@ set_error_from_code(pysqlite_state *state, int code) /** * Checks the SQLite error code and sets the appropriate DB-API exception. */ -void +int set_error_from_db(pysqlite_state *state, sqlite3 *db) { int errorcode = sqlite3_errcode(db); PyObject *exc_class = get_exception_class(state, errorcode); if (exc_class == NULL) { // No new exception need be raised. - return; + return SQLITE_OK; } /* Create and set the exception. */ @@ -150,6 +150,7 @@ set_error_from_db(pysqlite_state *state, sqlite3 *db) // sqlite3_errmsg() always returns an UTF-8 encoded message const char *errmsg = sqlite3_errmsg(db); raise_exception(exc_class, extended_errcode, errmsg); + return errorcode; } #ifdef WORDS_BIGENDIAN diff --git a/Modules/_sqlite/util.h b/Modules/_sqlite/util.h index f8e45baffae..c00369496f9 100644 --- a/Modules/_sqlite/util.h +++ b/Modules/_sqlite/util.h @@ -31,7 +31,7 @@ /** * Checks the SQLite error code and sets the appropriate DB-API exception. */ -void set_error_from_db(pysqlite_state *state, sqlite3 *db); +int set_error_from_db(pysqlite_state *state, sqlite3 *db); void set_error_from_code(pysqlite_state *state, int code); sqlite_int64 _pysqlite_long_as_int64(PyObject * value); diff --git a/Modules/atexitmodule.c b/Modules/atexitmodule.c index 4b068967a6c..4536b03fbc4 100644 --- a/Modules/atexitmodule.c +++ b/Modules/atexitmodule.c @@ -112,6 +112,7 @@ atexit_callfuncs(struct atexit_state *state) { PyErr_FormatUnraisable("Exception ignored while " "copying atexit callbacks"); + atexit_cleanup(state); return; } diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 71f87ac8ec7..7bd3d0ebfaa 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -186,6 +186,140 @@ exit: return return_value; } +#if defined(HAVE_STATX) + +PyDoc_STRVAR(os_statx__doc__, +"statx($module, /, path, mask, *, flags=0, dir_fd=None,\n" +" follow_symlinks=True)\n" +"--\n" +"\n" +"Perform a statx system call on the given path.\n" +"\n" +" path\n" +" Path to be examined; can be string, bytes, a path-like object or\n" +" open-file-descriptor int.\n" +" mask\n" +" A bitmask of STATX_* constants defining the requested information.\n" +" flags\n" +" A bitmask of AT_NO_AUTOMOUNT and/or AT_STATX_* flags.\n" +" dir_fd\n" +" If not None, it should be a file descriptor open to a directory,\n" +" and path should be a relative string; path will then be relative to\n" +" that directory.\n" +" follow_symlinks\n" +" If False, and the last element of the path is a symbolic link,\n" +" statx will examine the symbolic link itself instead of the file\n" +" the link points to.\n" +"\n" +"It\'s an error to use dir_fd or follow_symlinks when specifying path as\n" +" an open file descriptor."); + +#define OS_STATX_METHODDEF \ + {"statx", _PyCFunction_CAST(os_statx), METH_FASTCALL|METH_KEYWORDS, os_statx__doc__}, + +static PyObject * +os_statx_impl(PyObject *module, path_t *path, unsigned int mask, int flags, + int dir_fd, int follow_symlinks); + +static PyObject * +os_statx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 5 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(path), &_Py_ID(mask), &_Py_ID(flags), &_Py_ID(dir_fd), &_Py_ID(follow_symlinks), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", "mask", "flags", "dir_fd", "follow_symlinks", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "statx", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[5]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + path_t path = PATH_T_INITIALIZE_P("statx", "path", 0, 0, 0, 1); + unsigned int mask; + int flags = 0; + int dir_fd = DEFAULT_DIR_FD; + int follow_symlinks = 1; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!path_converter(args[0], &path)) { + goto exit; + } + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &mask, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[2]) { + flags = PyLong_AsInt(args[2]); + if (flags == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (args[3]) { + if (!dir_fd_converter(args[3], &dir_fd)) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + follow_symlinks = PyObject_IsTrue(args[4]); + if (follow_symlinks < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = os_statx_impl(module, &path, mask, flags, dir_fd, follow_symlinks); + +exit: + /* Cleanup for path */ + path_cleanup(&path); + + return return_value; +} + +#endif /* defined(HAVE_STATX) */ + PyDoc_STRVAR(os_access__doc__, "access($module, /, path, mode, *, dir_fd=None, effective_ids=False,\n" " follow_symlinks=True)\n" @@ -12793,6 +12927,10 @@ exit: #endif /* defined(__EMSCRIPTEN__) */ +#ifndef OS_STATX_METHODDEF + #define OS_STATX_METHODDEF +#endif /* !defined(OS_STATX_METHODDEF) */ + #ifndef OS_TTYNAME_METHODDEF #define OS_TTYNAME_METHODDEF #endif /* !defined(OS_TTYNAME_METHODDEF) */ @@ -13472,4 +13610,4 @@ exit: #ifndef OS__EMSCRIPTEN_LOG_METHODDEF #define OS__EMSCRIPTEN_LOG_METHODDEF #endif /* !defined(OS__EMSCRIPTEN_LOG_METHODDEF) */ -/*[clinic end generated code: output=67f0df7cd5a7de20 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=44f7a1a16dad2e08 input=a9049054013a1b77]*/ diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 60ef6f9ff4c..8685eff8be6 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -376,7 +376,7 @@ pairwise_next(PyObject *op) } result = po->result; - if (Py_REFCNT(result) == 1) { + if (_PyObject_IsUniquelyReferenced(result)) { Py_INCREF(result); PyObject *last_old = PyTuple_GET_ITEM(result, 0); PyObject *last_new = PyTuple_GET_ITEM(result, 1); @@ -802,7 +802,7 @@ teedataobject_traverse(PyObject *op, visitproc visit, void * arg) static void teedataobject_safe_decref(PyObject *obj) { - while (obj && Py_REFCNT(obj) == 1) { + while (obj && _PyObject_IsUniquelyReferenced(obj)) { teedataobject *tmp = teedataobject_CAST(obj); PyObject *nextlink = tmp->nextlink; tmp->nextlink = NULL; @@ -2129,7 +2129,7 @@ product_next_lock_held(PyObject *op) Py_ssize_t *indices = lz->indices; /* Copy the previous result tuple or re-use it if available */ - if (Py_REFCNT(result) > 1) { + if (!_PyObject_IsUniquelyReferenced(result)) { PyObject *old_result = result; result = PyTuple_FromArray(_PyTuple_ITEMS(old_result), npools); if (result == NULL) @@ -2364,7 +2364,7 @@ combinations_next_lock_held(PyObject *op) } } else { /* Copy the previous result tuple or re-use it if available */ - if (Py_REFCNT(result) > 1) { + if (!_PyObject_IsUniquelyReferenced(result)) { PyObject *old_result = result; result = PyTuple_FromArray(_PyTuple_ITEMS(old_result), r); if (result == NULL) @@ -2618,7 +2618,7 @@ cwr_next(PyObject *op) } } else { /* Copy the previous result tuple or re-use it if available */ - if (Py_REFCNT(result) > 1) { + if (!_PyObject_IsUniquelyReferenced(result)) { PyObject *old_result = result; result = PyTuple_FromArray(_PyTuple_ITEMS(old_result), r); if (result == NULL) @@ -2879,7 +2879,7 @@ permutations_next(PyObject *op) goto empty; /* Copy the previous result tuple or re-use it if available */ - if (Py_REFCNT(result) > 1) { + if (!_PyObject_IsUniquelyReferenced(result)) { PyObject *old_result = result; result = PyTuple_FromArray(_PyTuple_ITEMS(old_result), r); if (result == NULL) @@ -3847,7 +3847,7 @@ zip_longest_next(PyObject *op) return NULL; if (lz->numactive == 0) return NULL; - if (Py_REFCNT(result) == 1) { + if (_PyObject_IsUniquelyReferenced(result)) { Py_INCREF(result); for (i=0 ; i < tuplesize ; i++) { it = PyTuple_GET_ITEM(lz->ittuple, i); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 38ddc3ec4ff..2dd43e50e79 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -40,6 +40,7 @@ // --- System includes ------------------------------------------------------ +#include // offsetof() #include // ctermid() #include // system() @@ -408,6 +409,31 @@ extern char *ctermid_r(char *); # define STRUCT_STAT struct stat #endif +#ifdef HAVE_STATX +/* until we can assume glibc 2.28 at runtime, we must weakly link */ +# pragma weak statx +static const unsigned int _Py_STATX_KNOWN = (STATX_BASIC_STATS | STATX_BTIME +#ifdef STATX_MNT_ID + | STATX_MNT_ID +#endif +#ifdef STATX_DIOALIGN + | STATX_DIOALIGN +#endif +#ifdef STATX_MNT_ID_UNIQUE + | STATX_MNT_ID_UNIQUE +#endif +#ifdef STATX_SUBVOL + | STATX_SUBVOL +#endif +#ifdef STATX_WRITE_ATOMIC + | STATX_WRITE_ATOMIC +#endif +#ifdef STATX_DIO_READ_ALIGN + | STATX_DIO_READ_ALIGN +#endif + ); +#endif /* HAVE_STATX */ + #if !defined(EX_OK) && defined(EXIT_SUCCESS) # define EX_OK EXIT_SUCCESS @@ -1169,6 +1195,9 @@ typedef struct { #endif newfunc statresult_new_orig; PyObject *StatResultType; +#ifdef HAVE_STATX + PyObject *StatxResultType; +#endif PyObject *StatVFSResultType; PyObject *TerminalSizeType; PyObject *TimesResultType; @@ -2549,6 +2578,9 @@ _posix_clear(PyObject *module) Py_CLEAR(state->SchedParamType); #endif Py_CLEAR(state->StatResultType); +#ifdef HAVE_STATX + Py_CLEAR(state->StatxResultType); +#endif Py_CLEAR(state->StatVFSResultType); Py_CLEAR(state->TerminalSizeType); Py_CLEAR(state->TimesResultType); @@ -2574,6 +2606,9 @@ _posix_traverse(PyObject *module, visitproc visit, void *arg) Py_VISIT(state->SchedParamType); #endif Py_VISIT(state->StatResultType); +#ifdef HAVE_STATX + Py_VISIT(state->StatxResultType); +#endif Py_VISIT(state->StatVFSResultType); Py_VISIT(state->TerminalSizeType); Py_VISIT(state->TimesResultType); @@ -2594,12 +2629,44 @@ _posix_free(void *module) _posix_clear((PyObject *)module); } + +#define SEC_TO_NS (1000000000LL) +static PyObject * +stat_nanosecond_timestamp(_posixstate *state, time_t sec, unsigned long nsec) +{ + /* 1677-09-21 00:12:44 to 2262-04-11 23:47:15 UTC inclusive */ + if ((LLONG_MIN/SEC_TO_NS) <= sec && sec <= (LLONG_MAX/SEC_TO_NS - 1)) { + return PyLong_FromLongLong(sec * SEC_TO_NS + nsec); + } + else { + PyObject *ns_total = NULL; + PyObject *s_in_ns = NULL; + PyObject *s = _PyLong_FromTime_t(sec); + PyObject *ns_fractional = PyLong_FromUnsignedLong(nsec); + if (s == NULL || ns_fractional == NULL) { + goto exit; + } + + s_in_ns = PyNumber_Multiply(s, state->billion); + if (s_in_ns == NULL) { + goto exit; + } + + ns_total = PyNumber_Add(s_in_ns, ns_fractional); + + exit: + Py_XDECREF(s); + Py_XDECREF(ns_fractional); + Py_XDECREF(s_in_ns); + return ns_total; + } +} + static int fill_time(_posixstate *state, PyObject *v, int s_index, int f_index, int ns_index, time_t sec, unsigned long nsec) { assert(!PyErr_Occurred()); -#define SEC_TO_NS (1000000000LL) assert(nsec < SEC_TO_NS); if (s_index >= 0) { @@ -2618,50 +2685,18 @@ fill_time(_posixstate *state, PyObject *v, int s_index, int f_index, PyStructSequence_SET_ITEM(v, f_index, float_s); } - int res = -1; if (ns_index >= 0) { - /* 1677-09-21 00:12:44 to 2262-04-11 23:47:15 UTC inclusive */ - if ((LLONG_MIN/SEC_TO_NS) <= sec && sec <= (LLONG_MAX/SEC_TO_NS - 1)) { - PyObject *ns_total = PyLong_FromLongLong(sec * SEC_TO_NS + nsec); - if (ns_total == NULL) { - return -1; - } - PyStructSequence_SET_ITEM(v, ns_index, ns_total); - assert(!PyErr_Occurred()); - res = 0; - } - else { - PyObject *s_in_ns = NULL; - PyObject *ns_total = NULL; - PyObject *s = _PyLong_FromTime_t(sec); - PyObject *ns_fractional = PyLong_FromUnsignedLong(nsec); - if (s == NULL || ns_fractional == NULL) { - goto exit; - } - - s_in_ns = PyNumber_Multiply(s, state->billion); - if (s_in_ns == NULL) { - goto exit; - } - - ns_total = PyNumber_Add(s_in_ns, ns_fractional); - if (ns_total == NULL) { - goto exit; - } - PyStructSequence_SET_ITEM(v, ns_index, ns_total); - assert(!PyErr_Occurred()); - res = 0; - - exit: - Py_XDECREF(s); - Py_XDECREF(ns_fractional); - Py_XDECREF(s_in_ns); + PyObject *ns_total = stat_nanosecond_timestamp(state, sec, nsec); + if (ns_total == NULL) { + return -1; } + PyStructSequence_SET_ITEM(v, ns_index, ns_total); } - return res; - #undef SEC_TO_NS + assert(!PyErr_Occurred()); + return 0; } +#undef SEC_TO_NS #ifdef MS_WINDOWS static PyObject* @@ -3276,6 +3311,307 @@ os_lstat_impl(PyObject *module, path_t *path, int dir_fd) } +#ifdef HAVE_STATX +typedef struct { + PyObject_HEAD + double atime_sec, btime_sec, ctime_sec, mtime_sec; + dev_t rdev, dev; + struct statx stx; +} Py_statx_result; + +#define M(attr, type, offset, doc) \ + {attr, type, offset, Py_READONLY, PyDoc_STR(doc)} +#define MM(attr, type, member, doc) \ + M(#attr, type, offsetof(Py_statx_result, stx.stx_##member), doc) +#define MX(attr, type, member, doc) \ + M(#attr, type, offsetof(Py_statx_result, member), doc) + +static PyMemberDef pystatx_result_members[] = { + MM(stx_mask, Py_T_UINT, mask, "member validity mask"), + MM(st_blksize, Py_T_UINT, blksize, "blocksize for filesystem I/O"), + MM(stx_attributes, Py_T_ULONGLONG, attributes, "Linux inode attribute bits"), + MM(st_nlink, Py_T_UINT, nlink, "number of hard links"), + MM(st_uid, Py_T_UINT, uid, "user ID of owner"), + MM(st_gid, Py_T_UINT, gid, "group ID of owner"), + MM(st_mode, Py_T_USHORT, mode, "protection bits"), + MM(st_ino, Py_T_ULONGLONG, ino, "inode"), + MM(st_size, Py_T_ULONGLONG, size, "total size, in bytes"), + MM(st_blocks, Py_T_ULONGLONG, blocks, "number of blocks allocated"), + MM(stx_attributes_mask, Py_T_ULONGLONG, attributes_mask, + "Mask of supported bits in stx_attributes"), + MX(st_atime, Py_T_DOUBLE, atime_sec, "time of last access"), + MX(st_birthtime, Py_T_DOUBLE, btime_sec, "time of creation"), + MX(st_ctime, Py_T_DOUBLE, ctime_sec, "time of last change"), + MX(st_mtime, Py_T_DOUBLE, mtime_sec, "time of last modification"), + MM(stx_rdev_major, Py_T_UINT, rdev_major, "represented device major number"), + MM(stx_rdev_minor, Py_T_UINT, rdev_minor, "represented device minor number"), + MX(st_rdev, Py_T_ULONGLONG, rdev, "device type (if inode device)"), + MM(stx_dev_major, Py_T_UINT, dev_major, "containing device major number"), + MM(stx_dev_minor, Py_T_UINT, dev_minor, "containing device minor number"), + MX(st_dev, Py_T_ULONGLONG, dev, "device"), +#ifdef STATX_MNT_ID + MM(stx_mnt_id, Py_T_ULONGLONG, mnt_id, "mount ID"), +#endif +#ifdef STATX_DIOALIGN + MM(stx_dio_mem_align, Py_T_UINT, dio_mem_align, + "direct I/O memory buffer alignment"), + MM(stx_dio_offset_align, Py_T_UINT, dio_offset_align, + "direct I/O file offset alignment"), +#endif +#ifdef STATX_SUBVOL + MM(stx_subvol, Py_T_ULONGLONG, subvol, "subvolume ID"), +#endif +#ifdef STATX_WRITE_ATOMIC + MM(stx_atomic_write_unit_min, Py_T_UINT, atomic_write_unit_min, + "minimum size for direct I/O with torn-write protection"), + MM(stx_atomic_write_unit_max, Py_T_UINT, atomic_write_unit_max, + "maximum size for direct I/O with torn-write protection"), + MM(stx_atomic_write_unit_max_opt, Py_T_UINT, atomic_write_unit_max_opt, + "maximum optimized size for direct I/O with torn-write protection"), + MM(stx_atomic_write_segments_max, Py_T_UINT, atomic_write_segments_max, + "maximum iovecs for direct I/O with torn-write protection"), +#endif +#ifdef STATX_DIO_READ_ALIGN + MM(stx_dio_read_offset_align, Py_T_UINT, dio_read_offset_align, + "direct I/O file offset alignment for reads"), +#endif + {NULL}, +}; + +#undef MX +#undef MM +#undef M + +static PyObject * +pystatx_result_get_nsec(PyObject *op, void *context) +{ + uint16_t offset = (uintptr_t)context; + struct statx_timestamp *ts = (void*)op + offset; + _posixstate *state = PyType_GetModuleState(Py_TYPE(op)); + assert(state != NULL); + return stat_nanosecond_timestamp(state, ts->tv_sec, ts->tv_nsec); +} + +/* The low 16 bits of the context pointer are the offset from the start of + Py_statx_result to the struct statx member. */ +#define GM(attr, type, member, doc) \ + {#attr, pystatx_result_get_##type, NULL, PyDoc_STR(doc), \ + (void *)(offsetof(Py_statx_result, stx.stx_##member))} + +static PyGetSetDef pystatx_result_getset[] = { + GM(st_atime_ns, nsec, atime, "time of last access in nanoseconds"), + GM(st_birthtime_ns, nsec, btime, "time of creation in nanoseconds"), + GM(st_ctime_ns, nsec, ctime, "time of last change in nanoseconds"), + GM(st_mtime_ns, nsec, mtime, "time of last modification in nanoseconds"), + {NULL}, +}; + +#undef GM + +static PyObject * +pystatx_result_repr(PyObject *op) +{ + PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); + if (writer == NULL) { + return NULL; + } +#define WRITE_ASCII(s) \ + do { \ + if (PyUnicodeWriter_WriteASCII(writer, s, strlen(s)) < 0) { \ + goto error; \ + } \ + } while (0) + + WRITE_ASCII("os.statx_result("); + + for (size_t i = 0; i < Py_ARRAY_LENGTH(pystatx_result_members) - 1; ++i) { + if (i > 0) { + WRITE_ASCII(", "); + } + + PyMemberDef *d = &pystatx_result_members[i]; + WRITE_ASCII(d->name); + WRITE_ASCII("="); + + PyObject *o = PyMember_GetOne((const char *)op, d); + if (o == NULL) { + goto error; + } + if (PyUnicodeWriter_WriteRepr(writer, o) < 0) { + Py_DECREF(o); + goto error; + } + Py_DECREF(o); + } + + if (Py_ARRAY_LENGTH(pystatx_result_members) > 1 + && Py_ARRAY_LENGTH(pystatx_result_getset) > 1) { + WRITE_ASCII(", "); + } + + for (size_t i = 0; i < Py_ARRAY_LENGTH(pystatx_result_getset) - 1; ++i) { + if (i > 0) { + WRITE_ASCII(", "); + } + + PyGetSetDef *d = &pystatx_result_getset[i]; + WRITE_ASCII(d->name); + WRITE_ASCII("="); + + PyObject *o = d->get(op, d->closure); + if (o == NULL) { + goto error; + } + if (PyUnicodeWriter_WriteRepr(writer, o) < 0) { + Py_DECREF(o); + goto error; + } + Py_DECREF(o); + } + + WRITE_ASCII(")"); + return PyUnicodeWriter_Finish(writer); +#undef WRITE_ASCII + +error: + PyUnicodeWriter_Discard(writer); + return NULL; +} + +static int +pystatx_result_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} + +static void +pystatx_result_dealloc(PyObject *op) +{ + Py_statx_result *self = (Py_statx_result *) op; + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); + tp->tp_free(self); + Py_DECREF(tp); +} + +static PyType_Slot pystatx_result_slots[] = { + {Py_tp_repr, pystatx_result_repr}, + {Py_tp_traverse, pystatx_result_traverse}, + {Py_tp_dealloc, pystatx_result_dealloc}, + {Py_tp_members, pystatx_result_members}, + {Py_tp_getset, pystatx_result_getset}, + {0, NULL}, +}; + +static PyType_Spec pystatx_result_spec = { + .name = "os.statx_result", + .basicsize = sizeof(Py_statx_result), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .slots = pystatx_result_slots, +}; + +/*[clinic input] + +os.statx + + path : path_t(allow_fd=True) + Path to be examined; can be string, bytes, a path-like object or + open-file-descriptor int. + + mask: unsigned_int(bitwise=True) + A bitmask of STATX_* constants defining the requested information. + + * + + flags: int = 0 + A bitmask of AT_NO_AUTOMOUNT and/or AT_STATX_* flags. + + dir_fd : dir_fd = None + If not None, it should be a file descriptor open to a directory, + and path should be a relative string; path will then be relative to + that directory. + + follow_symlinks: bool = True + If False, and the last element of the path is a symbolic link, + statx will examine the symbolic link itself instead of the file + the link points to. + +Perform a statx system call on the given path. + +It's an error to use dir_fd or follow_symlinks when specifying path as + an open file descriptor. + +[clinic start generated code]*/ + +static PyObject * +os_statx_impl(PyObject *module, path_t *path, unsigned int mask, int flags, + int dir_fd, int follow_symlinks) +/*[clinic end generated code: output=e3765979ac6fe15b input=f0116380c5dc4f2f]*/ +{ + if (path_and_dir_fd_invalid("statx", path, dir_fd) || + dir_fd_and_fd_invalid("statx", dir_fd, path->fd) || + fd_and_follow_symlinks_invalid("statx", path->fd, follow_symlinks)) { + return NULL; + } + + /* reject flags covered by kwargs, but allow unknown flags that may be + future AT_STATX_* extensions */ + if (flags & (AT_SYMLINK_NOFOLLOW | AT_SYMLINK_FOLLOW)) { + PyErr_Format(PyExc_ValueError, + "use follow_symlinks kwarg instead of AT_SYMLINK_* flag"); + return NULL; + } + if (flags & AT_EMPTY_PATH) { + PyErr_Format(PyExc_ValueError, + "use dir_fd kwarg instead of AT_EMPTY_PATH flag"); + return NULL; + } + + /* Future bits may refer to members beyond the current size of struct + statx, so we need to mask them off to prevent memory corruption. */ + mask &= _Py_STATX_KNOWN; + + _posixstate *state = get_posix_state(module); + PyTypeObject *tp = (PyTypeObject *)state->StatxResultType; + Py_statx_result *v = (Py_statx_result *)tp->tp_alloc(tp, 0); + if (v == NULL) { + return NULL; + } + + int result; + Py_BEGIN_ALLOW_THREADS + if (path->fd != -1) { + result = statx(path->fd, "", flags | AT_EMPTY_PATH, mask, &v->stx); + } + else { + result = statx(dir_fd, path->narrow, flags, mask, &v->stx); + } + Py_END_ALLOW_THREADS + + if (result != 0) { + Py_DECREF(v); + return path_error(path); + } + + v->atime_sec = ((double)v->stx.stx_atime.tv_sec + + 1e-9 * v->stx.stx_atime.tv_nsec); + v->btime_sec = ((double)v->stx.stx_btime.tv_sec + + 1e-9 * v->stx.stx_btime.tv_nsec); + v->ctime_sec = ((double)v->stx.stx_ctime.tv_sec + + 1e-9 * v->stx.stx_ctime.tv_nsec); + v->mtime_sec = ((double)v->stx.stx_mtime.tv_sec + + 1e-9 * v->stx.stx_mtime.tv_nsec); + v->rdev = makedev(v->stx.stx_rdev_major, v->stx.stx_rdev_minor); + v->dev = makedev(v->stx.stx_dev_major, v->stx.stx_dev_minor); + + assert(!PyErr_Occurred()); + return (PyObject *)v; +} +#endif /* HAVE_STATX */ + + /*[clinic input] os.access -> bool @@ -17051,6 +17387,7 @@ os__emscripten_log_impl(PyObject *module, const char *arg) static PyMethodDef posix_methods[] = { OS_STAT_METHODDEF + OS_STATX_METHODDEF OS_ACCESS_METHODDEF OS_TTYNAME_METHODDEF OS_CHDIR_METHODDEF @@ -17897,6 +18234,49 @@ all_ins(PyObject *m) if (PyModule_Add(m, "NODEV", _PyLong_FromDev(NODEV))) return -1; #endif +#ifdef AT_NO_AUTOMOUNT + if (PyModule_AddIntMacro(m, AT_NO_AUTOMOUNT)) return -1; +#endif + +#ifdef HAVE_STATX + if (PyModule_AddIntMacro(m, STATX_TYPE)) return -1; + if (PyModule_AddIntMacro(m, STATX_MODE)) return -1; + if (PyModule_AddIntMacro(m, STATX_NLINK)) return -1; + if (PyModule_AddIntMacro(m, STATX_UID)) return -1; + if (PyModule_AddIntMacro(m, STATX_GID)) return -1; + if (PyModule_AddIntMacro(m, STATX_ATIME)) return -1; + if (PyModule_AddIntMacro(m, STATX_MTIME)) return -1; + if (PyModule_AddIntMacro(m, STATX_CTIME)) return -1; + if (PyModule_AddIntMacro(m, STATX_INO)) return -1; + if (PyModule_AddIntMacro(m, STATX_SIZE)) return -1; + if (PyModule_AddIntMacro(m, STATX_BLOCKS)) return -1; + if (PyModule_AddIntMacro(m, STATX_BASIC_STATS)) return -1; + if (PyModule_AddIntMacro(m, STATX_BTIME)) return -1; +#ifdef STATX_MNT_ID + if (PyModule_AddIntMacro(m, STATX_MNT_ID)) return -1; +#endif +#ifdef STATX_DIOALIGN + if (PyModule_AddIntMacro(m, STATX_DIOALIGN)) return -1; +#endif +#ifdef STATX_MNT_ID_UNIQUE + if (PyModule_AddIntMacro(m, STATX_MNT_ID_UNIQUE)) return -1; +#endif +#ifdef STATX_SUBVOL + if (PyModule_AddIntMacro(m, STATX_SUBVOL)) return -1; +#endif +#ifdef STATX_WRITE_ATOMIC + if (PyModule_AddIntMacro(m, STATX_WRITE_ATOMIC)) return -1; +#endif +#ifdef STATX_DIO_READ_ALIGN + if (PyModule_AddIntMacro(m, STATX_DIO_READ_ALIGN)) return -1; +#endif + /* STATX_ALL intentionally omitted because it is deprecated */ + if (PyModule_AddIntMacro(m, AT_STATX_SYNC_AS_STAT)) return -1; + if (PyModule_AddIntMacro(m, AT_STATX_FORCE_SYNC)) return -1; + if (PyModule_AddIntMacro(m, AT_STATX_DONT_SYNC)) return -1; + /* STATX_ATTR_* constants are in the stat module */ +#endif /* HAVE_STATX */ + #if defined(__APPLE__) if (PyModule_AddIntConstant(m, "_COPYFILE_DATA", COPYFILE_DATA)) return -1; if (PyModule_AddIntConstant(m, "_COPYFILE_STAT", COPYFILE_STAT)) return -1; @@ -18168,6 +18548,24 @@ posixmodule_exec(PyObject *m) } #endif +#ifdef HAVE_STATX + if (statx == NULL) { + PyObject* dct = PyModule_GetDict(m); + if (dct == NULL) { + return -1; + } + if (PyDict_PopString(dct, "statx", NULL) < 0) { + return -1; + } + } + else { + state->StatxResultType = PyType_FromModuleAndSpec(m, &pystatx_result_spec, NULL); + if (PyModule_AddObjectRef(m, "statx_result", state->StatxResultType) < 0) { + return -1; + } + } +#endif + /* Initialize environ dictionary */ if (PyModule_Add(m, "environ", convertenviron()) != 0) { return -1; diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index 36c933bf618..6bac09aa6c2 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -2015,6 +2015,27 @@ zlib_crc32_combine_impl(PyObject *module, unsigned int crc1, return crc32_combine(crc1, crc2, len); } +static PyObject * +zlib_getattr(PyObject *self, PyObject *args) +{ + PyObject *name; + if (!PyArg_UnpackTuple(args, "__getattr__", 1, 1, &name)) { + return NULL; + } + + if (PyUnicode_Check(name) && PyUnicode_EqualToUTF8(name, "__version__")) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "'__version__' is deprecated and slated for removal in Python 3.20", + 1) < 0) { + return NULL; + } + return PyUnicode_FromString("1.0"); + } + + PyErr_Format(PyExc_AttributeError, "module 'zlib' has no attribute %R", name); + return NULL; +} + static PyMethodDef zlib_methods[] = { ZLIB_ADLER32_METHODDEF @@ -2025,6 +2046,7 @@ static PyMethodDef zlib_methods[] = ZLIB_CRC32_COMBINE_METHODDEF ZLIB_DECOMPRESS_METHODDEF ZLIB_DECOMPRESSOBJ_METHODDEF + {"__getattr__", zlib_getattr, METH_VARARGS, "Module __getattr__"}, {NULL, NULL} }; @@ -2221,9 +2243,6 @@ zlib_exec(PyObject *mod) return -1; } #endif - if (PyModule_AddStringConstant(mod, "__version__", "1.0") < 0) { - return -1; - } return 0; } diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index de8ab26db1e..9c807b3dd16 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3171,7 +3171,7 @@ PyBytes_Concat(PyObject **pv, PyObject *w) return; } - if (Py_REFCNT(*pv) == 1 && PyBytes_CheckExact(*pv)) { + if (_PyObject_IsUniquelyReferenced(*pv) && PyBytes_CheckExact(*pv)) { /* Only one reference, so we can resize in place */ Py_ssize_t oldsize; Py_buffer wb; @@ -3256,7 +3256,7 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) Py_DECREF(v); return 0; } - if (Py_REFCNT(v) != 1) { + if (!_PyObject_IsUniquelyReferenced(v)) { if (oldsize < newsize) { *pv = _PyBytes_FromSize(newsize, 0); if (*pv) { diff --git a/Objects/dictobject.c b/Objects/dictobject.c index b95a4a8dc5e..73d4db4cac7 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5667,22 +5667,10 @@ dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, #endif -static bool -has_unique_reference(PyObject *op) -{ -#ifdef Py_GIL_DISABLED - return (_Py_IsOwnedByCurrentThread(op) && - op->ob_ref_local == 1 && - _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared) == 0); -#else - return Py_REFCNT(op) == 1; -#endif -} - static bool acquire_iter_result(PyObject *result) { - if (has_unique_reference(result)) { + if (_PyObject_IsUniquelyReferenced(result)) { Py_INCREF(result); return true; } @@ -5831,7 +5819,7 @@ dictreviter_iter_lock_held(PyDictObject *d, PyObject *self) } else if (Py_IS_TYPE(di, &PyDictRevIterItem_Type)) { result = di->di_result; - if (Py_REFCNT(result) == 1) { + if (_PyObject_IsUniquelyReferenced(result)) { PyObject *oldkey = PyTuple_GET_ITEM(result, 0); PyObject *oldvalue = PyTuple_GET_ITEM(result, 1); PyTuple_SET_ITEM(result, 0, Py_NewRef(key)); diff --git a/Objects/longobject.c b/Objects/longobject.c index 5eb4063f861..298399210af 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -352,7 +352,7 @@ _PyLong_Negate(PyLongObject **x_p) PyLongObject *x; x = (PyLongObject *)*x_p; - if (Py_REFCNT(x) == 1) { + if (_PyObject_IsUniquelyReferenced((PyObject *)x)) { _PyLong_FlipSign(x); return; } @@ -5849,7 +5849,7 @@ _PyLong_GCD(PyObject *aarg, PyObject *barg) assert(size_a >= 0); _PyLong_SetSignAndDigitCount(c, 1, size_a); } - else if (Py_REFCNT(a) == 1) { + else if (_PyObject_IsUniquelyReferenced((PyObject *)a)) { c = (PyLongObject*)Py_NewRef(a); } else { @@ -5863,7 +5863,8 @@ _PyLong_GCD(PyObject *aarg, PyObject *barg) assert(size_a >= 0); _PyLong_SetSignAndDigitCount(d, 1, size_a); } - else if (Py_REFCNT(b) == 1 && size_a <= alloc_b) { + else if (_PyObject_IsUniquelyReferenced((PyObject *)b) + && size_a <= alloc_b) { d = (PyLongObject*)Py_NewRef(b); assert(size_a >= 0); _PyLong_SetSignAndDigitCount(d, 1, size_a); diff --git a/Objects/setobject.c b/Objects/setobject.c index d8340499be5..213bd821d8a 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -2444,7 +2444,7 @@ set_init(PyObject *so, PyObject *args, PyObject *kwds) if (!PyArg_UnpackTuple(args, Py_TYPE(self)->tp_name, 0, 1, &iterable)) return -1; - if (Py_REFCNT(self) == 1 && self->fill == 0) { + if (_PyObject_IsUniquelyReferenced((PyObject *)self) && self->fill == 0) { self->hash = -1; if (iterable == NULL) { return 0; @@ -2774,7 +2774,7 @@ int PySet_Add(PyObject *anyset, PyObject *key) { if (!PySet_Check(anyset) && - (!PyFrozenSet_Check(anyset) || Py_REFCNT(anyset) != 1)) { + (!PyFrozenSet_Check(anyset) || !_PyObject_IsUniquelyReferenced(anyset))) { PyErr_BadInternalCall(); return -1; } diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 94b7ae7e642..cd90b06d499 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -118,7 +118,7 @@ int PyTuple_SetItem(PyObject *op, Py_ssize_t i, PyObject *newitem) { PyObject **p; - if (!PyTuple_Check(op) || Py_REFCNT(op) != 1) { + if (!PyTuple_Check(op) || !_PyObject_IsUniquelyReferenced(op)) { Py_XDECREF(newitem); PyErr_BadInternalCall(); return -1; @@ -923,7 +923,7 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) v = (PyTupleObject *) *pv; if (v == NULL || !Py_IS_TYPE(v, &PyTuple_Type) || - (Py_SIZE(v) != 0 && Py_REFCNT(v) != 1)) { + (Py_SIZE(v) != 0 && !_PyObject_IsUniquelyReferenced(*pv))) { *pv = 0; Py_XDECREF(v); PyErr_BadInternalCall(); diff --git a/Python/marshal.c b/Python/marshal.c index 4f444d4671c..8b56de65755 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -11,6 +11,7 @@ #include "pycore_code.h" // _PyCode_New() #include "pycore_hashtable.h" // _Py_hashtable_t #include "pycore_long.h" // _PyLong_IsZero() +#include "pycore_object.h" // _PyObject_IsUniquelyReferenced #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_setobject.h" // _PySet_NextEntryRef() #include "pycore_unicodeobject.h" // _PyUnicode_InternImmortal() @@ -388,7 +389,7 @@ w_ref(PyObject *v, char *flag, WFILE *p) * But we use TYPE_REF always for interned string, to PYC file stable * as possible. */ - if (Py_REFCNT(v) == 1 && + if (_PyObject_IsUniquelyReferenced(v) && !(PyUnicode_CheckExact(v) && PyUnicode_CHECK_INTERNED(v))) { return 0; } diff --git a/configure b/configure index 9757b3419d3..28005cd1924 100755 --- a/configure +++ b/configure @@ -20191,6 +20191,12 @@ if test "x$ac_cv_func_splice" = xyes then : printf "%s\n" "#define HAVE_SPLICE 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "statx" "ac_cv_func_statx" +if test "x$ac_cv_func_statx" = xyes +then : + printf "%s\n" "#define HAVE_STATX 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "strftime" "ac_cv_func_strftime" if test "x$ac_cv_func_strftime" = xyes diff --git a/configure.ac b/configure.ac index f244e0b71a6..d20f6f8c40a 100644 --- a/configure.ac +++ b/configure.ac @@ -5251,7 +5251,7 @@ AC_CHECK_FUNCS([ \ setitimer setlocale setpgid setpgrp setpriority setregid setresgid \ setresuid setreuid setsid setuid setvbuf shutdown sigaction sigaltstack \ sigfillset siginterrupt sigpending sigrelse sigtimedwait sigwait \ - sigwaitinfo snprintf splice strftime strlcpy strsignal symlinkat sync \ + sigwaitinfo snprintf splice statx strftime strlcpy strsignal symlinkat sync \ sysconf tcgetpgrp tcsetpgrp tempnam timegm times tmpfile \ tmpnam tmpnam_r truncate ttyname_r umask uname unlinkat unlockpt utimensat utimes vfork \ wait wait3 wait4 waitid waitpid wcscoll wcsftime wcsxfrm wmemcmp writev \ diff --git a/pyconfig.h.in b/pyconfig.h.in index 72870411bc0..611408d88f0 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1285,6 +1285,9 @@ /* Define to 1 if you have the 'statvfs' function. */ #undef HAVE_STATVFS +/* Define to 1 if you have the 'statx' function. */ +#undef HAVE_STATX + /* Define if you have struct stat.st_mtim.tv_nsec */ #undef HAVE_STAT_TV_NSEC