Merge remote-tracking branch 'upstream/main' into gh-158803-bytes-join-race

Author: christianaurichzm

Doc/data/stable_abi.dat

@@ -1022,6 +1022,19 @@ Debug options
 
    .. versionadded:: 3.6
 
+.. option:: --with-hwaddress-sanitizer
+
+   Enable HWAddressSanitizer memory error detector, ``hwasan`` (default is no).
+   Note that on x86-64 this uses `page aliasing
+   <https://clang.llvm.org/docs/HardwareAssistedAddressSanitizerDesign.html#supported-architectures>`_,
+   which only tags heap allocations and is unsafe for programs that ``fork()``,
+   including much of the test suite.
+   See the `LLVM HWASan design documentation
+   <https://clang.llvm.org/docs/HardwareAssistedAddressSanitizerDesign.html>`_
+   for more information.
+
+   .. versionadded:: next
+
 .. option:: --with-memory-sanitizer
 
    Enable MemorySanitizer allocation error detector, ``msan`` (default is no).

Doc/using/configure.rst

@@ -1171,6 +1171,11 @@ Build changes
 
   (Contributed by Stan Ulbrych in :gh:`139314`.)
 
+* Add the :option:`--with-hwaddress-sanitizer` :program:`configure` option to
+  build with `HWAddressSanitizer <https://clang.llvm.org/docs/HardwareAssistedAddressSanitizerDesign.html>`_.
+
+  (Contributed by Stan Ulbrych, Florian Mayer and AnnaAr321 in :gh:`156049`.)
+
 
 C API changes
 =============
@@ -1181,6 +1186,10 @@ New features
 * Add :c:func:`Py_HashBuffer` to the limited C API.
   (Contributed by Victor Stinner in :gh:`148233`.)
 
+* Add :c:macro:`Py_SETREF` and :c:macro:`Py_XSETREF` macros to the limited C
+  API.
+  (Contributed by Nathan Goldbaum and Victor Stinner in :gh:`158842`.)
+
 Porting to Python 3.16
 ----------------------
 

Doc/whatsnew/3.16.rst

@@ -326,87 +326,6 @@ _PyObject_GenericSetAttrWithDict(PyObject *, PyObject *,
 
 PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *);
 
-/* Safely decref `dst` and set `dst` to `src`.
- *
- * As in case of Py_CLEAR "the obvious" code can be deadly:
- *
- *     Py_DECREF(dst);
- *     dst = src;
- *
- * The safe way is:
- *
- *      Py_SETREF(dst, src);
- *
- * That arranges to set `dst` to `src` _before_ decref'ing, so that any code
- * triggered as a side-effect of `dst` getting torn down no longer believes
- * `dst` points to a valid object.
- *
- * Temporary variables are used to only evaluate macro arguments once and so
- * avoid the duplication of side effects. _Py_TYPEOF(), C++ auto, or memcpy()
- * is used to avoid a miscompilation caused by type punning. See Py_CLEAR()
- * comment for implementation details about type punning.
- *
- * The memcpy() implementation does not emit a compiler warning if 'src' has
- * not the same type than 'src': any pointer type is accepted for 'src'.
- */
-#ifdef _Py_TYPEOF
-#define Py_SETREF(dst, src) \
-    do { \
-        _Py_TYPEOF(&(dst)) _tmp_dst_ptr = &(dst); \
-        _Py_TYPEOF(dst) _tmp_old_dst = (*_tmp_dst_ptr); \
-        *_tmp_dst_ptr = (src); \
-        Py_DECREF(_tmp_old_dst); \
-    } while (0)
-#elif defined(__cplusplus) && (__cplusplus >= 201103L ||  _MSVC_LANG >= 201103L)
-#define Py_SETREF(dst, src) \
-    do { \
-        auto _tmp_dst_ptr = &(dst); \
-        auto _tmp_old_dst = (*_tmp_dst_ptr); \
-        *_tmp_dst_ptr = (src); \
-        Py_DECREF(_tmp_old_dst); \
-    } while (0)
-#else
-#define Py_SETREF(dst, src) \
-    do { \
-        PyObject **_tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \
-        PyObject *_tmp_old_dst = (*_tmp_dst_ptr); \
-        PyObject *_tmp_src = _PyObject_CAST(src); \
-        memcpy(_tmp_dst_ptr, &_tmp_src, sizeof(PyObject*)); \
-        Py_DECREF(_tmp_old_dst); \
-    } while (0)
-#endif
-
-/* Py_XSETREF() is a variant of Py_SETREF() that uses Py_XDECREF() instead of
- * Py_DECREF().
- */
-#ifdef _Py_TYPEOF
-#define Py_XSETREF(dst, src) \
-    do { \
-        _Py_TYPEOF(&(dst)) _tmp_dst_ptr = &(dst); \
-        _Py_TYPEOF(dst) _tmp_old_dst = (*_tmp_dst_ptr); \
-        *_tmp_dst_ptr = (src); \
-        Py_XDECREF(_tmp_old_dst); \
-    } while (0)
-#elif defined(__cplusplus) && (__cplusplus >= 201103L ||  _MSVC_LANG >= 201103L)
-#define Py_XSETREF(dst, src) \
-    do { \
-        auto _tmp_dst_ptr = &(dst); \
-        auto _tmp_old_dst = (*_tmp_dst_ptr); \
-        *_tmp_dst_ptr = (src); \
-        Py_XDECREF(_tmp_old_dst); \
-    } while (0)
-#else
-#define Py_XSETREF(dst, src) \
-    do { \
-        PyObject **_tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \
-        PyObject *_tmp_old_dst = (*_tmp_dst_ptr); \
-        PyObject *_tmp_src = _PyObject_CAST(src); \
-        memcpy(_tmp_dst_ptr, &_tmp_src, sizeof(PyObject*)); \
-        Py_XDECREF(_tmp_old_dst); \
-    } while (0)
-#endif
-
-
 /* Define a pair of assertion macros:
    _PyObject_ASSERT_FROM(), _PyObject_ASSERT_WITH_MSG() and _PyObject_ASSERT().
 

Include/cpython/object.h

@@ -567,6 +567,12 @@ extern "C" {
 #      define _Py_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address))
 #    endif
 #  endif
+#  if __has_feature(hwaddress_sanitizer)
+#    if !defined(_Py_ADDRESS_SANITIZER)
+#      define _Py_ADDRESS_SANITIZER
+#      define _Py_NO_SANITIZE_ADDRESS __attribute__((no_sanitize("hwaddress")))
+#    endif
+#  endif
 #  if __has_feature(thread_sanitizer)
 #    if !defined(_Py_THREAD_SANITIZER)
 #      define _Py_THREAD_SANITIZER
@@ -577,6 +583,9 @@ extern "C" {
 #  if defined(__SANITIZE_ADDRESS__)
 #    define _Py_ADDRESS_SANITIZER
 #    define _Py_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address))
+#  elif defined(__SANITIZE_HWADDRESS__)
+#    define _Py_ADDRESS_SANITIZER
+#    define _Py_NO_SANITIZE_ADDRESS __attribute__((no_sanitize("hwaddress")))
 #  endif
 #  if defined(__SANITIZE_THREAD__)
 #    define _Py_THREAD_SANITIZER

Include/pyport.h

@@ -516,6 +516,87 @@ static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op)
 #endif
 
 
+/* Safely decref `dst` and set `dst` to `src`.
+ *
+ * As in case of Py_CLEAR "the obvious" code can be deadly:
+ *
+ *     Py_DECREF(dst);
+ *     dst = src;
+ *
+ * The safe way is:
+ *
+ *      Py_SETREF(dst, src);
+ *
+ * That arranges to set `dst` to `src` _before_ decref'ing, so that any code
+ * triggered as a side-effect of `dst` getting torn down no longer believes
+ * `dst` points to a valid object.
+ *
+ * Temporary variables are used to only evaluate macro arguments once and so
+ * avoid the duplication of side effects. _Py_TYPEOF(), C++ auto, or memcpy()
+ * is used to avoid a miscompilation caused by type punning. See Py_CLEAR()
+ * comment for implementation details about type punning.
+ *
+ * The memcpy() implementation does not emit a compiler warning if 'src' has
+ * not the same type than 'src': any pointer type is accepted for 'src'.
+ */
+#ifdef _Py_TYPEOF
+#define Py_SETREF(dst, src) \
+    do { \
+        _Py_TYPEOF(&(dst)) _tmp_dst_ptr = &(dst); \
+        _Py_TYPEOF(dst) _tmp_old_dst = (*_tmp_dst_ptr); \
+        *_tmp_dst_ptr = (src); \
+        Py_DECREF(_tmp_old_dst); \
+    } while (0)
+#elif defined(__cplusplus) && (__cplusplus >= 201103L ||  _MSVC_LANG >= 201103L)
+#define Py_SETREF(dst, src) \
+    do { \
+        auto _tmp_dst_ptr = &(dst); \
+        auto _tmp_old_dst = (*_tmp_dst_ptr); \
+        *_tmp_dst_ptr = (src); \
+        Py_DECREF(_tmp_old_dst); \
+    } while (0)
+#else
+#define Py_SETREF(dst, src) \
+    do { \
+        PyObject **_tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \
+        PyObject *_tmp_old_dst = (*_tmp_dst_ptr); \
+        PyObject *_tmp_src = _PyObject_CAST(src); \
+        memcpy(_tmp_dst_ptr, &_tmp_src, sizeof(PyObject*)); \
+        Py_DECREF(_tmp_old_dst); \
+    } while (0)
+#endif
+
+/* Py_XSETREF() is a variant of Py_SETREF() that uses Py_XDECREF() instead of
+ * Py_DECREF().
+ */
+#ifdef _Py_TYPEOF
+#define Py_XSETREF(dst, src) \
+    do { \
+        _Py_TYPEOF(&(dst)) _tmp_dst_ptr = &(dst); \
+        _Py_TYPEOF(dst) _tmp_old_dst = (*_tmp_dst_ptr); \
+        *_tmp_dst_ptr = (src); \
+        Py_XDECREF(_tmp_old_dst); \
+    } while (0)
+#elif defined(__cplusplus) && (__cplusplus >= 201103L ||  _MSVC_LANG >= 201103L)
+#define Py_XSETREF(dst, src) \
+    do { \
+        auto _tmp_dst_ptr = &(dst); \
+        auto _tmp_old_dst = (*_tmp_dst_ptr); \
+        *_tmp_dst_ptr = (src); \
+        Py_XDECREF(_tmp_old_dst); \
+    } while (0)
+#else
+#define Py_XSETREF(dst, src) \
+    do { \
+        PyObject **_tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \
+        PyObject *_tmp_old_dst = (*_tmp_dst_ptr); \
+        PyObject *_tmp_src = _PyObject_CAST(src); \
+        memcpy(_tmp_dst_ptr, &_tmp_src, sizeof(PyObject*)); \
+        Py_XDECREF(_tmp_old_dst); \
+    } while (0)
+#endif
+
+
 /* Function to use in case the object pointer can be NULL: */
 static inline void Py_XINCREF(PyObject *op)
 {

Include/refcount.h

@@ -546,9 +546,11 @@ def test_interrupt_blocking_call(self):
         # gh-74112: interrupt the main thread blocked in time.sleep().
         timer = threading.Timer(0.1, self.ex.interrupt_the_server)
         self.addCleanup(timer.join)
-        timer.start()
         start = time.monotonic()
         with self.assertRaises(KeyboardInterrupt):
+            # On a loaded machine the signal can arrive before the main
+            # thread reaches time.sleep(), so start the timer in the block.
+            timer.start()
             time.sleep(support.SHORT_TIMEOUT)
         self.assertLess(time.monotonic() - start, support.SHORT_TIMEOUT / 2)
 

Lib/idlelib/idle_test/test_run.py

@@ -437,9 +437,10 @@ def parse_config_h(fp, vars=None):
     if vars is None:
         vars = {}
     import re
-    define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n")
-    undef_rx = re.compile("/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n")
-    quoted_re = re.compile('^"(.*)"$')
+    name_rx = '(?:[A-Z]|_Py_)[A-Za-z0-9_]+'
+    define_rx = re.compile(fr"#define ({name_rx}) (.*)\n")
+    undef_rx = re.compile(fr"/[*] #undef ({name_rx}) [*]/\n")
+    quoted_re = re.compile(r'"(.*)"')
 
     while True:
         line = fp.readline()
@@ -448,7 +449,7 @@ def parse_config_h(fp, vars=None):
         m = define_rx.match(line)
         if m:
             n, v = m.group(1, 2)
-            if mq := quoted_re.match(v):
+            if mq := quoted_re.fullmatch(v):
                 v = mq.group(1)
             try:
                 if n in _ALWAYS_STR:

Lib/sysconfig/__init__.py

@@ -397,7 +397,9 @@ def get_build_info():
 
     # --with-address-sanitizer
     sanitizers = []
-    if support.check_sanitizer(address=True):
+    if support.check_sanitizer(hwaddress=True):
+        sanitizers.append("HWASAN")
+    elif support.check_sanitizer(address=True):
         sanitizers.append("ASAN")
     # --with-memory-sanitizer
     if support.check_sanitizer(memory=True):

Lib/test/libregrtest/utils.py

@@ -448,11 +448,12 @@ def skip_if_buildbot(reason=None):
         isbuildbot = False
     return unittest.skipIf(isbuildbot, reason)
 
-def check_sanitizer(*, address=False, memory=False, ub=False, thread=False,
-                    function=True):
+def check_sanitizer(*, address=False, hwaddress=False, memory=False, ub=False,
+                    thread=False, function=True):
     """Returns True if Python is compiled with sanitizer support"""
-    if not (address or memory or ub or thread):
-        raise ValueError('At least one of address, memory, ub or thread must be True')
+    if not (address or hwaddress or memory or ub or thread):
+        raise ValueError('At least one of address, hwaddress, memory, ub or '
+                         'thread must be True')
 
 
     cflags = sysconfig.get_config_var('CFLAGS') or ''
@@ -461,9 +462,14 @@ def check_sanitizer(*, address=False, memory=False, ub=False, thread=False,
         '-fsanitize=memory' in cflags or
         '--with-memory-sanitizer' in config_args
     )
+    hwaddress_sanitizer = (
+        '-fsanitize=hwaddress' in cflags or
+        '--with-hwaddress-sanitizer' in config_args
+    )
     address_sanitizer = (
         '-fsanitize=address' in cflags or
-        '--with-address-sanitizer' in config_args
+        '--with-address-sanitizer' in config_args or
+        hwaddress_sanitizer
     )
     ub_sanitizer = (
         '-fsanitize=undefined' in cflags or
@@ -479,6 +485,7 @@ def check_sanitizer(*, address=False, memory=False, ub=False, thread=False,
     return (
         (memory and memory_sanitizer) or
         (address and address_sanitizer) or
+        (hwaddress and hwaddress_sanitizer) or
         (ub and ub_sanitizer) or
         (thread and thread_sanitizer) or
         (function and function_sanitizer)