bpo-45434: Remove pystrhex.h header file (GH-28923)

Move Include/pystrhex.h to Include/internal/pycore_strhex.h.
The header file only contains private functions.

The following C extensions are now built with Py_BUILD_CORE_MODULE
macro defined to get access to the internal C API:

* _blake2
* _hashopenssl
* _md5
* _sha1
* _sha3
* _ssl
* binascii

Doc/whatsnew/3.11.rst

@@ -598,3 +598,7 @@ Removed
   since Python 3.3. Use ``PyUnicode_CopyCharacters()`` or ``memcpy()``
   (``wchar_t*`` string), and ``PyUnicode_Fill()`` functions instead.
   (Contributed by Victor Stinner in :issue:`41123`.)
+
+* Remove the ``pystrhex.h`` header file. It only contains private functions.
+  C extensions should only include the main ``<Python.h>`` header file.
+  (Contributed by Victor Stinner in :issue:`45434`.)

Include/internal/pycore_strhex.h

@@ -0,0 +1,36 @@
+#ifndef Py_INTERNAL_STRHEX_H
+#define Py_INTERNAL_STRHEX_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+// Returns a str() containing the hex representation of argbuf.
+PyAPI_FUNC(PyObject*) _Py_strhex(const
+    char* argbuf,
+    const Py_ssize_t arglen);
+
+// Returns a bytes() containing the ASCII hex representation of argbuf.
+PyAPI_FUNC(PyObject*) _Py_strhex_bytes(
+    const char* argbuf,
+    const Py_ssize_t arglen);
+
+// These variants include support for a separator between every N bytes:
+PyAPI_FUNC(PyObject*) _Py_strhex_with_sep(
+    const char* argbuf,
+    const Py_ssize_t arglen,
+    const PyObject* sep,
+    const int bytes_per_group);
+PyAPI_FUNC(PyObject*) _Py_strhex_bytes_with_sep(
+    const char* argbuf,
+    const Py_ssize_t arglen,
+    const PyObject* sep,
+    const int bytes_per_group);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERNAL_STRHEX_H */

Makefile.pre.in

@@ -1179,7 +1179,6 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/pyport.h \
 		$(srcdir)/Include/pystate.h \
 		$(srcdir)/Include/pystrcmp.h \
-		$(srcdir)/Include/pystrhex.h \
 		$(srcdir)/Include/pystrtod.h \
 		$(srcdir)/Include/pythonrun.h \
 		$(srcdir)/Include/pythread.h \
@@ -1271,6 +1270,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/internal/pycore_pymem.h \
 		$(srcdir)/Include/internal/pycore_pystate.h \
 		$(srcdir)/Include/internal/pycore_runtime.h \
+		$(srcdir)/Include/internal/pycore_strhex.h \
 		$(srcdir)/Include/internal/pycore_structseq.h \
 		$(srcdir)/Include/internal/pycore_symtable.h \
 		$(srcdir)/Include/internal/pycore_sysmodule.h \

Misc/NEWS.d/next/C API/2021-10-13-14-42-46.bpo-45434.INNEEt.rst

@@ -0,0 +1,3 @@
+Remove the ``pystrhex.h`` header file. It only contains private functions. C
+extensions should only include the main ``<Python.h>`` header file. Patch by
+Victor Stinner.

Modules/Setup

@@ -212,19 +212,23 @@ _symtable symtablemodule.c
 # OPENSSL=/path/to/openssl/directory
 # _ssl _ssl.c \
 #     -I$(OPENSSL)/include -L$(OPENSSL)/lib \
-#     -lssl -lcrypto
+#     -lssl -lcrypto \
+#     -DPy_BUILD_CORE_BUILTIN
 #_hashlib _hashopenssl.c \
 #     -I$(OPENSSL)/include -L$(OPENSSL)/lib \
-#     -lcrypto
+#     -lcrypto \
+#     -DPy_BUILD_CORE_BUILTIN
 
 # To statically link OpenSSL:
 # _ssl _ssl.c \
 #     -I$(OPENSSL)/include -L$(OPENSSL)/lib \
 #     -l:libssl.a -Wl,--exclude-libs,libssl.a \
-#     -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a
+#     -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a \
+#     -DPy_BUILD_CORE_BUILTIN
 #_hashlib _hashopenssl.c \
 #     -I$(OPENSSL)/include -L$(OPENSSL)/lib \
-#     -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a
+#     -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a \
+#     -DPy_BUILD_CORE_BUILTIN
 
 # The crypt module is now disabled by default because it breaks builds
 # on many systems (where -lcrypt is needed), e.g. Linux (I believe).
@@ -255,18 +259,18 @@ _symtable symtablemodule.c
 # The _md5 module implements the RSA Data Security, Inc. MD5
 # Message-Digest Algorithm, described in RFC 1321.
 
-#_md5 md5module.c
+#_md5 md5module.c -DPy_BUILD_CORE_BUILTIN
 
 
 # The _sha module implements the SHA checksum algorithms.
 # (NIST's Secure Hash Algorithms.)
-#_sha1 sha1module.c
+#_sha1 sha1module.c -DPy_BUILD_CORE_BUILTIN
 #_sha256 sha256module.c -DPy_BUILD_CORE_BUILTIN
 #_sha512 sha512module.c -DPy_BUILD_CORE_BUILTIN
-#_sha3 _sha3/sha3module.c
+#_sha3 _sha3/sha3module.c -DPy_BUILD_CORE_BUILTIN
 
 # _blake module
-#_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c
+#_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c -DPy_BUILD_CORE_BUILTIN
 
 # The _tkinter module.
 #
@@ -339,7 +343,7 @@ _symtable symtablemodule.c
 
 
 # Helper module for various ascii-encoders
-#binascii binascii.c
+#binascii binascii.c -DPy_BUILD_CORE_MODULE
 
 # Andrew Kuchling's zlib module.
 # This require zlib 1.1.3 (or later).

Modules/_blake2/blake2b_impl.c

@@ -14,7 +14,7 @@
  */
 
 #include "Python.h"
-#include "pystrhex.h"
+#include "pycore_strhex.h"       // _Py_strhex()
 
 #include "../hashlib.h"
 #include "blake2ns.h"

Modules/_blake2/blake2s_impl.c

@@ -14,7 +14,7 @@
  */
 
 #include "Python.h"
-#include "pystrhex.h"
+#include "pycore_strhex.h"        // _Py_strhex()
 
 #include "../hashlib.h"
 #include "blake2ns.h"

Modules/_hashopenssl.c

@@ -22,7 +22,7 @@
 
 #include "Python.h"
 #include "hashlib.h"
-#include "pystrhex.h"
+#include "pycore_strhex.h"        // _Py_strhex()
 
 /* EVP is the preferred interface to hashing in OpenSSL */
 #include <openssl/evp.h>

Modules/_sha3/sha3module.c

@@ -16,7 +16,7 @@
  */
 
 #include "Python.h"
-#include "pystrhex.h"
+#include "pycore_strhex.h"        // _Py_strhex()
 #include "../hashlib.h"
 
 /* **************************************************************************

Modules/binascii.c

@@ -56,9 +56,9 @@
 #define PY_SSIZE_T_CLEAN
 
 #include "Python.h"
-#include "pystrhex.h"
+#include "pycore_strhex.h"        // _Py_strhex_bytes_with_sep()
 #ifdef USE_ZLIB_CRC32
-#include "zlib.h"
+#  include "zlib.h"
 #endif
 
 typedef struct binascii_state {

Modules/md5module.c

@@ -18,7 +18,7 @@
 
 #include "Python.h"
 #include "hashlib.h"
-#include "pystrhex.h"
+#include "pycore_strhex.h"        // _Py_strhex()
 
 /*[clinic input]
 module _md5

Modules/sha1module.c

@@ -18,7 +18,7 @@
 
 #include "Python.h"
 #include "hashlib.h"
-#include "pystrhex.h"
+#include "pycore_strhex.h"        // _Py_strhex()
 
 /*[clinic input]
 module _sha1

Modules/sha256module.c

@@ -18,9 +18,9 @@
 
 #include "Python.h"
 #include "pycore_bitutils.h"      // _Py_bswap32()
+#include "pycore_strhex.h"        // _Py_strhex()
 #include "structmember.h"         // PyMemberDef
 #include "hashlib.h"
-#include "pystrhex.h"
 
 /*[clinic input]
 module _sha256

Modules/sha512module.c

@@ -18,9 +18,9 @@
 
 #include "Python.h"
 #include "pycore_bitutils.h"      // _Py_bswap64()
+#include "pycore_strhex.h"        // _Py_strhex()
 #include "structmember.h"         // PyMemberDef
 #include "hashlib.h"
-#include "pystrhex.h"
 
 /*[clinic input]
 module _sha512

Objects/bytearrayobject.c

@@ -4,9 +4,9 @@
 #include "Python.h"
 #include "pycore_abstract.h"      // _PyIndex_Check()
 #include "pycore_bytes_methods.h"
-#include "pycore_object.h"
+#include "pycore_object.h"        // _PyObject_GC_UNTRACK()
+#include "pycore_strhex.h"        // _Py_strhex_with_sep()
 #include "bytesobject.h"
-#include "pystrhex.h"
 
 /*[clinic input]
 class bytearray "PyByteArrayObject *" "&PyByteArray_Type"

Objects/bytesobject.c

@@ -10,8 +10,8 @@
 #include "pycore_initconfig.h"    // _PyStatus_OK()
 #include "pycore_object.h"        // _PyObject_GC_TRACK
 #include "pycore_pymem.h"         // PYMEM_CLEANBYTE
+#include "pycore_strhex.h"        // _Py_strhex_with_sep()
 
-#include "pystrhex.h"
 #include <stddef.h>
 
 /*[clinic input]

Objects/memoryobject.c

@@ -11,10 +11,10 @@
  */
 
 #include "Python.h"
-#include "pycore_abstract.h"   // _PyIndex_Check()
-#include "pycore_object.h"
-#include "pystrhex.h"
-#include <stddef.h>
+#include "pycore_abstract.h"      // _PyIndex_Check()
+#include "pycore_object.h"        // _PyObject_GC_UNTRACK()
+#include "pycore_strhex.h"        // _Py_strhex_with_sep()
+#include <stddef.h>               // offsetof()
 
 /*[clinic input]
 class memoryview "PyMemoryViewObject *" "&PyMemoryView_Type"

PCbuild/pythoncore.vcxproj

@@ -209,6 +209,7 @@
     <ClInclude Include="..\Include\internal\pycore_pymem.h" />
     <ClInclude Include="..\Include\internal\pycore_pystate.h" />
     <ClInclude Include="..\Include\internal\pycore_runtime.h" />
+    <ClInclude Include="..\Include\internal\pycore_strhex.h" />
     <ClInclude Include="..\Include\internal\pycore_structseq.h" />
     <ClInclude Include="..\Include\internal\pycore_sysmodule.h" />
     <ClInclude Include="..\Include\internal\pycore_symtable.h" />
@@ -248,7 +249,6 @@
     <ClInclude Include="..\Include\pyport.h" />
     <ClInclude Include="..\Include\pystate.h" />
     <ClInclude Include="..\Include\pystrcmp.h" />
-    <ClInclude Include="..\Include\pystrhex.h" />
     <ClInclude Include="..\Include\pystrtod.h" />
     <ClInclude Include="..\Include\pythonrun.h" />
     <ClInclude Include="..\Include\pythread.h" />

PCbuild/pythoncore.vcxproj.filters

@@ -201,9 +201,6 @@
     <ClInclude Include="..\Include\pystrtod.h">
       <Filter>Include</Filter>
     </ClInclude>
-    <ClInclude Include="..\Include\pystrhex.h">
-      <Filter>Include</Filter>
-    </ClInclude>
     <ClInclude Include="..\Include\Python.h">
       <Filter>Include</Filter>
     </ClInclude>
@@ -588,6 +585,9 @@
     <ClInclude Include="..\Include\internal\pycore_runtime.h">
       <Filter>Include\internal</Filter>
     </ClInclude>
+    <ClInclude Include="..\Include\internal\pycore_strhex.h">
+      <Filter>Include\internal</Filter>
+    </ClInclude>
     <ClInclude Include="..\Include\internal\pycore_sysmodule.h">
       <Filter>Include\internal</Filter>
     </ClInclude>

Python/pystrhex.c

@@ -1,8 +1,8 @@
-/* bytes to hex implementation */
+/* Format bytes as hexadecimal */
 
 #include "Python.h"
+#include "pycore_strhex.h"       // _Py_strhex_with_sep()
 
-#include "pystrhex.h"
 
 static PyObject *_Py_strhex_impl(const char* argbuf, const Py_ssize_t arglen,
                                  const PyObject* sep, int bytes_per_sep_group,

setup.py

@@ -1709,12 +1709,12 @@ def detect_compress_exts(self):
 
         # Helper module for various ascii-encoders.  Uses zlib for an optimized
         # crc32 if we have it.  Otherwise binascii uses its own.
+        extra_compile_args = ['-DPy_BUILD_CORE_MODULE']
         if have_zlib:
-            extra_compile_args = ['-DUSE_ZLIB_CRC32']
+            extra_compile_args.append('-DUSE_ZLIB_CRC32')
             libraries = ['z']
             extra_link_args = zlib_extra_link_args
         else:
-            extra_compile_args = []
             libraries = []
             extra_link_args = []
         self.add(Extension('binascii', ['binascii.c'],
@@ -2469,6 +2469,7 @@ def split_var(name, sep):
             library_dirs=openssl_libdirs,
             libraries=openssl_libs,
             runtime_library_dirs=runtime_library_dirs,
+            extra_compile_args=['-DPy_BUILD_CORE_MODULE'],
         )
 
         # This static linking is NOT OFFICIALLY SUPPORTED.
@@ -2530,27 +2531,29 @@ def detect_hash_builtins(self):
         if "sha256" in configured:
             self.add(Extension(
                 '_sha256', ['sha256module.c'],
+                depends=['hashlib.h'],
                 extra_compile_args=['-DPy_BUILD_CORE_MODULE'],
-                depends=['hashlib.h']
             ))
 
         if "sha512" in configured:
             self.add(Extension(
                 '_sha512', ['sha512module.c'],
+                depends=['hashlib.h'],
                 extra_compile_args=['-DPy_BUILD_CORE_MODULE'],
-                depends=['hashlib.h']
             ))
 
         if "md5" in configured:
             self.add(Extension(
                 '_md5', ['md5module.c'],
-                depends=['hashlib.h']
+                depends=['hashlib.h'],
+                extra_compile_args=['-DPy_BUILD_CORE_MODULE'],
             ))
 
         if "sha1" in configured:
             self.add(Extension(
                 '_sha1', ['sha1module.c'],
-                depends=['hashlib.h']
+                depends=['hashlib.h'],
+                extra_compile_args=['-DPy_BUILD_CORE_MODULE'],
             ))
 
         if "blake2" in configured:
@@ -2565,7 +2568,8 @@ def detect_hash_builtins(self):
                     '_blake2/blake2b_impl.c',
                     '_blake2/blake2s_impl.c'
                 ],
-                depends=blake2_deps
+                depends=blake2_deps,
+                extra_compile_args=['-DPy_BUILD_CORE_MODULE'],
             ))
 
         if "sha3" in configured:
@@ -2576,7 +2580,8 @@ def detect_hash_builtins(self):
             self.add(Extension(
                 '_sha3',
                 ['_sha3/sha3module.c'],
-                depends=sha3_deps
+                depends=sha3_deps,
+                extra_compile_args=['-DPy_BUILD_CORE_MODULE'],
             ))
 
     def detect_nis(self):