From 464e6616be86129e33af6d9e43540c260d6804d5 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sun, 14 Nov 2021 11:02:24 +0200 Subject: [PATCH] bpo-45800: Move pyexpat build setup into configure (GH-29547) Settings for :mod:`pyexpat` C extension are now detected by ``configure``. The bundled ``expat`` library is built in ``Makefile``. Signed-off-by: Christian Heimes --- Makefile.pre.in | 54 ++++++- .../2021-11-13-16-40-05.bpo-45800.5Hz6nr.rst | 2 + Modules/Setup | 4 +- configure | 21 +++ configure.ac | 14 ++ setup.py | 140 +++++++++++------- 6 files changed, 177 insertions(+), 58 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2021-11-13-16-40-05.bpo-45800.5Hz6nr.rst diff --git a/Makefile.pre.in b/Makefile.pre.in index a17054b6f65de0..b6e4c2316bccc1 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -537,6 +537,33 @@ LIBMPDEC_HEADERS= \ LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a +########################################################################## +# pyexpat's expat library + +LIBEXPAT_OBJS= \ + Modules/expat/xmlparse.o \ + Modules/expat/xmlrole.o \ + Modules/expat/xmltok.o + +LIBEXPAT_HEADERS= \ + Modules/expat/ascii.h \ + Modules/expat/asciitab.h \ + Modules/expat/expat.h \ + Modules/expat/expat_config.h \ + Modules/expat/expat_external.h \ + Modules/expat/iasciitab.h \ + Modules/expat/internal.h \ + Modules/expat/latin1tab.h \ + Modules/expat/nametab.h \ + Modules/expat/pyexpatns.h \ + Modules/expat/siphash.h \ + Modules/expat/utf8tab.h \ + Modules/expat/xmlrole.h \ + Modules/expat/xmltok.h \ + Modules/expat/xmltok_impl.h + +LIBEXPAT_A= Modules/expat/libexpat.a + ######################################################################### # Rules @@ -688,7 +715,7 @@ $(srcdir)/Modules/_blake2/blake2s_impl.c: $(srcdir)/Modules/_blake2/blake2b_impl # -s, --silent or --quiet is always the first char. # Under BSD make, MAKEFLAGS might be " -s -v x=y". # Ignore macros passed by GNU make, passed after -- -sharedmods: $(BUILDPYTHON) pybuilddir.txt @LIBMPDEC_INTERNAL@ +sharedmods: $(BUILDPYTHON) pybuilddir.txt @LIBMPDEC_INTERNAL@ @LIBEXPAT_INTERNAL@ @case "`echo X $$MAKEFLAGS | sed 's/^X //;s/ -- .*//'`" in \ *\ -s*|s*) quiet="-q";; \ *) quiet="";; \ @@ -828,6 +855,27 @@ $(LIBMPDEC_A): $(LIBMPDEC_OBJS) -rm -f $@ $(AR) $(ARFLAGS) $@ $(LIBMPDEC_OBJS) +########################################################################## +# Build static libexpat.a +LIBEXPAT_CFLAGS=$(PY_STDMODULE_CFLAGS) $(CCSHARED) @LIBEXPAT_CFLAGS@ + +# for setup.py +EXPAT_CFLAGS=@LIBEXPAT_CFLAGS@ +EXPAT_LDFLAGS=@LIBEXPAT_LDFLAGS@ + +Modules/expat/xmlparse.o: $(srcdir)/Modules/expat/xmlparse.c $(LIBEXPAT_HEADERS) $(PYTHON_HEADERS) + $(CC) -c $(LIBEXPAT_CFLAGS) -o $@ $(srcdir)/Modules/expat/xmlparse.c + +Modules/expat/xmlrole.o: $(srcdir)/Modules/expat/xmlrole.c $(LIBEXPAT_HEADERS) $(PYTHON_HEADERS) + $(CC) -c $(LIBEXPAT_CFLAGS) -o $@ $(srcdir)/Modules/expat/xmlrole.c + +Modules/expat/xmltok.o: $(srcdir)/Modules/expat/xmltok.c $(LIBEXPAT_HEADERS) $(PYTHON_HEADERS) + $(CC) -c $(LIBEXPAT_CFLAGS) -o $@ $(srcdir)/Modules/expat/xmltok.c + +$(LIBEXPAT_A): $(LIBEXPAT_OBJS) + -rm -f $@ + $(AR) $(ARFLAGS) $@ $(LIBEXPAT_OBJS) + # create relative links from build/lib.platform/egg.so to Modules/egg.so # pybuilddir.txt is created too late. We cannot use it in Makefile # targets. ln --relative is not portable. @@ -2407,12 +2455,12 @@ Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h MODULE_CMATH_DEPS=$(srcdir)/Modules/_math.h MODULE_MATH_DEPS=$(srcdir)/Modules/_math.h -MODULE_PYEXPAT_DEPS=$(srcdir)/Modules/expat/ascii.h $(srcdir)/Modules/expat/asciitab.h $(srcdir)/Modules/expat/expat.h $(srcdir)/Modules/expat/expat_config.h $(srcdir)/Modules/expat/expat_external.h $(srcdir)/Modules/expat/internal.h $(srcdir)/Modules/expat/latin1tab.h $(srcdir)/Modules/expat/utf8tab.h $(srcdir)/Modules/expat/xmlrole.h $(srcdir)/Modules/expat/xmltok.h $(srcdir)/Modules/expat/xmltok_impl.h +MODULE_PYEXPAT_DEPS=$(LIBEXPAT_HEADERS) @LIBEXPAT_INTERNAL@ MODULE_UNICODEDATA_DEPS=$(srcdir)/Modules/unicodedata_db.h $(srcdir)/Modules/unicodename_db.h MODULE__BLAKE2_DEPS=$(srcdir)/Modules/_blake2/impl/blake2-config.h $(srcdir)/Modules/_blake2/impl/blake2-dispatch.c $(srcdir)/Modules/_blake2/impl/blake2-impl.h $(srcdir)/Modules/_blake2/impl/blake2-kat.h $(srcdir)/Modules/_blake2/impl/blake2.h $(srcdir)/Modules/_blake2/impl/blake2b-load-sse2.h $(srcdir)/Modules/_blake2/impl/blake2b-load-sse41.h $(srcdir)/Modules/_blake2/impl/blake2b-ref.c $(srcdir)/Modules/_blake2/impl/blake2b-round.h $(srcdir)/Modules/_blake2/impl/blake2b-test.c $(srcdir)/Modules/_blake2/impl/blake2b.c $(srcdir)/Modules/_blake2/impl/blake2bp-test.c $(srcdir)/Modules/_blake2/impl/blake2bp.c $(srcdir)/Modules/_blake2/impl/blake2s-load-sse2.h $(srcdir)/Modules/_blake2/impl/blake2s-load-sse41.h $(srcdir)/Modules/_blake2/impl/blake2s-load-xop.h $(srcdir)/Modules/_blake2/impl/blake2s-ref.c $(srcdir)/Modules/_blake2/impl/blake2s-round.h $(srcdir)/Modules/_blake2/impl/blake2s-test.c $(srcdir)/Modules/_blake2/impl/blake2s.c $(srcdir)/Modules/_blake2/impl/blake2sp-test.c $(srcdir)/Modules/_blake2/impl/blake2sp.c $(srcdir)/Modules/hashlib.h MODULE__CTYPES_DEPS=$(srcdir)/Modules/_ctypes/ctypes.h MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h $(LIBMPDEC_HEADERS) @LIBMPDEC_INTERNAL@ -MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/expat/ascii.h $(srcdir)/Modules/expat/asciitab.h $(srcdir)/Modules/expat/expat.h $(srcdir)/Modules/expat/expat_config.h $(srcdir)/Modules/expat/expat_external.h $(srcdir)/Modules/expat/internal.h $(srcdir)/Modules/expat/latin1tab.h $(srcdir)/Modules/expat/utf8tab.h $(srcdir)/Modules/expat/xmlparse.c $(srcdir)/Modules/expat/xmlrole.c $(srcdir)/Modules/expat/xmlrole.h $(srcdir)/Modules/expat/xmltok.c $(srcdir)/Modules/expat/xmltok.h $(srcdir)/Modules/expat/xmltok_impl.h $(srcdir)/Modules/pyexpat.c +MODULE__ELEMENTTREE_DEPS=$(LIBEXPAT_HEADERS) @LIBEXPAT_INTERNAL@ MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h diff --git a/Misc/NEWS.d/next/Build/2021-11-13-16-40-05.bpo-45800.5Hz6nr.rst b/Misc/NEWS.d/next/Build/2021-11-13-16-40-05.bpo-45800.5Hz6nr.rst new file mode 100644 index 00000000000000..8da3fcaebd06c8 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-11-13-16-40-05.bpo-45800.5Hz6nr.rst @@ -0,0 +1,2 @@ +Settings for :mod:`pyexpat` C extension are now detected by ``configure``. +The bundled ``expat`` library is built in ``Makefile``. diff --git a/Modules/Setup b/Modules/Setup index 39dc39d534494e..608866d9cedd53 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -172,8 +172,8 @@ time timemodule.c #select selectmodule.c # XML -#_elementtree -I$(srcdir)/Modules/expat _elementtree.c -#pyexpat -I$(srcdir)/Modules/expat expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c +#_elementtree _elementtree.c $(EXPAT_CFLAGS) +#pyexpat pyexpat.c $(EXPAT_CFLAGS) $(EXPAT_LDFLAGS) # hashing builtins #_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c diff --git a/configure b/configure index 43516a4f797362..53dc4a23f80631 100755 --- a/configure +++ b/configure @@ -664,6 +664,9 @@ LIBMPDEC_INTERNAL LIBMPDEC_LDFLAGS LIBMPDEC_CFLAGS LIBFFI_INCLUDEDIR +LIBEXPAT_INTERNAL +LIBEXPAT_LDFLAGS +LIBEXPAT_CFLAGS TZPATH SHLIBS CFLAGSFORSHARED @@ -10715,6 +10718,24 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_system_expat" >&5 $as_echo "$with_system_expat" >&6; } +if test "x$with_system_expat" = xyes; then : + + LIBEXPAT_CFLAGS="" + LIBEXPAT_LDFLAGS="-lexpat" + LIBEXPAT_INTERNAL= + +else + + LIBEXPAT_CFLAGS="-I\$(srcdir)/Modules/expat" + LIBEXPAT_LDFLAGS="-lm \$(LIBEXPAT_A)" + LIBEXPAT_INTERNAL="\$(LIBEXPAT_A)" + +fi + + + + + # Check for use of the system libffi library { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-system-ffi" >&5 $as_echo_n "checking for --with-system-ffi... " >&6; } diff --git a/configure.ac b/configure.ac index c3b52e3910d517..f43030e481068e 100644 --- a/configure.ac +++ b/configure.ac @@ -3017,6 +3017,20 @@ AC_ARG_WITH(system_expat, AC_MSG_RESULT($with_system_expat) +AS_VAR_IF([with_system_expat], [yes], [ + LIBEXPAT_CFLAGS="" + LIBEXPAT_LDFLAGS="-lexpat" + LIBEXPAT_INTERNAL= +], [ + LIBEXPAT_CFLAGS="-I\$(srcdir)/Modules/expat" + LIBEXPAT_LDFLAGS="-lm \$(LIBEXPAT_A)" + LIBEXPAT_INTERNAL="\$(LIBEXPAT_A)" +]) + +AC_SUBST([LIBEXPAT_CFLAGS]) +AC_SUBST([LIBEXPAT_LDFLAGS]) +AC_SUBST([LIBEXPAT_INTERNAL]) + # Check for use of the system libffi library AC_MSG_CHECKING(for --with-system-ffi) AC_ARG_WITH(system_ffi, diff --git a/setup.py b/setup.py index 86ce5d2d91f37a..c62f0427fd3787 100644 --- a/setup.py +++ b/setup.py @@ -364,6 +364,57 @@ def find_module_file(module, dirlist): return os.path.join(dirs[0], module) +def parse_cflags(flags): + """Parse a string with compiler flags (-I, -D, -U, extra) + + Distutils appends extra args to the compiler arguments. Some flags like + -I must appear earlier. Otherwise the pre-processor picks up files + from system inclue directories. + """ + include_dirs = [] + define_macros = [] + undef_macros = [] + extra_compile_args = [] + if flags is not None: + # shlex.split(None) reads from stdin + for token in shlex.split(flags): + switch = token[0:2] + value = token[2:] + if switch == '-I': + include_dirs.append(value) + elif switch == '-D': + key, _, val = value.partition("=") + if not val: + val = None + define_macros.append((key, val)) + elif switch == '-U': + undef_macros.append(value) + else: + extra_compile_args.append(token) + + return include_dirs, define_macros, undef_macros, extra_compile_args + + +def parse_ldflags(flags): + """Parse a string with linker flags (-L, -l, extra)""" + library_dirs = [] + libraries = [] + extra_link_args = [] + if flags is not None: + # shlex.split(None) reads from stdin + for token in shlex.split(flags): + switch = token[0:2] + value = token[2:] + if switch == '-L': + library_dirs.append(value) + elif switch == '-l': + libraries.append(value) + else: + extra_link_args.append(token) + + return library_dirs, libraries, extra_link_args + + class PyBuildExt(build_ext): def __init__(self, dist): @@ -1469,59 +1520,39 @@ def detect_expat_elementtree(self): # # More information on Expat can be found at www.libexpat.org. # - if '--with-system-expat' in sysconfig.get_config_var("CONFIG_ARGS"): - expat_inc = [] - extra_compile_args = [] - expat_lib = ['expat'] - expat_sources = [] - expat_depends = [] - else: - expat_inc = [os.path.join(self.srcdir, 'Modules', 'expat')] - extra_compile_args = [] - # bpo-44394: libexpat uses isnan() of math.h and needs linkage - # against the libm - expat_lib = ['m'] - expat_sources = ['expat/xmlparse.c', - 'expat/xmlrole.c', - 'expat/xmltok.c'] - expat_depends = ['expat/ascii.h', - 'expat/asciitab.h', - 'expat/expat.h', - 'expat/expat_config.h', - 'expat/expat_external.h', - 'expat/internal.h', - 'expat/latin1tab.h', - 'expat/utf8tab.h', - 'expat/xmlrole.h', - 'expat/xmltok.h', - 'expat/xmltok_impl.h' - ] - - cc = sysconfig.get_config_var('CC').split()[0] - ret = run_command( - '"%s" -Werror -Wno-unreachable-code -E -xc /dev/null >/dev/null 2>&1' % cc) - if ret == 0: - extra_compile_args.append('-Wno-unreachable-code') + cflags = parse_cflags(sysconfig.get_config_var("EXPAT_CFLAGS")) + include_dirs, define_macros, undef_macros, extra_compile_args = cflags + # ldflags includes either system libexpat or full path to + # our static libexpat.a. + ldflags = parse_ldflags(sysconfig.get_config_var("EXPAT_LDFLAGS")) + library_dirs, libraries, extra_link_args = ldflags + + expat_depends = [] + libexpat_a = sysconfig.get_config_var("LIBEXPAT_A") + if libexpat_a: + expat_depends.append(libexpat_a) self.add(Extension('pyexpat', + include_dirs=include_dirs, + define_macros=define_macros, + undef_macros=undef_macros, extra_compile_args=extra_compile_args, - include_dirs=expat_inc, - libraries=expat_lib, - sources=['pyexpat.c'] + expat_sources, + library_dirs=library_dirs, + libraries=libraries, + extra_link_args=extra_link_args, + sources=['pyexpat.c'], depends=expat_depends)) # Fredrik Lundh's cElementTree module. Note that this also # uses expat (via the CAPI hook in pyexpat). - - if os.path.isfile(os.path.join(self.srcdir, 'Modules', '_elementtree.c')): - self.add(Extension('_elementtree', - include_dirs=expat_inc, - libraries=expat_lib, - sources=['_elementtree.c'], - depends=['pyexpat.c', *expat_sources, - *expat_depends])) - else: - self.missing.append('_elementtree') + self.add(Extension('_elementtree', + include_dirs=include_dirs, + define_macros=define_macros, + undef_macros=undef_macros, + extra_compile_args=extra_compile_args, + # no EXPAT_LDFLAGS + sources=['_elementtree.c'], + depends=['pyexpat.c', *expat_depends])) def detect_multibytecodecs(self): # Hye-Shik Chang's CJKCodecs modules. @@ -2019,15 +2050,14 @@ def detect_decimal(self): # Stefan Krah's _decimal module sources = ['_decimal/_decimal.c'] depends = ['_decimal/docstrings.h'] - define_macros = [] - - cflags = sysconfig.get_config_var("DECIMAL_CFLAGS") - extra_compile_args = shlex.split(cflags) if cflags else None + + cflags = parse_cflags(sysconfig.get_config_var("DECIMAL_CFLAGS")) + include_dirs, define_macros, undef_macros, extra_compile_args = cflags # ldflags includes either system libmpdec or full path to # our static libmpdec.a. - ldflags = sysconfig.get_config_var("DECIMAL_LDFLAGS") - extra_link_args = shlex.split(ldflags) if ldflags else None - + ldflags = parse_ldflags(sysconfig.get_config_var("DECIMAL_LDFLAGS")) + library_dirs, libraries, extra_link_args = ldflags + libmpdec_a = sysconfig.get_config_var("LIBMPDEC_A") if libmpdec_a: depends.append(libmpdec_a) @@ -2035,8 +2065,12 @@ def detect_decimal(self): # Uncomment for extra functionality: #define_macros.append(('EXTRA_FUNCTIONALITY', 1)) self.add(Extension('_decimal', + include_dirs=include_dirs, define_macros=define_macros, + undef_macros=undef_macros, extra_compile_args=extra_compile_args, + library_dirs=library_dirs, + libraries=libraries, extra_link_args=extra_link_args, sources=sources, depends=depends))