Skip to content

Commit

Permalink
bpo-45800: Move pyexpat build setup into configure (GH-29547)
Browse files Browse the repository at this point in the history 
Settings for :mod:`pyexpat` C extension are now detected by ``configure``.
The bundled ``expat`` library is built in ``Makefile``.

Signed-off-by: Christian Heimes <[email protected]>
  • Loading branch information
@tiran
tiran authored Nov 14, 2021
1 parent 25835c5 commit 464e661
Show file tree
Hide file tree
Showing 6 changed files with 177 additions and 58 deletions.
54 changes: 51 additions & 3 deletions Makefile.pre.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -537,6 +537,33 @@ LIBMPDEC_HEADERS= \

LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a

##########################################################################
# pyexpat's expat library

LIBEXPAT_OBJS= \
Modules/expat/xmlparse.o \
Modules/expat/xmlrole.o \
Modules/expat/xmltok.o

LIBEXPAT_HEADERS= \
Modules/expat/ascii.h \
Modules/expat/asciitab.h \
Modules/expat/expat.h \
Modules/expat/expat_config.h \
Modules/expat/expat_external.h \
Modules/expat/iasciitab.h \
Modules/expat/internal.h \
Modules/expat/latin1tab.h \
Modules/expat/nametab.h \
Modules/expat/pyexpatns.h \
Modules/expat/siphash.h \
Modules/expat/utf8tab.h \
Modules/expat/xmlrole.h \
Modules/expat/xmltok.h \
Modules/expat/xmltok_impl.h

LIBEXPAT_A= Modules/expat/libexpat.a

#########################################################################
# Rules

Expand Down Expand Up @@ -688,7 +715,7 @@ $(srcdir)/Modules/_blake2/blake2s_impl.c: $(srcdir)/Modules/_blake2/blake2b_impl
# -s, --silent or --quiet is always the first char.
# Under BSD make, MAKEFLAGS might be " -s -v x=y".
# Ignore macros passed by GNU make, passed after --
sharedmods: $(BUILDPYTHON) pybuilddir.txt @LIBMPDEC_INTERNAL@
sharedmods: $(BUILDPYTHON) pybuilddir.txt @LIBMPDEC_INTERNAL@ @LIBEXPAT_INTERNAL@
@case "`echo X $$MAKEFLAGS | sed 's/^X //;s/ -- .*//'`" in \
*\ -s*|s*) quiet="-q";; \
*) quiet="";; \
Expand Down Expand Up @@ -828,6 +855,27 @@ $(LIBMPDEC_A): $(LIBMPDEC_OBJS)
-rm -f $@
$(AR) $(ARFLAGS) $@ $(LIBMPDEC_OBJS)

##########################################################################
# Build static libexpat.a
LIBEXPAT_CFLAGS=$(PY_STDMODULE_CFLAGS) $(CCSHARED) @LIBEXPAT_CFLAGS@

# for setup.py
EXPAT_CFLAGS=@LIBEXPAT_CFLAGS@
EXPAT_LDFLAGS=@LIBEXPAT_LDFLAGS@

Modules/expat/xmlparse.o: $(srcdir)/Modules/expat/xmlparse.c $(LIBEXPAT_HEADERS) $(PYTHON_HEADERS)
$(CC) -c $(LIBEXPAT_CFLAGS) -o $@ $(srcdir)/Modules/expat/xmlparse.c

Modules/expat/xmlrole.o: $(srcdir)/Modules/expat/xmlrole.c $(LIBEXPAT_HEADERS) $(PYTHON_HEADERS)
$(CC) -c $(LIBEXPAT_CFLAGS) -o $@ $(srcdir)/Modules/expat/xmlrole.c

Modules/expat/xmltok.o: $(srcdir)/Modules/expat/xmltok.c $(LIBEXPAT_HEADERS) $(PYTHON_HEADERS)
$(CC) -c $(LIBEXPAT_CFLAGS) -o $@ $(srcdir)/Modules/expat/xmltok.c

$(LIBEXPAT_A): $(LIBEXPAT_OBJS)
-rm -f $@
$(AR) $(ARFLAGS) $@ $(LIBEXPAT_OBJS)

# create relative links from build/lib.platform/egg.so to Modules/egg.so
# pybuilddir.txt is created too late. We cannot use it in Makefile
# targets. ln --relative is not portable.
Expand Down Expand Up @@ -2407,12 +2455,12 @@ Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h

MODULE_CMATH_DEPS=$(srcdir)/Modules/_math.h
MODULE_MATH_DEPS=$(srcdir)/Modules/_math.h
MODULE_PYEXPAT_DEPS=$(srcdir)/Modules/expat/ascii.h $(srcdir)/Modules/expat/asciitab.h $(srcdir)/Modules/expat/expat.h $(srcdir)/Modules/expat/expat_config.h $(srcdir)/Modules/expat/expat_external.h $(srcdir)/Modules/expat/internal.h $(srcdir)/Modules/expat/latin1tab.h $(srcdir)/Modules/expat/utf8tab.h $(srcdir)/Modules/expat/xmlrole.h $(srcdir)/Modules/expat/xmltok.h $(srcdir)/Modules/expat/xmltok_impl.h
MODULE_PYEXPAT_DEPS=$(LIBEXPAT_HEADERS) @LIBEXPAT_INTERNAL@
MODULE_UNICODEDATA_DEPS=$(srcdir)/Modules/unicodedata_db.h $(srcdir)/Modules/unicodename_db.h
MODULE__BLAKE2_DEPS=$(srcdir)/Modules/_blake2/impl/blake2-config.h $(srcdir)/Modules/_blake2/impl/blake2-dispatch.c $(srcdir)/Modules/_blake2/impl/blake2-impl.h $(srcdir)/Modules/_blake2/impl/blake2-kat.h $(srcdir)/Modules/_blake2/impl/blake2.h $(srcdir)/Modules/_blake2/impl/blake2b-load-sse2.h $(srcdir)/Modules/_blake2/impl/blake2b-load-sse41.h $(srcdir)/Modules/_blake2/impl/blake2b-ref.c $(srcdir)/Modules/_blake2/impl/blake2b-round.h $(srcdir)/Modules/_blake2/impl/blake2b-test.c $(srcdir)/Modules/_blake2/impl/blake2b.c $(srcdir)/Modules/_blake2/impl/blake2bp-test.c $(srcdir)/Modules/_blake2/impl/blake2bp.c $(srcdir)/Modules/_blake2/impl/blake2s-load-sse2.h $(srcdir)/Modules/_blake2/impl/blake2s-load-sse41.h $(srcdir)/Modules/_blake2/impl/blake2s-load-xop.h $(srcdir)/Modules/_blake2/impl/blake2s-ref.c $(srcdir)/Modules/_blake2/impl/blake2s-round.h $(srcdir)/Modules/_blake2/impl/blake2s-test.c $(srcdir)/Modules/_blake2/impl/blake2s.c $(srcdir)/Modules/_blake2/impl/blake2sp-test.c $(srcdir)/Modules/_blake2/impl/blake2sp.c $(srcdir)/Modules/hashlib.h
MODULE__CTYPES_DEPS=$(srcdir)/Modules/_ctypes/ctypes.h
MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h $(LIBMPDEC_HEADERS) @LIBMPDEC_INTERNAL@
MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/expat/ascii.h $(srcdir)/Modules/expat/asciitab.h $(srcdir)/Modules/expat/expat.h $(srcdir)/Modules/expat/expat_config.h $(srcdir)/Modules/expat/expat_external.h $(srcdir)/Modules/expat/internal.h $(srcdir)/Modules/expat/latin1tab.h $(srcdir)/Modules/expat/utf8tab.h $(srcdir)/Modules/expat/xmlparse.c $(srcdir)/Modules/expat/xmlrole.c $(srcdir)/Modules/expat/xmlrole.h $(srcdir)/Modules/expat/xmltok.c $(srcdir)/Modules/expat/xmltok.h $(srcdir)/Modules/expat/xmltok_impl.h $(srcdir)/Modules/pyexpat.c
MODULE__ELEMENTTREE_DEPS=$(LIBEXPAT_HEADERS) @LIBEXPAT_INTERNAL@
MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h
MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h
MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h
Expand Down
2 changes: 2 additions & 0 deletions Misc/NEWS.d/next/Build/2021-11-13-16-40-05.bpo-45800.5Hz6nr.rst
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Settings for :mod:`pyexpat` C extension are now detected by ``configure``.
The bundled ``expat`` library is built in ``Makefile``.
4 changes: 2 additions & 2 deletions Modules/Setup
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,8 +172,8 @@ time timemodule.c
#select selectmodule.c

# XML
#_elementtree -I$(srcdir)/Modules/expat _elementtree.c
#pyexpat -I$(srcdir)/Modules/expat expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c
#_elementtree _elementtree.c $(EXPAT_CFLAGS)
#pyexpat pyexpat.c $(EXPAT_CFLAGS) $(EXPAT_LDFLAGS)

# hashing builtins
#_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c
Expand Down
21 changes: 21 additions & 0 deletions configure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -664,6 +664,9 @@ LIBMPDEC_INTERNAL
LIBMPDEC_LDFLAGS
LIBMPDEC_CFLAGS
LIBFFI_INCLUDEDIR
LIBEXPAT_INTERNAL
LIBEXPAT_LDFLAGS
LIBEXPAT_CFLAGS
TZPATH
SHLIBS
CFLAGSFORSHARED
Expand Down Expand Up @@ -10715,6 +10718,24 @@ fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_system_expat" >&5
$as_echo "$with_system_expat" >&6; }

if test "x$with_system_expat" = xyes; then :

LIBEXPAT_CFLAGS=""
LIBEXPAT_LDFLAGS="-lexpat"
LIBEXPAT_INTERNAL=

else

LIBEXPAT_CFLAGS="-I\$(srcdir)/Modules/expat"
LIBEXPAT_LDFLAGS="-lm \$(LIBEXPAT_A)"
LIBEXPAT_INTERNAL="\$(LIBEXPAT_A)"

fi





# Check for use of the system libffi library
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-system-ffi" >&5
$as_echo_n "checking for --with-system-ffi... " >&6; }
Expand Down
14 changes: 14 additions & 0 deletions configure.ac
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3017,6 +3017,20 @@ AC_ARG_WITH(system_expat,

AC_MSG_RESULT($with_system_expat)

AS_VAR_IF([with_system_expat], [yes], [
LIBEXPAT_CFLAGS=""
LIBEXPAT_LDFLAGS="-lexpat"
LIBEXPAT_INTERNAL=
], [
LIBEXPAT_CFLAGS="-I\$(srcdir)/Modules/expat"
LIBEXPAT_LDFLAGS="-lm \$(LIBEXPAT_A)"
LIBEXPAT_INTERNAL="\$(LIBEXPAT_A)"
])

AC_SUBST([LIBEXPAT_CFLAGS])
AC_SUBST([LIBEXPAT_LDFLAGS])
AC_SUBST([LIBEXPAT_INTERNAL])

# Check for use of the system libffi library
AC_MSG_CHECKING(for --with-system-ffi)
AC_ARG_WITH(system_ffi,
Expand Down
140 changes: 87 additions & 53 deletions setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,57 @@ def find_module_file(module, dirlist):
return os.path.join(dirs[0], module)


def parse_cflags(flags):
"""Parse a string with compiler flags (-I, -D, -U, extra)
Distutils appends extra args to the compiler arguments. Some flags like
-I must appear earlier. Otherwise the pre-processor picks up files
from system inclue directories.
"""
include_dirs = []
define_macros = []
undef_macros = []
extra_compile_args = []
if flags is not None:
# shlex.split(None) reads from stdin
for token in shlex.split(flags):
switch = token[0:2]
value = token[2:]
if switch == '-I':
include_dirs.append(value)
elif switch == '-D':
key, _, val = value.partition("=")
if not val:
val = None
define_macros.append((key, val))
elif switch == '-U':
undef_macros.append(value)
else:
extra_compile_args.append(token)

return include_dirs, define_macros, undef_macros, extra_compile_args


def parse_ldflags(flags):
"""Parse a string with linker flags (-L, -l, extra)"""
library_dirs = []
libraries = []
extra_link_args = []
if flags is not None:
# shlex.split(None) reads from stdin
for token in shlex.split(flags):
switch = token[0:2]
value = token[2:]
if switch == '-L':
library_dirs.append(value)
elif switch == '-l':
libraries.append(value)
else:
extra_link_args.append(token)

return library_dirs, libraries, extra_link_args


class PyBuildExt(build_ext):

def __init__(self, dist):
Expand Down Expand Up @@ -1469,59 +1520,39 @@ def detect_expat_elementtree(self):
#
# More information on Expat can be found at www.libexpat.org.
#
if '--with-system-expat' in sysconfig.get_config_var("CONFIG_ARGS"):
expat_inc = []
extra_compile_args = []
expat_lib = ['expat']
expat_sources = []
expat_depends = []
else:
expat_inc = [os.path.join(self.srcdir, 'Modules', 'expat')]
extra_compile_args = []
# bpo-44394: libexpat uses isnan() of math.h and needs linkage
# against the libm
expat_lib = ['m']
expat_sources = ['expat/xmlparse.c',
'expat/xmlrole.c',
'expat/xmltok.c']
expat_depends = ['expat/ascii.h',
'expat/asciitab.h',
'expat/expat.h',
'expat/expat_config.h',
'expat/expat_external.h',
'expat/internal.h',
'expat/latin1tab.h',
'expat/utf8tab.h',
'expat/xmlrole.h',
'expat/xmltok.h',
'expat/xmltok_impl.h'
]

cc = sysconfig.get_config_var('CC').split()[0]
ret = run_command(
'"%s" -Werror -Wno-unreachable-code -E -xc /dev/null >/dev/null 2>&1' % cc)
if ret == 0:
extra_compile_args.append('-Wno-unreachable-code')
cflags = parse_cflags(sysconfig.get_config_var("EXPAT_CFLAGS"))
include_dirs, define_macros, undef_macros, extra_compile_args = cflags
# ldflags includes either system libexpat or full path to
# our static libexpat.a.
ldflags = parse_ldflags(sysconfig.get_config_var("EXPAT_LDFLAGS"))
library_dirs, libraries, extra_link_args = ldflags

expat_depends = []
libexpat_a = sysconfig.get_config_var("LIBEXPAT_A")
if libexpat_a:
expat_depends.append(libexpat_a)

self.add(Extension('pyexpat',
include_dirs=include_dirs,
define_macros=define_macros,
undef_macros=undef_macros,
extra_compile_args=extra_compile_args,
include_dirs=expat_inc,
libraries=expat_lib,
sources=['pyexpat.c'] + expat_sources,
library_dirs=library_dirs,
libraries=libraries,
extra_link_args=extra_link_args,
sources=['pyexpat.c'],
depends=expat_depends))

# Fredrik Lundh's cElementTree module. Note that this also
# uses expat (via the CAPI hook in pyexpat).

if os.path.isfile(os.path.join(self.srcdir, 'Modules', '_elementtree.c')):
self.add(Extension('_elementtree',
include_dirs=expat_inc,
libraries=expat_lib,
sources=['_elementtree.c'],
depends=['pyexpat.c', *expat_sources,
*expat_depends]))
else:
self.missing.append('_elementtree')
self.add(Extension('_elementtree',
include_dirs=include_dirs,
define_macros=define_macros,
undef_macros=undef_macros,
extra_compile_args=extra_compile_args,
# no EXPAT_LDFLAGS
sources=['_elementtree.c'],
depends=['pyexpat.c', *expat_depends]))

def detect_multibytecodecs(self):
# Hye-Shik Chang's CJKCodecs modules.
Expand Down Expand Up @@ -2019,24 +2050,27 @@ def detect_decimal(self):
# Stefan Krah's _decimal module
sources = ['_decimal/_decimal.c']
depends = ['_decimal/docstrings.h']
define_macros = []

cflags = sysconfig.get_config_var("DECIMAL_CFLAGS")
extra_compile_args = shlex.split(cflags) if cflags else None

cflags = parse_cflags(sysconfig.get_config_var("DECIMAL_CFLAGS"))
include_dirs, define_macros, undef_macros, extra_compile_args = cflags
# ldflags includes either system libmpdec or full path to
# our static libmpdec.a.
ldflags = sysconfig.get_config_var("DECIMAL_LDFLAGS")
extra_link_args = shlex.split(ldflags) if ldflags else None
ldflags = parse_ldflags(sysconfig.get_config_var("DECIMAL_LDFLAGS"))
library_dirs, libraries, extra_link_args = ldflags

libmpdec_a = sysconfig.get_config_var("LIBMPDEC_A")
if libmpdec_a:
depends.append(libmpdec_a)

# Uncomment for extra functionality:
#define_macros.append(('EXTRA_FUNCTIONALITY', 1))
self.add(Extension('_decimal',
include_dirs=include_dirs,
define_macros=define_macros,
undef_macros=undef_macros,
extra_compile_args=extra_compile_args,
library_dirs=library_dirs,
libraries=libraries,
extra_link_args=extra_link_args,
sources=sources,
depends=depends))
Expand Down

0 comments on commit 464e661

Please sign in to comment.