Stop excluding unisolated packages from build dependencies

.pre-commit-config.yaml

@@ -1,3 +1,4 @@
+exclude: (^integration_tests)
 default_language_version:
   python: "3.12"
 repos:

CHANGELOG.md

@@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.30.0] - 2024/10/15
+
+### Changed
+
+- Changed how build dependencies "numpy" and "scipy" is handled.
+  Previously, if a package depends on "numpy" or "scipy", the build system would
+  not install those packages. Instead, it pointed to the pre-built packages that pyodide-build provides.
+  This caused some issues when the package relies on some tools that are not available in the pre-built packages.
+  Now, the build system will install the "numpy" and "scipy" during the build process, but replace some of the
+  files with the pre-built ones to make sure the target platform is compatible to WebAssembly.
+  [#21](https://github.com/pyodide/pyodide-build/pull/21)
+
 ## [0.29.0] - 2024/09/19
 
 ### Added

integration_tests/recipes/libf2c/extras/make.inc

@@ -0,0 +1,80 @@
+# -*- Makefile -*-
+####################################################################
+#  LAPACK make include file.                                       #
+#  LAPACK, Version 3.2.1                                           #
+#  June 2009		                                               #
+####################################################################
+#
+# See the INSTALL/ directory for more examples.
+#
+SHELL = /usr/bin/env sh
+#
+#  The machine (platform) identifier to append to the library names
+#
+# WA for WebAssembly
+PLAT = _WA
+#
+#  Modify the FORTRAN and OPTS definitions to refer to the
+#  compiler and desired compiler options for your machine.  NOOPT
+#  refers to the compiler options desired when NO OPTIMIZATION is
+#  selected.  Define LOADER and LOADOPTS to refer to the loader
+#  and desired load options for your machine.
+#
+#######################################################
+# This is used to compile C library
+#CC        = gcc  # inherit $CC from emmake
+# if no wrapping of the blas library is needed, uncomment next line
+#CC        = gcc -DNO_BLAS_WRAP
+CFLAGS    = -O3 -I$(TOPDIR)/INCLUDE -fPIC -DNO_BLAS_WRAP
+LDFLAGS	  = -O3
+LOADER    = $(CC)
+LOADOPTS  =
+NOOPT     = -O0 -I$(TOPDIR)/INCLUDE -fPIC
+DRVCFLAGS = $(CFLAGS)
+F2CCFLAGS = $(CFLAGS)
+#######################################################################
+
+#
+# Timer for the SECOND and DSECND routines
+#
+# Default : SECOND and DSECND will use a call to the EXTERNAL FUNCTION ETIME
+# TIMER    = EXT_ETIME
+# For RS6K : SECOND and DSECND will use a call to the EXTERNAL FUNCTION ETIME_
+# TIMER    = EXT_ETIME_
+# For gfortran compiler: SECOND and DSECND will use a call to the INTERNAL FUNCTION ETIME
+# TIMER    = INT_ETIME
+# If your Fortran compiler does not provide etime (like Nag Fortran Compiler, etc...)
+# SECOND and DSECND will use a call to the Fortran standard INTERNAL FUNCTION CPU_TIME
+TIMER    = INT_CPU_TIME
+# If neither of this works...you can use the NONE value... In that case, SECOND and DSECND will always return 0
+# TIMER     = NONE
+#
+#  The archiver and the flag(s) to use when building archive (library)
+#  If you system has no ranlib, set RANLIB = echo.
+#
+ARCH     = $(AR)
+ARCHFLAGS= cr
+#RANLIB   = ranlib
+#
+#  The location of BLAS library for linking the testing programs.
+#  The target's machine-specific, optimized BLAS library should be
+#  used whenever possible.
+#
+BLASLIB      = ../../blas$(PLAT).a
+#
+#  Location of the extended-precision BLAS (XBLAS) Fortran library
+#  used for building and testing extended-precision routines.  The
+#  relevant routines will be compiled and XBLAS will be linked only if
+#  USEXBLAS is defined.
+#
+# USEXBLAS    = Yes
+XBLASLIB     =
+# XBLASLIB    = -lxblas
+#
+#  Names of generated libraries.
+#
+LAPACKLIB    = lapack$(PLAT).a
+F2CLIB       = ../../F2CLIBS/libf2c.a
+TMGLIB       = tmglib$(PLAT).a
+EIGSRCLIB    = eigsrc$(PLAT).a
+LINSRCLIB    = linsrc$(PLAT).a

integration_tests/recipes/libf2c/meta.yaml

@@ -0,0 +1,46 @@
+# We still download the full CLAPACK but we are only using the libf2c part of CLAPACK.
+# libf2c part is needed for the f2ced Fortran files in scipy for example to
+# define things like pow_dd, i_len, etc...
+#
+# Note f2clib package only creates f2clib.a, and f2clib.a symbols are added to
+# libopenblas.so in the OpenBLAS meta.yaml.
+package:
+  name: libf2c
+  version: CLAPACK-3.2.1
+  tag:
+    - library
+source:
+  sha256: 6dc4c382164beec8aaed8fd2acc36ad24232c406eda6db462bd4c41d5e455fac
+  url: http://www.netlib.org/clapack/clapack.tgz
+  extract_dir: CLAPACK-3.2.1
+  patches:
+    - patches/0001-fix-arith.h.patch
+    - patches/0002-fix-f2clibs-build.patch
+    - patches/0003-remove-redundant-symbols.patch
+    - patches/0004-correct-return-types.patch
+    - patches/0005-Remove-symbols-defined-in-OpenBLAS.patch
+    # In CLAPACK's F2CLIBS/libf2c Makefile, some commands are mistakenly (?) hardcoded
+    # instead of using the right variables
+    - patches/0006-adjust-ld-ar-ranlib.patch
+    - patches/0007-add-singlecomplex.patch
+
+  extras:
+    - [extras/make.inc, make.inc]
+
+build:
+  type: static_library
+  script: |
+    # The archive's contents have default permission 0750. If we use docker
+    # to build, then we will not own the contents in the host, which means
+    # we cannot navigate into the folder. Setting it to 0750 makes it
+    # easier to debug.
+    chmod -R o+rx .
+
+    ARCH="emar" \
+    emmake make -j ${PYODIDE_JOBS:-3} f2clib
+    mkdir -p ${WASM_LIBRARY_DIR}/{lib,include}
+    cp INCLUDE/f2c.h ${WASM_LIBRARY_DIR}/include
+    cp F2CLIBS/libf2c.a ${WASM_LIBRARY_DIR}/lib
+about:
+  home: https://www.netlib.org/clapack/
+  license: BSD-3-Clause

integration_tests/recipes/libf2c/patches/0001-fix-arith.h.patch

@@ -0,0 +1,30 @@
+From 01990867ee7a641078505efba367a413a97f7802 Mon Sep 17 00:00:00 2001
+From: Michael Droettboom <[email protected]>
+Date: Fri, 18 Mar 2022 19:59:25 -0700
+Subject: [PATCH 1/5] fix arith.h
+
+arith.h is a file generated at build time by compiling and running a C program.
+Since we use emscripten to build throughout, the C program becomes a wasm file
+and we call it differently.
+---
+ F2CLIBS/libf2c/Makefile | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/F2CLIBS/libf2c/Makefile b/F2CLIBS/libf2c/Makefile
+index 0a3ed0d..a473ed8 100644
+--- a/F2CLIBS/libf2c/Makefile
++++ b/F2CLIBS/libf2c/Makefile
+@@ -173,8 +173,8 @@ xwsne.o:	fmt.h
+ arith.h: arithchk.c
+ 	$(CC) $(CFLAGS) -DNO_FPINIT arithchk.c -lm ||\
+ 	 $(CC) -DNO_LONG_LONG $(CFLAGS) -DNO_FPINIT arithchk.c -lm
+-	./a.out >arith.h
+-	rm -f a.out arithchk.o
++	node a.out.js >arith.h
++	rm -f a.out.js a.out.wasm
+
+ check:
+ 	xsum Notice README abort_.c arithchk.c backspac.c c_abs.c c_cos.c \
+-- 
+2.25.1
+

integration_tests/recipes/libf2c/patches/0002-fix-f2clibs-build.patch

@@ -0,0 +1,31 @@
+From d88133066f9f6312145c1186116fdb6446d3f7a5 Mon Sep 17 00:00:00 2001
+From: Michael Droettboom <[email protected]>
+Date: Fri, 18 Mar 2022 20:00:51 -0700
+Subject: [PATCH 2/5] fix f2clibs build
+
+emscripten produces LLVM bitcode here, not genuine object files, so it doesn't
+make sense to strip symbols.
+
+(It would also fail because emcc uses the file extension to determine what kind
+of object to output, and .xxx is not a recognized extension; this is the error
+message you would receive if you try to run the commands)
+---
+ F2CLIBS/libf2c/Makefile | 2 --
+ 1 file changed, 2 deletions(-)
+
+diff --git a/F2CLIBS/libf2c/Makefile b/F2CLIBS/libf2c/Makefile
+index a473ed8..e51d826 100644
+--- a/F2CLIBS/libf2c/Makefile
++++ b/F2CLIBS/libf2c/Makefile
+@@ -19,8 +19,6 @@ include ../../make.inc
+ # compile, then strip unnecessary symbols
+ .c.o:
+ 	$(CC) -c -DSkip_f2c_Undefs $(CFLAGS) $*.c
+-	ld -r -x -o $*.xxx $*.o
+-	mv $*.xxx $*.o
+ ## Under Solaris (and other systems that do not understand ld -x),
+ ## omit -x in the ld line above.
+ ## If your system does not have the ld command, comment out
+-- 
+2.25.1
+

integration_tests/recipes/libf2c/patches/0003-remove-redundant-symbols.patch

@@ -0,0 +1,34 @@
+From 78ff0cec961d9eb4e94193995fe151e1ecdae9df Mon Sep 17 00:00:00 2001
+From: Roman Yurchak <[email protected]>
+Date: Fri, 18 Mar 2022 20:01:39 -0700
+Subject: [PATCH 3/5] remove redundant symbols
+
+Remove a few symbols from LAPACK that are redundantly defined with BLAS or are
+ported in scipy. It wouldn't be an issue if we were linking dynamically, but
+because of static linking otherwise we get errors at link time about symbols
+defined twice.
+
+ - Roman Yurchak (https://github.com/pyodide/pyodide/pull/238)
+---
+ SRC/Makefile | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/SRC/Makefile b/SRC/Makefile
+index 5f1eb22..32e669b 100644
+--- a/SRC/Makefile
++++ b/SRC/Makefile
+@@ -48,9 +48,9 @@ include ../make.inc
+ #
+ #######################################################################
+
+-ALLAUX = maxloc.o ilaenv.o ieeeck.o lsamen.o xerbla.o xerbla_array.o iparmq.o	\
++ALLAUX = maxloc.o ilaenv.o ieeeck.o lsamen.o iparmq.o	\
+     ilaprec.o ilatrans.o ilauplo.o iladiag.o chla_transtype.o \
+-    ../INSTALL/ilaver.o ../INSTALL/lsame.o
++    ../INSTALL/ilaver.o
+
+ ALLXAUX =
+
+-- 
+2.25.1
+

integration_tests/recipes/libf2c/patches/0004-correct-return-types.patch

@@ -0,0 +1,81 @@
+From 572a3e20ba040b4f29bbef97a9db6658c10077d3 Mon Sep 17 00:00:00 2001
+From: Joe Marshall <[email protected]>
+Date: Fri, 18 Mar 2022 20:02:42 -0700
+Subject: [PATCH 4/5] correct return types
+
+Make return types to fortran subroutines consistently be int. Some functions are defined within clapack as variously
+void and int return. Normal C compilers don't care, but emscripten is strict about return values.
+---
+ F2CLIBS/libf2c/ef1asc_.c | 2 +-
+ F2CLIBS/libf2c/f2ch.add  | 4 ++--
+ F2CLIBS/libf2c/s_cat.c   | 6 +++---
+ F2CLIBS/libf2c/s_copy.c  | 4 ++--
+ 4 files changed, 8 insertions(+), 8 deletions(-)
+
+diff --git a/F2CLIBS/libf2c/ef1asc_.c b/F2CLIBS/libf2c/ef1asc_.c
+index 70be0bc..b2a82a2 100644
+--- a/F2CLIBS/libf2c/ef1asc_.c
++++ b/F2CLIBS/libf2c/ef1asc_.c
+@@ -13,7 +13,7 @@ extern "C" {
+ extern VOID s_copy();
+ ef1asc_(a, la, b, lb) ftnint *a, *b; ftnlen *la, *lb;
+ #else
+-extern void s_copy(char*,char*,ftnlen,ftnlen);
++extern int s_copy(char*,char*,ftnlen,ftnlen);
+ int ef1asc_(ftnint *a, ftnlen *la, ftnint *b, ftnlen *lb)
+ #endif
+ {
+diff --git a/F2CLIBS/libf2c/f2ch.add b/F2CLIBS/libf2c/f2ch.add
+index a2acc17..f3f0466 100644
+--- a/F2CLIBS/libf2c/f2ch.add
++++ b/F2CLIBS/libf2c/f2ch.add
+@@ -124,9 +124,9 @@ extern double r_sinh(float *);
+ extern double r_sqrt(float *);
+ extern double r_tan(float *);
+ extern double r_tanh(float *);
+-extern void s_cat(char *, char **, integer *, integer *, ftnlen);
++extern int s_cat(char *, char **, integer *, integer *, ftnlen);
+ extern integer s_cmp(char *, char *, ftnlen, ftnlen);
+-extern void s_copy(char *, char *, ftnlen, ftnlen);
++extern int s_copy(char *, char *, ftnlen, ftnlen);
+ extern int s_paus(char *, ftnlen);
+ extern integer s_rdfe(cilist *);
+ extern integer s_rdue(cilist *);
+diff --git a/F2CLIBS/libf2c/s_cat.c b/F2CLIBS/libf2c/s_cat.c
+index 8d92a63..54c4ff1 100644
+--- a/F2CLIBS/libf2c/s_cat.c
++++ b/F2CLIBS/libf2c/s_cat.c
+@@ -28,11 +28,11 @@ extern
+ extern "C" {
+ #endif
+
+- VOID
++ 
+ #ifdef KR_headers
+-s_cat(lp, rpp, rnp, np, ll) char *lp, *rpp[]; ftnint rnp[], *np; ftnlen ll;
++int s_cat(lp, rpp, rnp, np, ll) char *lp, *rpp[]; ftnint rnp[], *np; ftnlen ll;
+ #else
+-s_cat(char *lp, char *rpp[], ftnint rnp[], ftnint *np, ftnlen ll)
++int s_cat(char *lp, char *rpp[], ftnint rnp[], ftnint *np, ftnlen ll)
+ #endif
+ {
+ 	ftnlen i, nc;
+diff --git a/F2CLIBS/libf2c/s_copy.c b/F2CLIBS/libf2c/s_copy.c
+index 9dacfc7..8d8963f 100644
+--- a/F2CLIBS/libf2c/s_copy.c
++++ b/F2CLIBS/libf2c/s_copy.c
+@@ -12,9 +12,9 @@ extern "C" {
+ /* assign strings:  a = b */
+
+ #ifdef KR_headers
+-VOID s_copy(a, b, la, lb) register char *a, *b; ftnlen la, lb;
++int s_copy(a, b, la, lb) register char *a, *b; ftnlen la, lb;
+ #else
+-void s_copy(register char *a, register char *b, ftnlen la, ftnlen lb)
++int s_copy(register char *a, register char *b, ftnlen la, ftnlen lb)
+ #endif
+ {
+ 	register char *aend, *bend;
+-- 
+2.25.1
+

integration_tests/recipes/libf2c/patches/0005-Remove-symbols-defined-in-OpenBLAS.patch

@@ -0,0 +1,27 @@
+From eaf5c5db6e956036869255cb51831e720474d01d Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= <[email protected]>
+Date: Fri, 7 Apr 2023 15:20:18 +0200
+Subject: [PATCH 5/5] Remove symbols defined in OpenBLAS
+
+---
+ F2CLIBS/libf2c/Makefile | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/F2CLIBS/libf2c/Makefile b/F2CLIBS/libf2c/Makefile
+index 57eff0d..136050f 100644
+--- a/F2CLIBS/libf2c/Makefile
++++ b/F2CLIBS/libf2c/Makefile
+@@ -31,8 +31,8 @@ MISC =	f77vers.o i77vers.o main.o s_rnge.o abort_.o exit_.o getarg_.o iargc_.o\
+ 	getenv_.o signal_.o s_stop.o s_paus.o system_.o cabs.o ctype.o\
+ 	derf_.o derfc_.o erf_.o erfc_.o sig_die.o uninit.o
+ POW =	pow_ci.o pow_dd.o pow_di.o pow_hh.o pow_ii.o pow_ri.o pow_zi.o pow_zz.o
+-CX =	c_abs.o c_cos.o c_div.o c_exp.o c_log.o c_sin.o c_sqrt.o
+-DCX =	z_abs.o z_cos.o z_div.o z_exp.o z_log.o z_sin.o z_sqrt.o
++CX =	c_cos.o c_div.o c_exp.o c_log.o c_sin.o c_sqrt.o
++DCX =	z_cos.o z_div.o z_exp.o z_log.o z_sin.o z_sqrt.o
+ REAL =	r_abs.o r_acos.o r_asin.o r_atan.o r_atn2.o r_cnjg.o r_cos.o\
+ 	r_cosh.o r_dim.o r_exp.o r_imag.o r_int.o\
+ 	r_lg10.o r_log.o r_mod.o r_nint.o r_sign.o\
+-- 
+2.34.1
+

integration_tests/recipes/libf2c/patches/0006-adjust-ld-ar-ranlib.patch

@@ -0,0 +1,33 @@
+Index: CLAPACK-3.2.1/F2CLIBS/libf2c/Makefile
+===================================================================
+--- CLAPACK-3.2.1.orig/F2CLIBS/libf2c/Makefile
++++ CLAPACK-3.2.1/F2CLIBS/libf2c/Makefile
+@@ -70,8 +70,8 @@ OFILES = $(MISC) $(POW) $(CX) $(DCX) $(R
+ all: f2c.h signal1.h sysdep1.h libf2c.a clapack_install
+
+ libf2c.a: $(OFILES)
+-	ar r libf2c.a $?
+-	-ranlib libf2c.a
++	$(ARCH) r libf2c.a $?
++	$(RANLIB) libf2c.a
+
+ ## Shared-library variant: the following rule works on Linux
+ ## systems.  Details are system-dependent.  Under Linux, -fPIC
+@@ -80,7 +80,7 @@ libf2c.a: $(OFILES)
+ ## of "cc -shared".
+
+ libf2c.so: $(OFILES)
+-	cc -shared -o libf2c.so $(OFILES)
++	$(CC) -shared -o libf2c.so $(OFILES)
+
+ ### If your system lacks ranlib, you don't need it; see README.
+
+@@ -117,7 +117,7 @@ sysdep1.h: sysdep1.h0
+
+ install: libf2c.a
+ 	cp libf2c.a $(LIBDIR)
+-	-ranlib $(LIBDIR)/libf2c.a
++	$(RANLIB) $(LIBDIR)/libf2c.a
+
+ clapack_install: libf2c.a
+ 	mv libf2c.a ..