Standard ABI generation code and library refactor.

.gitignore

@@ -288,6 +288,7 @@ ompi/tools/ompi_info/ompi_info
 
 ompi/tools/wrappers/mpic++-wrapper-data.txt
 ompi/tools/wrappers/mpicc-wrapper-data.txt
+ompi/tools/wrappers/mpicc_abi-wrapper-data.txt
 ompi/tools/wrappers/mpifort-wrapper-data.txt
 ompi/tools/wrappers/ompi_wrapper_script
 ompi/tools/wrappers/ompi.pc
@@ -534,3 +535,9 @@ docs/_templates
 # Common Python virtual environment directory names
 venv
 py??
+
+# Generated C Bindings
+ompi/mpi/c/ompi_*.c
+ompi/mpi/c/standard_*.c
+ompi/mpi/c/abi.h
+ompi/mpi/c/standard_abi

config/ompi_config_files.m4

@@ -52,6 +52,7 @@ AC_DEFUN([OMPI_CONFIG_FILES],[
         ompi/tools/ompi_info/Makefile
         ompi/tools/wrappers/Makefile
         ompi/tools/wrappers/mpicc-wrapper-data.txt
+        ompi/tools/wrappers/mpicc_abi-wrapper-data.txt
         ompi/tools/wrappers/mpic++-wrapper-data.txt
         ompi/tools/wrappers/mpifort-wrapper-data.txt
         ompi/tools/wrappers/ompi.pc

config/ompi_configure_options.m4

@@ -244,5 +244,27 @@ else
 fi
 AM_CONDITIONAL(OMPI_OMPIO_SUPPORT, test "$ompi_want_ompio" = "1")
 
+# If the ABI source files don't exist, then we need Python to generate them
+AM_PATH_PYTHON([3.6],,[:])
+abi_file="${srcdir}/ompi/mpi/c/ompi_send.c"
+AS_IF([! test -e "$abi_file" && test "$PYTHON" = ":"],
+      [AC_MSG_ERROR([Open MPI requires Python >=3.6 for generating the C ABI files. Aborting])])
+AM_CONDITIONAL(OMPI_GENERATE_C_INTERFACE_FILES,[test "$PYTHON" != ":"])
+
+AC_MSG_CHECKING([if want to enable standard ABI library])
+AC_ARG_ENABLE([standard-abi],
+    [AS_HELP_STRING([--enable-standard-abi],
+                    [Enable building the standard ABI library (default: disabled)])])
+if test "$enable_standard_abi" = "yes"; then
+    AC_MSG_RESULT([yes])
+    ompi_standard_abi=1
+else
+    AC_MSG_RESULT([no])
+    ompi_standard_abi=0
+fi
+AC_DEFINE_UNQUOTED([OMPI_STANDARD_ABI],[$ompi_standard_abi],
+                   [Whether we want to build the standard ABI library])
+AM_CONDITIONAL(OMPI_STANDARD_ABI,[test "$enable_standard_abi" = "yes"])
+
 ])dnl
 

ompi/Makefile.am

@@ -126,36 +126,70 @@ DIST_SUBDIRS = \
         $(MCA_ompi_FRAMEWORKS_SUBDIRS) \
         $(MCA_ompi_FRAMEWORK_COMPONENT_ALL_SUBDIRS)
 
-# Build the main MPI library
-
+noinst_LTLIBRARIES = libopen-mpi.la
 lib_LTLIBRARIES = lib@[email protected]
-lib@OMPI_LIBMPI_NAME@_la_SOURCES =
-lib@OMPI_LIBMPI_NAME@_la_LIBADD = \
+if OMPI_STANDARD_ABI
+lib_LTLIBRARIES += libmpi_abi.la
+endif
+
+#
+# ABI Refactor
+#
+# As required by the standard ABI, ompi must now provide libmpi_abi.la for
+# ABI-specific code. Since we also want to avoid breaking the existing ompi
+# ABI, the libraries in this directory are now organized as follows:
+#
+# +----------------------+----------------------+
+# |     libmpi_abi.la    |        libmpi.la     |
+# +----------------------+----------------------+
+# |              libopen-mpi.la                 |
+# +----------------------+----------------------+
+#
+# This includes a new library, libopen-mpi.la, that links in all backend code
+# built in this directory or SUBDIRs of this directory. Previously everything
+# was just linked directly into libmpi.la (lib@[email protected]).
+#
+# libmpi_abi.la and libmpi.la both now link in libopen-mpi.la, the only
+# difference between them being that one includes the standard ABI functions
+# and the other the ompi-specific versions of those.
+#
+
+# Build the Open MPI internal library
+libopen_mpi_la_SOURCES =
+libopen_mpi_la_LIBADD = \
         datatype/libdatatype.la \
         debuggers/libdebuggers.la \
+        $(OMPI_TOP_BUILDDIR)/opal/lib@[email protected] \
+        $(MCA_ompi_FRAMEWORK_LIBS) \
+        $(OMPI_LIBMPI_EXTRA_LIBS)
+libopen_mpi_la_DEPENDENCIES = $(libopen_mpi_la_LIBADD)
+
+# Build the main MPI library
+lib@OMPI_LIBMPI_NAME@_la_SOURCES =
+lib@OMPI_LIBMPI_NAME@_la_LIBADD = \
+	libopen-mpi.la \
         mpi/c/libmpi_c.la \
         mpi/tool/libmpi_mpit.la \
         $(c_mpi_lib) \
         $(c_pmpi_lib) \
-        $(mpi_fortran_base_lib) \
-        $(MCA_ompi_FRAMEWORK_LIBS) \
         $(OMPI_MPIEXT_C_LIBS) \
-        $(OMPI_LIBMPI_EXTRA_LIBS)
+        $(mpi_fortran_base_lib)
 
-
-lib@OMPI_LIBMPI_NAME@_la_LIBADD += \
-        $(OMPI_TOP_BUILDDIR)/opal/lib@[email protected]
-lib@OMPI_LIBMPI_NAME@_la_DEPENDENCIES = $(lib@OMPI_LIBMPI_NAME@_la_LIBADD)
 lib@OMPI_LIBMPI_NAME@_la_LDFLAGS = \
        -version-info $(libmpi_so_version) \
         $(OMPI_LIBMPI_EXTRA_LDFLAGS)
 
+# The MPI Standard ABI library
+libmpi_abi_la_SOURCES =
+libmpi_abi_la_LIBADD = \
+        libopen-mpi.la \
+	mpi/c/libmpi_c_abi.la
+
 # included subdirectory Makefile.am's and appended-to variables
 headers =
-noinst_LTLIBRARIES =
 include_HEADERS =
 dist_ompidata_DATA =
-lib@OMPI_LIBMPI_NAME@_la_SOURCES += $(headers)
+libopen_mpi_la_SOURCES += $(headers)
 
 # Conditionally install the header files
 

ompi/attribute/Makefile.am

@@ -22,6 +22,6 @@
 headers += \
         attribute/attribute.h
 
-lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
+libopen_mpi_la_SOURCES += \
         attribute/attribute.c \
         attribute/attribute_predefined.c

ompi/class/Makefile.am

@@ -23,6 +23,5 @@
 headers += \
         class/ompi_seq_tracker.h
 
-lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
+libopen_mpi_la_SOURCES += \
         class/ompi_seq_tracker.c
-

ompi/communicator/Makefile.am

@@ -28,14 +28,14 @@ headers += \
         communicator/communicator.h \
 	communicator/comm_request.h
 
-lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
+libopen_mpi_la_SOURCES += \
         communicator/comm_init.c \
 	communicator/comm.c \
 	communicator/comm_cid.c \
 	communicator/comm_request.c
 
 if WANT_FT_MPI
-lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
+libopen_mpi_la_SOURCES += \
 	communicator/ft/comm_ft.c communicator/ft/comm_ft_reliable_bcast.c communicator/ft/comm_ft_propagator.c communicator/ft/comm_ft_detector.c communicator/ft/comm_ft_revoke.c
 endif # WANT_FT_MPI
 

ompi/dpm/Makefile.am

@@ -16,6 +16,6 @@ dist_ompidata_DATA += dpm/help-dpm.txt
 headers += \
         dpm/dpm.h
 
-lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
+libopen_mpi_la_SOURCES += \
         dpm/dpm.c
 

ompi/errhandler/Makefile.am

@@ -29,7 +29,7 @@ headers += \
         errhandler/errhandler.h \
         errhandler/errhandler_predefined.h
 
-lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
+libopen_mpi_la_SOURCES += \
         errhandler/errhandler.c \
         errhandler/errhandler_invoke.c \
         errhandler/errhandler_predefined.c \

ompi/file/Makefile.am

@@ -22,5 +22,5 @@
 headers += \
         file/file.h
 
-lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
+libopen_mpi_la_SOURCES += \
         file/file.c

ompi/group/Makefile.am

@@ -25,7 +25,7 @@ headers += \
         group/group.h \
 	group/group_dbg.h
 
-lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
+libopen_mpi_la_SOURCES += \
         group/group.c \
         group/group_init.c \
         group/group_set_rank.c \

ompi/include/mpi.h.in

@@ -1388,7 +1388,10 @@ OMPI_DECLSPEC extern struct ompi_predefined_datatype_t ompi_mpi_ub;
 /*
  * MPI API
  */
-
+#ifndef OMPI_NO_MPI_PROTOTYPES
+OMPI_DECLSPEC  int MPI_Abi_supported(int *flag);
+OMPI_DECLSPEC  int MPI_Abi_version(int *abi_major, int *abi_minor);
+OMPI_DECLSPEC  int MPI_Abi_details(int *buflen, char *details, MPI_Info *info);
 OMPI_DECLSPEC  int MPI_Abort(MPI_Comm comm, int errorcode);
 OMPI_DECLSPEC  int MPI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
                                   int target_rank, MPI_Aint target_disp, int target_count,
@@ -1418,6 +1421,8 @@ OMPI_DECLSPEC  int MPI_Alloc_mem(MPI_Aint size, MPI_Info info,
                                  void *baseptr);
 OMPI_DECLSPEC  int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
                                  MPI_Op op, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Allreduce_c(const void *sendbuf, void *recvbuf, MPI_Count count, MPI_Datatype datatype,
+                                   MPI_Op op, MPI_Comm comm);
 OMPI_DECLSPEC  int MPI_Iallreduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
                                   MPI_Op op, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Allreduce_init(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
@@ -1773,10 +1778,14 @@ OMPI_DECLSPEC  int MPI_Iprobe(int source, int tag, MPI_Comm comm, int *flag,
                               MPI_Status *status);
 OMPI_DECLSPEC  int MPI_Irecv(void *buf, int count, MPI_Datatype datatype, int source,
                              int tag, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Irecv_c(void *buf, MPI_Count count, MPI_Datatype datatype, int source,
+                               int tag, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Irsend(const void *buf, int count, MPI_Datatype datatype, int dest,
                               int tag, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Isend(const void *buf, int count, MPI_Datatype datatype, int dest,
                              int tag, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Isend_c(const void *buf, MPI_Count count, MPI_Datatype datatype, int dest,
+                               int tag, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Isendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                  int dest, int sendtag, void *recvbuf, int recvcount,
                                  MPI_Datatype recvtype, int source, int recvtag,
@@ -1881,6 +1890,8 @@ OMPI_DECLSPEC  int MPI_Recv_init(void *buf, int count, MPI_Datatype datatype, in
                                  int tag, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int source,
                             int tag, MPI_Comm comm, MPI_Status *status);
+OMPI_DECLSPEC  int MPI_Recv_c(void *buf, MPI_Count count, MPI_Datatype datatype, int source,
+                              int tag, MPI_Comm comm, MPI_Status *status);
 OMPI_DECLSPEC  int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
                               MPI_Op op, int root, MPI_Comm comm);
 OMPI_DECLSPEC  int MPI_Ireduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
@@ -1956,6 +1967,8 @@ OMPI_DECLSPEC  int MPI_Send_init(const void *buf, int count, MPI_Datatype dataty
                                  MPI_Request *request);
 OMPI_DECLSPEC  int MPI_Send(const void *buf, int count, MPI_Datatype datatype, int dest,
                             int tag, MPI_Comm comm);
+OMPI_DECLSPEC  int MPI_Send_c(const void *buf, MPI_Count count, MPI_Datatype datatype, int dest,
+                              int tag, MPI_Comm comm);
 OMPI_DECLSPEC  int MPI_Sendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                 int dest, int sendtag, void *recvbuf, int recvcount,
                                 MPI_Datatype recvtype, int source, int recvtag,
@@ -2152,6 +2165,9 @@ OMPI_DECLSPEC  int PMPI_Abort(MPI_Comm comm, int errorcode);
 OMPI_DECLSPEC  int PMPI_Accumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
                                    int target_rank, MPI_Aint target_disp, int target_count,
                                    MPI_Datatype target_datatype, MPI_Op op, MPI_Win win);
+OMPI_DECLSPEC  int PMPI_Accumulate_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype,
+                                     int target_rank, MPI_Aint target_disp, MPI_Count target_count,
+                                     MPI_Datatype target_datatype, MPI_Op op, MPI_Win win);
 OMPI_DECLSPEC  int PMPI_Add_error_class(int *errorclass);
 OMPI_DECLSPEC  int PMPI_Add_error_code(int errorclass, int *errorcode);
 OMPI_DECLSPEC  int PMPI_Add_error_string(int errorcode, const char *string);
@@ -2177,6 +2193,8 @@ OMPI_DECLSPEC  int PMPI_Alloc_mem(MPI_Aint size, MPI_Info info,
                                   void *baseptr);
 OMPI_DECLSPEC  int PMPI_Allreduce(const void *sendbuf, void *recvbuf, int count,
                                   MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Allreduce_c(const void *sendbuf, void *recvbuf, MPI_Count count,
+                                    MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
 OMPI_DECLSPEC  int PMPI_Iallreduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
                                    MPI_Op op, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Allreduce_init(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
@@ -2534,10 +2552,14 @@ OMPI_DECLSPEC  int PMPI_Iprobe(int source, int tag, MPI_Comm comm, int *flag,
                                MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_Irecv(void *buf, int count, MPI_Datatype datatype, int source,
                               int tag, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Irecv_c(void *buf, MPI_Count count, MPI_Datatype datatype, int source,
+                                int tag, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Irsend(const void *buf, int count, MPI_Datatype datatype, int dest,
                                int tag, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Isend(const void *buf, int count, MPI_Datatype datatype, int dest,
                               int tag, MPI_Comm comm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Isend_c(const void *buf, MPI_Count count, MPI_Datatype datatype, int dest,
+                                int tag, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Isendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                   int dest, int sendtag, void *recvbuf, int recvcount,
                                   MPI_Datatype recvtype, int source, int recvtag,
@@ -2642,6 +2664,8 @@ OMPI_DECLSPEC  int PMPI_Recv_init(void *buf, int count, MPI_Datatype datatype, i
                                   int tag, MPI_Comm comm, MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Recv(void *buf, int count, MPI_Datatype datatype, int source,
                              int tag, MPI_Comm comm, MPI_Status *status);
+OMPI_DECLSPEC  int PMPI_Recv_c(void *buf, MPI_Count count, MPI_Datatype datatype, int source,
+                               int tag, MPI_Comm comm, MPI_Status *status);
 OMPI_DECLSPEC  int PMPI_Reduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
                                MPI_Op op, int root, MPI_Comm comm);
 OMPI_DECLSPEC  int PMPI_Ireduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
@@ -2717,6 +2741,8 @@ OMPI_DECLSPEC  int PMPI_Send_init(const void *buf, int count, MPI_Datatype datat
                                   MPI_Request *request);
 OMPI_DECLSPEC  int PMPI_Send(const void *buf, int count, MPI_Datatype datatype, int dest,
                              int tag, MPI_Comm comm);
+OMPI_DECLSPEC  int PMPI_Send_c(const void *buf, MPI_Count count, MPI_Datatype datatype, int dest,
+                               int tag, MPI_Comm comm);
 OMPI_DECLSPEC  int PMPI_Sendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                                  int dest, int sendtag, void *recvbuf, int recvcount,
                                  MPI_Datatype recvtype, int source, int recvtag,
@@ -3175,6 +3201,8 @@ OMPI_DECLSPEC  int PMPI_Type_ub(MPI_Datatype mtype, MPI_Aint *ub)
 #define MPI_Type_ub(...)  THIS_FUNCTION_WAS_REMOVED_IN_MPI30(MPI_Type_ub, MPI_Type_get_extent)
 #endif
 
+#endif /* OMPI_NO_MPI_PROTOTYPES */
+
 #if defined(c_plusplus) || defined(__cplusplus)
 }
 #endif

ompi/info/Makefile.am

@@ -23,5 +23,5 @@
 headers += \
         info/info.h
 
-lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
+libopen_mpi_la_SOURCES += \
         info/info.c

ompi/instance/Makefile.am

@@ -23,4 +23,4 @@
 
 headers += instance/instance.h
 
-lib@OMPI_LIBMPI_NAME@_la_SOURCES += instance/instance.c
+libopen_mpi_la_SOURCES += instance/instance.c

ompi/interlib/Makefile.am

@@ -25,5 +25,5 @@
 headers += \
         interlib/interlib.h
 
-lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
+libopen_mpi_la_SOURCES += \
         interlib/interlib.c

ompi/mca/bml/r2/Makefile.am

@@ -36,7 +36,7 @@ mcacomponentdir = $(ompilibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_bml_r2_la_SOURCES = $(r2_sources)
 mca_bml_r2_la_LDFLAGS = -module -avoid-version
-mca_bml_r2_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la
+mca_bml_r2_la_LIBADD = $(top_builddir)/ompi/libopen-mpi.la
 
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_bml_r2_la_SOURCES = $(r2_sources)

ompi/mca/coll/basic/Makefile.am

@@ -64,7 +64,7 @@ mcacomponentdir = $(ompilibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_coll_basic_la_SOURCES = $(sources)
 mca_coll_basic_la_LDFLAGS = -module -avoid-version
-mca_coll_basic_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la
+mca_coll_basic_la_LIBADD = $(top_builddir)/ompi/libopen-mpi.la
 
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_coll_basic_la_SOURCES =$(sources)

ompi/mca/coll/cuda/Makefile.am

@@ -32,7 +32,7 @@ mcacomponentdir = $(ompilibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_coll_cuda_la_SOURCES = $(sources)
 mca_coll_cuda_la_LDFLAGS = -module -avoid-version
-mca_coll_cuda_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la
+mca_coll_cuda_la_LIBADD = $(top_builddir)/ompi/libopen-mpi.la
 
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_coll_cuda_la_SOURCES =$(sources)

ompi/mca/coll/demo/Makefile.am

@@ -57,7 +57,7 @@ mcacomponentdir = $(ompilibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_coll_demo_la_SOURCES = $(sources)
 mca_coll_demo_la_LDFLAGS = -module -avoid-version
-mca_coll_demo_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la
+mca_coll_demo_la_LIBADD = $(top_builddir)/ompi/libopen-mpi.la
 
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_coll_demo_la_SOURCES = $(sources)

ompi/mca/coll/hcoll/Makefile.am

@@ -39,7 +39,7 @@ endif
 mcacomponentdir = $(ompilibdir)
 mcacomponent_LTLIBRARIES  = $(component_install)
 mca_coll_hcoll_la_SOURCES = $(coll_hcoll_sources)
-mca_coll_hcoll_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
+mca_coll_hcoll_la_LIBADD = $(top_builddir)/ompi/libopen-mpi.la \
 	$(coll_hcoll_LIBS)
 mca_coll_hcoll_la_LDFLAGS = -module -avoid-version $(coll_hcoll_LDFLAGS)
 

ompi/mca/coll/inter/Makefile.am

@@ -33,7 +33,7 @@ mcacomponentdir = $(ompilibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_coll_inter_la_SOURCES = $(sources)
 mca_coll_inter_la_LDFLAGS = -module -avoid-version
-mca_coll_inter_la_LIBADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la
+mca_coll_inter_la_LIBADD = $(top_builddir)/ompi/libopen-mpi.la
 
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_coll_inter_la_SOURCES = $(sources)