Merge pull request #83 from SLongshaw/master

Completed Fortran wrapper and CMake build system

Author: SLongshaw

CMakeLists.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.0)
-project(MUI VERSION 1.1.3 DESCRIPTION "Multiscale Universal Interface" LANGUAGES CXX C Fortran)
+project(MUI VERSION 1.2 DESCRIPTION "Multiscale Universal Interface" LANGUAGES CXX C Fortran)
 
 set(CMAKE_CXX_STANDARD 11)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -9,19 +9,19 @@ set(CMAKE_CXX_EXTENSIONS OFF)
 option(USE_RBF "Include RBF support" OFF)
 # Wrapper options
 option(C_WRAPPER "Compile C wrapper" OFF)
-option(FORTRAN_WRAPPER "Compile Fortran wrapper" OFF) #ToDo: Add Fortran wrapper compilation
+option(FORTRAN_WRAPPER "Compile Fortran wrapper" OFF)
 option(PYTHON_WRAPPER "Compile Python wrapper" OFF)
 option(PIP_INSTALL "Install compiled Python wrapper using pip" OFF)
 
 include(CMakePackageConfigHelpers)
 include(CMake/sourcelist.cmake)
 
 find_package(MPI REQUIRED)
-if (MPI_FOUND)
+if(MPI_FOUND)
 	include_directories(SYSTEM ${MPI_INCLUDE_PATH})
-else (MPI_FOUND)
+elseif(NOT MPI_FOUND)
 	message(SEND_ERROR "MPI not found")
-endif (MPI_FOUND)
+endif(MPI_FOUND)
 
 if(USE_RBF)
 	find_package(Eigen3 REQUIRED)
@@ -47,7 +47,9 @@ target_compile_definitions(MUI INTERFACE LIBRARY_HEADER_ONLY)
 install(TARGETS MUI EXPORT muiTargets INCLUDES DESTINATION include LIBRARY DESTINATION lib)
 install(DIRECTORY ${CMAKE_SOURCE_DIR}/
 	DESTINATION ${CMAKE_INSTALL_PREFIX}/MUI-${PROJECT_VERSION}/include
-	FILES_MATCHING PATTERN "*.h"
+	FILES_MATCHING
+	PATTERN "*.h"
+	PATTERN "*.f90"
 	PATTERN ".git" EXCLUDE
 	PATTERN "build" EXCLUDE
 	PATTERN "wrappers/Python" EXCLUDE
@@ -83,6 +85,29 @@ if(C_WRAPPER)
 endif(C_WRAPPER)
 
 if(FORTRAN_WRAPPER)
+	message("-- MUI Fortran wrapper: Selected")
+
+	add_library(MUI_Fortran_wrapper SHARED
+		./wrappers/Fortran/mui_f_wrapper_general.cpp
+		./wrappers/Fortran/mui_f_wrapper_1d.cpp
+		./wrappers/Fortran/mui_f_wrapper_2d.cpp
+		./wrappers/Fortran/mui_f_wrapper_3d.cpp)
+	set_target_properties(MUI_Fortran_wrapper PROPERTIES PUBLIC_HEADER ./wrappers/Fortran/mui_f_wrapper_general.f90)
+	set_target_properties(MUI_Fortran_wrapper PROPERTIES PUBLIC_HEADER ./wrappers/Fortran/mui_f_wrapper_1d.f90)
+	set_target_properties(MUI_Fortran_wrapper PROPERTIES PUBLIC_HEADER ./wrappers/Fortran/mui_f_wrapper_2d.f90)
+	set_target_properties(MUI_Fortran_wrapper PROPERTIES PUBLIC_HEADER ./wrappers/Fortran/mui_f_wrapper_3d.f90)
+	set_target_properties(MUI_Fortran_wrapper PROPERTIES PUBLIC_HEADER ./wrappers/Fortran/config_f_wrapper.h)
+	target_include_directories(MUI_Fortran_wrapper PRIVATE .)
+
+	if(USE_RBF)
+    		target_link_libraries(MUI_Fortran_wrapper Eigen3::Eigen ${MPI_CXX_LIBRARIES})
+	else()
+    		target_link_libraries(MUI_Fortran_wrapper ${MPI_CXX_LIBRARIES})
+	endif(USE_RBF)
+
+	install(TARGETS MUI_Fortran_wrapper
+    		LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/MUI-${PROJECT_VERSION}/lib/
+    		PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_PREFIX}/MUI-${PROJECT_VERSION}/include/wrappers/Fortran/)
 endif(FORTRAN_WRAPPER)
 
 if(PYTHON_WRAPPER)
@@ -97,10 +122,10 @@ if(PYTHON_WRAPPER)
 	endif()
 
 	if(USE_RBF)
-		add_custom_target(mui4py ALL COMMAND $(MAKE) COMPILER=${COMPILER_TYPE} package USE_RBF=1 INC_EIGEN="${EIGEN3_INCLUDE_DIR}"
+		add_custom_target(mui4py ALL COMMAND $(MAKE) COMPILER=${COMPILER_TYPE} CXX=${MPI_CXX_COMPILER} USE_RBF=1 INC_EIGEN="${EIGEN3_INCLUDE_DIR}" package
 			WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/wrappers/Python)
 	else()
-		add_custom_target(mui4py ALL COMMAND $(MAKE) COMPILER=${COMPILER_TYPE} package
+		add_custom_target(mui4py ALL COMMAND $(MAKE) COMPILER=${COMPILER_TYPE} CXX=${MPI_CXX_COMPILER} package
                         WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/wrappers/Python)
 	endif(USE_RBF)
 
@@ -113,6 +138,8 @@ if(PYTHON_WRAPPER)
 				${CMAKE_CURRENT_SOURCE_DIR}/wrappers/Python/mui4py/mui4py_mod.so)
 
 	if(PIP_INSTALL)
+		message("-- MUI Python wrapper installation: Selected")
+
 		add_custom_target(mui4py_pip ALL COMMAND pip3 install dist/mui4py*.tar.gz
                         WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/wrappers/Python)
 		add_dependencies(mui4py_pip mui4py)

README.md

@@ -1,9 +1,11 @@
 # MUI - Multiscale Universal Interface
-Concurrently coupled numerical simulations using heterogeneous solvers are powerful tools for modeling multiscale phenomena. However, major modifications to existing codes are often required to enable such simulations, posing significant difficulties in practice. Here we present Multiscale Universal Interface (MUI), which is capable of facilitating the coupling effort for a wide range of multiscale simulations. The library adopts a header-only form with minimal external dependency and hence can be easily dropped into existing codes. A data samplerconcept is introduced, combined with a hybrid dynamic/static typing mechanism, to create an easily customizable framework for solver-independent data interpretation. 
+Concurrently coupled numerical simulations using heterogeneous solvers are powerful tools for modeling both multiscale and multiphysics phenomena. However, major modifications to existing codes are often required to enable such simulations, posing significant difficulties in practice. Here we present the Multiscale Universal Interface (MUI), which is capable of facilitating the coupling effort for a wide range of simulation types. 
+
+The library adopts a header-only form with minimal external dependency and hence can be easily dropped into existing codes. A data sampler concept is introduced, combined with a hybrid dynamic/static typing mechanism, to create an easily customizable framework for solver-independent data interpretation. 
 
 The library integrates MPI MPMD support and an asynchronous communication protocol to handle inter-solver information exchange irrespective of the solvers’ own MPI awareness. Template metaprogramming is heavily employed to simultaneously improve runtime performance and code flexibility. 
 
-In the publication referenced below, the library is validated by solving three different multiscale problems, which also serve to demonstrate the flexibility of the framework in handling heterogeneous models and solvers associated with multiphysics problems. In the first example, a Couette flow was simulated using two concurrently coupled Smoothed Particle Hydrodynamics (SPH) simulations of different spatial resolutions. In the second example, we coupled the deterministic SPH method with the stochastic Dissipative Particle Dynamics (DPD) method to study the effect of surface grafting on the hydrodynamics properties on the surface. In the third example, we consider conjugate heat transfer between a solid domain and a fluid domain by coupling the particle-based energy-conserving DPD (eDPD) method with the Finite Element Method (FEM).
+In the publication referenced below, the library is validated by solving three different multiscale type problems, which also serve to demonstrate the flexibility of the framework in handling heterogeneous models and solvers associated with multiphysics problems. In the first example, a Couette flow was simulated using two concurrently coupled Smoothed Particle Hydrodynamics (SPH) simulations of different spatial resolutions. In the second example, we coupled the deterministic SPH method with the stochastic Dissipative Particle Dynamics (DPD) method to study the effect of surface grafting on the hydrodynamics properties on the surface. In the third example, we consider conjugate heat transfer between a solid domain and a fluid domain by coupling the particle-based energy-conserving DPD (eDPD) method with the Finite Element Method (FEM).
 
 ## Licensing
 
@@ -29,6 +31,8 @@ As a header-only library using MUI in your own source code is straight forward,
 
 <sup>1</sup> Only true if the Radial Basis Function (RBF) spatial interpolation scheme is included at compile-time.
 
+<b>Note:</b> if you are using MUI with a toolchain other than GNU, you may need to manually specify the endianness you wish the library to assume, this is done using a number of compile-time parameters: -DMUI_IGNORE_ENDIAN (never reorders); -DMUI_INT_LITTLE_ENDIAN combined with -DMUI_FLOAT_LITTLE_ENDIAN (assumed little endian reordering); -DMUI_INT_BIG_ENDIAN combined with -DMUI_FLOAT_BIG_ENDIAN (assumes big endian reordering). It is recommended you start by allowing the library to try and determine your endianness options, however if you receive many compilation erorrs relating to the stream parts of the library then you will need to manually define your choices.
+
 ## Publication
 
 **Tang** Y.-H., **Kudo**, S., **Bian**, X., **Li**, Z., & **Karniadakis**, G. E. Multiscale Universal Interface: A Concurrent Framework for Coupling Heterogeneous Solvers, *Journal of Computational Physics*, **2015**, 297.15, 13-31.

bin.h

@@ -250,7 +250,7 @@ struct bin_t {
 		}
 
 		size_t zero_count=0;
-		REAL vol = abs(max[0]-min[0]);
+		REAL vol = std::abs(max[0]-min[0]);
 		if(almost_equal(vol, static_cast<REAL>(0))) { // check if first dimension is zero size, if so set to 1
 			vol = 1.0;
 			zero_count++;
@@ -307,7 +307,6 @@ struct bin_t {
 		if( initialize_query_(bx,lda,lh) ) return map;
 		map.reserve(lda[D-1]*12);
 		set_map_<D-1>::apply( 0, lda, lh, displs, map );
-		map.shrink_to_fit();
 		return map;
 	}
 
@@ -322,8 +321,7 @@ struct bin_t {
 	REAL domain_size() {
 		REAL dim_size = norm(max-min);
 		// Special case if domain only contains a single point
-		if(almost_equal(dim_size, static_cast<REAL>(0)))
-			dim_size = 1.0;
+		if(dim_size == 0) dim_size = 1.0;
 		return dim_size;
 	}
 

comm_mpi_smart.h

@@ -65,30 +65,36 @@ class comm_mpi_smart : public comm_mpi {
 public:
 	comm_mpi_smart( const char URI[], MPI_Comm world = MPI_COMM_WORLD ) : comm_mpi(URI, world) {}
 	virtual ~comm_mpi_smart() {
-		// Call MPI_Test in blocking loop on any remaining MPI_Isend messages in buffer and if complete, pop before destruction
+		// Call blocking MPI_Test on any remaining MPI_Isend messages in buffer and if complete, pop before destruction or warn
 		test_completion_blocking();
 	}
 
 private:
 	void send_impl_( message msg, const std::vector<bool> &is_sending ) {
-		// Call non-blocking MPI_Test on MPI_Isend messages in buffer and if complete, pop
+		// Call non-blocking MPI_Test on outstanding MPI_Isend messages in buffer and if complete, pop
 		test_completion();
 		auto bytes = std::make_shared<std::vector<char> >(msg.detach());
-		for( int i = 0 ; i < remote_size_ ; i++ ){
-			if( is_sending[i] ){
-				if(bytes->size() > INT_MAX){
-					std::cerr << "MUI Error [comm_mpi_smart.h]: Trying to send more data than is possible with MPI_Isend." << std::endl
-							  << "This is likely because there is too much data per MPI rank." << std::endl
-							  << "The program will now abort. Try increasing the number of MPI ranks." << std::endl;
-					std::abort();
-				}
+
+		if(bytes->size() > INT_MAX){
+      std::cerr << "MUI Error [comm_mpi_smart.h]: Trying to send more data than is possible with MPI_Isend." << std::endl
+            << "This is likely because there is too much data per MPI rank." << std::endl
+            << "The program will now abort. Try increasing the number of MPI ranks." << std::endl;
+      std::abort();
+    }
+
+		int test;
+
+		for( int i = 0; i < remote_size_; i++ ) {
+			if( is_sending[i] ) {
 				send_buf.emplace_back(MPI_Request(), bytes);
-				MPI_Isend(bytes->data(), bytes->size(), MPI_BYTE, i, 0, 
+				MPI_Isend(bytes->data(), bytes->size(), MPI_BYTE, i, 0,
 				          domain_remote_, &(send_buf.back().first));
+
+				// Add immediate MPI_Test to ensure buffer cleared as quickly as possible
+        MPI_Test(&(send_buf.back().first),&test,MPI_STATUS_IGNORE);
+        if( test ) send_buf.pop_back();
 		 	}
 		}
-		// Call non-blocking MPI_Test on MPI_Isend messages in buffer and if complete, pop
-		test_completion();
 	}
 
 	message recv_impl_() {
@@ -118,21 +124,11 @@ class comm_mpi_smart : public comm_mpi {
 	/** \brief Time-limited blocking check for complete MPI_Isend calls
 	 */
 	void test_completion_blocking() {
-		int timeout = 0;
-		auto start = std::chrono::system_clock::now();
-		while (send_buf.size() > 0) {
+	  while (send_buf.size() > 0) {
 			for( auto itr=send_buf.begin(), end=send_buf.end(); itr != end; ){
-				int test = false;
-				MPI_Test(&(itr->first),&test,MPI_STATUS_IGNORE);
-				if( test ) itr = send_buf.erase(itr);
-				else ++itr;
-			}
-			if( (std::chrono::system_clock::now() - start) > std::chrono::seconds(5) ) {
-				timeout++;
-				int time_left = 60 - (timeout*5);
-				if( time_left == 0) break;
-				std::cout << "MUI Warning [comm_mpi_smart.h]: MPI_Isend calls spent over 5 seconds completing, all operations will timeout in "
-						  << time_left << "s" << std::endl;
+				//int test = false;
+				MPI_Wait(&(itr->first), MPI_STATUS_IGNORE);
+				itr = send_buf.erase(itr);
 			}
 		}
 	}

doc/.gitignore

@@ -0,0 +1 @@
+documentation/

doc/Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = "Multiscale Universal Interface"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 
+PROJECT_NUMBER         = 1.2
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -150,7 +150,7 @@ INLINE_INHERITED_MEMB  = NO
 # shortest path that makes the file name unique will be used
 # The default value is: YES.
 
-FULL_PATH_NAMES        = YES
+FULL_PATH_NAMES        = NO
 
 # The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path.
 # Stripping is only done if one of the specified strings matches the left-hand
@@ -790,7 +790,7 @@ WARN_LOGFILE           =
 # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
 # Note: If this tag is empty the current directory is searched.
 
-INPUT                  = 
+INPUT                  = ../
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
@@ -873,8 +873,10 @@ RECURSIVE              = YES
 # Note that relative paths are relative to the directory from which doxygen is
 # run.
 
-EXCLUDE                = /wrappers \
-                         /documentation
+EXCLUDE                = ../build \
+                         ../doc \
+                         ../CMake \
+                         ../wrappers
 
 # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
 # directories that are symbolic links (a Unix file system feature) are excluded

doc/resources/MUI_logo.png

@@ -189,21 +189,6 @@ SPECIALIZE(2fx,float,int64_t,2);
 SPECIALIZE(3fx,float,int64_t,3);
 #undef SPECIALIZE
 #endif
-// sample configuration class
-// modify to adapt to specific simulation scenario
-/******************************************************************************
-struct config_custom {
-	static const int D = 3;
-	using REAL = double;
-	using INT  = int;
-	using point_type = point<REAL,D>;
-	using time_type  = REAL;
-	static const bool DEBUG = false;
-	using data_types = type_list<int,double,float>;
-	using EXCEPTION = exception_segv;
-    static const bool FIXEDPOINTS = false;
-};
-******************************************************************************/
 
 // usage: SPECIALIZE( _your_custom_suffix, your_custom_config_structure )
 //        uniface_your_custom_suffix interface; ...

mui.h

@@ -75,7 +75,7 @@ class sampler_exact {
 
 	template<template<typename,typename> class CONTAINER>
 	inline OTYPE filter( point_type focus, const CONTAINER<ITYPE,CONFIG> &data_points ) const {
-		for( size_t i = 0 ; i < data_points.size() ; i++ ) {
+	  for( size_t i = 0 ; i < data_points.size() ; i++ ) {
 			if ( norm( focus - data_points[i].first ) < point_tolerance ) {
 				return data_points[i].second;
 			}

spatial_samplers/sampler_exact.h

@@ -67,12 +67,14 @@ class sampler_gauss {
 
 	template<template<typename,typename> class CONTAINER>
 	inline OTYPE filter( point_type focus, const CONTAINER<ITYPE,CONFIG> &data_points ) const {
-		REAL  wsum = 0;
+		REAL exp_val = -0.5/h;
+		REAL r2 = r*r;
+	  REAL  wsum = 0;
 		OTYPE vsum = 0;
 		for( size_t i = 0 ; i < data_points.size() ; i++ ) {
-			auto d = normsq( focus - data_points[i].first );
-			if ( d < r*r ) {
-			    REAL w = nh * std::exp( (REAL(-0.5)/h) * d );
+			REAL d = normsq( focus - data_points[i].first );
+			if ( d < r2 ) {
+			  REAL w = nh * std::exp( exp_val * d );
 				vsum += data_points[i].second * w;
 				wsum += w;
 			}