#----------------------------------*-sh-*--------------------------------------
# =========                 |
# \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
#  \\    /   O peration     | Website:  https://openfoam.org
#   \\  /    A nd           | Copyright (C) 2011-2025 OpenFOAM Foundation
#    \\/     M anipulation  |
#------------------------------------------------------------------------------
# License
#     This file is part of OpenFOAM.
#
#     OpenFOAM is free software: you can redistribute it and/or modify it
#     under the terms of the GNU General Public License as published by
#     the Free Software Foundation, either version 3 of the License, or
#     (at your option) any later version.
#
#     OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
#     ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
#     FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
#     for more details.
#
#     You should have received a copy of the GNU General Public License
#     along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
#
# File
#     etc/config.sh/mpi
#
# Description
#     Startup file for communications library (MPI) for OpenFOAM
#     Sourced from OpenFOAM-<VERSION>/etc/bashrc
#
#------------------------------------------------------------------------------

unset MPI_ARCH_PATH

case "$WM_MPLIB" in
SYSTEMOPENMPI)
    # Use the system installed openmpi, get library directory via mpicc
    export FOAM_MPI=openmpi-system

    # Undefine OPAL_PREFIX if set to one of the paths on foamOldDirs
    if [ -z "$($foamClean "$OPAL_PREFIX" "$foamOldDirs")" ]
    then
        unset OPAL_PREFIX
    fi

    # Get mpicc to report the link flags and extract the library directory
    libDir=`mpicc --showme:link | sed -e 's/.*\(-L\|-Wl,-rpath=\|-Wl,-rpath,\|-Wl,-rpath -Wl,\)\([^ ]*\).*/\2/'`

    # Bit of a hack: strip off 'lib' and hope this is the path to openmpi
    # include files and libraries.
    export MPI_ARCH_PATH="${libDir%/*}"

    _foamAddLib     $libDir
    unset libDir
    ;;

OPENMPI)
    # Look for the source directory
    if [ -z "$OPENMPI_VERSION" ]
    then
        openmpiSrcDir=$(_foamMostRecentDir "$WM_THIRD_PARTY_DIR"/openmpi-*)
        export OPENMPI_VERSION=${openmpiSrcDir##*openmpi-}
        unset openmpiSrcDir
    fi

    # Warn if OpenMPI was not found
    if [ ! -d "$WM_THIRD_PARTY_DIR"/openmpi-$OPENMPI_VERSION ]
    then
        [ -z "$OPENMPI_VERSION" ] && ver="?" || ver=$OPENMPI_VERSION

        echo 1>&2
        echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2
        echo "    OpenMPI sources directory" \
             " $WM_THIRD_PARTY_DIR/openmpi-$ver not found" 1>&2
        echo 1>&2

        unset ver
    fi

    export FOAM_MPI=openmpi-$OPENMPI_VERSION

    # Optional configuration tweaks
    _foamSource `$WM_PROJECT_DIR/bin/foamEtcFile config.sh/openmpi`

    export MPI_ARCH_PATH=$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI

    # Tell OpenMPI where to find its install directory
    export OPAL_PREFIX=$MPI_ARCH_PATH

    _foamAddPath    $MPI_ARCH_PATH/bin

    # 64-bit on OpenSuSE 12.1 uses lib64 others use lib
    _foamAddLib     $MPI_ARCH_PATH/lib$WM_COMPILER_LIB_ARCH
    _foamAddLib     $MPI_ARCH_PATH/lib

    _foamAddMan     $MPI_ARCH_PATH/share/man
    ;;

SYSTEMMPI)
    export FOAM_MPI=mpi-system

    if [ -z "$MPI_ROOT" ]
    then
        echo 1>&2
        echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2
        echo "    Please set the environment variable MPI_ROOT to point to" \
             " the base folder for the system MPI in use." 1>&2
        echo "    Example:" 1>&2
        echo 1>&2
        echo "        export MPI_ROOT=/opt/mpi" 1>&2
        echo 1>&2
    else
        export MPI_ARCH_PATH=$MPI_ROOT

        if [ -z "$MPI_ARCH_FLAGS" ]
        then
            echo 1>&2
            echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2
            echo "    MPI_ARCH_FLAGS is not set. Example:" 1>&2
            echo 1>&2
            echo "        export MPI_ARCH_FLAGS=\"-DOMPI_SKIP_MPICXX\"" 1>&2
            echo 1>&2
        fi

        if [ -z "$MPI_ARCH_INC" ]
        then
            echo 1>&2
            echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2
            echo "    MPI_ARCH_INC is not set. Example:" 1>&2
            echo 1>&2
            echo "        export MPI_ARCH_INC=\"-isystem \$MPI_ROOT/include\"" 1>&2
            echo 1>&2
        fi

        if [ -z "$MPI_ARCH_LIBS" ]
        then
            echo 1>&2
            echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2
            echo "    MPI_ARCH_LIBS is not set. Example:" 1>&2
            echo 1>&2
            echo "        export MPI_ARCH_LIBS=\"-L\$MPI_ROOT/lib -lmpi\"" 1>&2
            echo 1>&2
        fi
    fi
    ;;

MPICH)
    # Look for the source directory
    if [ -z "$MPICH_VERSION" ]
    then
        mpichSrcDir=$(_foamMostRecentDir "$WM_THIRD_PARTY_DIR"/mpich-*)
        export MPICH_VERSION=${mpichSrcDir##*mpich-}
        unset mpichSrcDir
    fi

    # Warn if MPICH was not found
    if [ ! -d "$WM_THIRD_PARTY_DIR"/mpich-$MPICH_VERSION ]
    then
        [ -z "$MPICH_VERSION" ] && ver="?" || ver=$MPICH_VERSION

        echo 1>&2
        echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2
        echo "    MPICH sources directory" \
             " $WM_THIRD_PARTY_DIR/mpich-$ver not found" 1>&2
        echo 1>&2

        unset ver
    fi

    export FOAM_MPI=mpich-$MPICH_VERSION

    export MPI_ARCH_PATH=$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI

    _foamAddPath    $MPI_ARCH_PATH/bin

    # 64-bit on OpenSuSE 12.1 uses lib64 others use lib
    _foamAddLib     $MPI_ARCH_PATH/lib$WM_COMPILER_LIB_ARCH
    _foamAddLib     $MPI_ARCH_PATH/lib

    _foamAddMan     $MPI_ARCH_PATH/share/man
    ;;

MPICH-GM)
    export FOAM_MPI=mpich-gm
    export MPI_ARCH_PATH=/opt/mpi
    export MPICH_PATH=$MPI_ARCH_PATH
    export GM_LIB_PATH=/opt/gm/lib64

    _foamAddPath    $MPI_ARCH_PATH/bin

    # 64-bit on OpenSuSE 12.1 uses lib64 others use lib
    _foamAddLib     $MPI_ARCH_PATH/lib$WM_COMPILER_LIB_ARCH
    _foamAddLib     $MPI_ARCH_PATH/lib

    _foamAddLib     $GM_LIB_PATH
    ;;

MV2MPI)
    export FOAM_MPI=mvapich2
    libDir=`mpicc -show -cc= | sed -e 's/.*-L\([^ ]*\).*/\1/'`
    export MPI_ARCH_PATH="${libDir%/*}"
    _foamAddLib $libDir
    unset libDir
    ;;

HPMPI)
    export FOAM_MPI=hpmpi
    export MPI_ARCH_PATH=/opt/hpmpi

    _foamAddPath $MPI_ARCH_PATH/bin

    case `uname -m` in
    i686)
        _foamAddLib $MPI_ARCH_PATH/lib/linux_ia32
        ;;

    x86_64)
        _foamAddLib $MPI_ARCH_PATH/lib/linux_amd64
        ;;
    *)
        echo Unknown processor type `uname -m` 1>&2
        ;;
    esac
    ;;

MPI)
    export FOAM_MPI=mpi
    export MPI_ARCH_PATH=/opt/mpi
    ;;

FJMPI)
    export FOAM_MPI=fjmpi
    export MPI_ARCH_PATH=/opt/FJSVmpi2

    _foamAddPath    $MPI_ARCH_PATH/bin
    _foamAddLib     $MPI_ARCH_PATH/lib/sparcv9
    _foamAddLib     /opt/FSUNf90/lib/sparcv9
    _foamAddLib     /opt/FJSVpnidt/lib
    ;;

QSMPI)
    export FOAM_MPI=qsmpi
    export MPI_ARCH_PATH=/usr/lib/mpi

    _foamAddPath    $MPI_ARCH_PATH/bin
    _foamAddLib     $MPI_ARCH_PATH/lib
    ;;

SGIMPI)
    # No trailing slash
    [ "${MPI_ROOT%/}" = "${MPI_ROOT}" ] || MPI_ROOT="${MPI_ROOT%/}"

    export FOAM_MPI="${MPI_ROOT##*/}"
    export MPI_ARCH_PATH=$MPI_ROOT

    if [ ! -d "$MPI_ROOT" -o -z "$MPI_ARCH_PATH" ]
    then
        echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2
        echo "    MPI_ROOT not a valid mpt installation directory or ending" \
             " in a '/'." 1>&2
        echo "    Please set MPI_ROOT to the mpt installation directory." 1>&2
        echo "    MPI_ROOT currently set to '$MPI_ROOT'" 1>&2
    fi

    if [ "$FOAM_VERBOSE" -a "$PS1" ]
    then
        echo "Using SGI MPT:" 1>&2
        echo "    MPI_ROOT : $MPI_ROOT" 1>&2
        echo "    FOAM_MPI : $FOAM_MPI" 1>&2
    fi

    _foamAddPath    $MPI_ARCH_PATH/bin
    _foamAddLib     $MPI_ARCH_PATH/lib
    ;;

INTELMPI)
    # No trailing slash
    [ "${MPI_ROOT%/}" = "${MPI_ROOT}" ] || MPI_ROOT="${MPI_ROOT%/}"

    export FOAM_MPI="${MPI_ROOT##*/}"
    export MPI_ARCH_PATH=$MPI_ROOT

    if [ ! -d "$MPI_ROOT" -o -z "$MPI_ARCH_PATH" ]
    then
        echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2
        echo "    MPI_ROOT not a valid mpt installation directory or ending" \
             " in a '/'." 1>&2
        echo "    Please set MPI_ROOT to the mpt installation directory." 1>&2
        echo "    MPI_ROOT currently set to '$MPI_ROOT'" 1>&2
    fi

    if [ "$FOAM_VERBOSE" -a "$PS1" ]
    then
        echo "Using INTEL MPI:" 1>&2
        echo "    MPI_ROOT : $MPI_ROOT" 1>&2
        echo "    FOAM_MPI : $FOAM_MPI" 1>&2
    fi

    _foamAddPath    $MPI_ARCH_PATH/bin64
    _foamAddLib     $MPI_ARCH_PATH/lib/release
    ;;
*)
    export FOAM_MPI=dummy
    ;;
esac

# Add (non-dummy) MPI implementation
# Dummy MPI already added to LD_LIBRARY_PATH and has no external libraries
if [ "$FOAM_MPI" != dummy ]
then
    _foamAddLib $FOAM_LIBBIN/$FOAM_MPI:$FOAM_EXT_LIBBIN/$FOAM_MPI
fi


# Set the minimum MPI buffer size (used by all platforms except SGI MPI)
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
: ${minBufferSize:=20000000}

if [ "${MPI_BUFFER_SIZE:=$minBufferSize}" -lt $minBufferSize ]
then
    MPI_BUFFER_SIZE=$minBufferSize
fi
export MPI_BUFFER_SIZE


# Cleanup environment:
# ~~~~~~~~~~~~~~~~~~~~
unset minBufferSize

#------------------------------------------------------------------------------