From 15d32baeee49224a4cf8979f06cf35d728b401f7 Mon Sep 17 00:00:00 2001
From: Alexey Matveichev <github@matveichev.com>
Date: Fri, 11 Nov 2016 08:53:28 +0100
Subject: [PATCH 4/5] System-wide Scotch and METIS

---
 src/parallel/decompose/metisDecomp/Allwmake        | 11 ++-
 src/parallel/decompose/metisDecomp/metisDecomp.C   | 85 ++++++++++++++--------
 src/parallel/decompose/ptscotchDecomp/Make/options | 21 ++++--
 .../decompose/ptscotchDecomp/ptscotchDecomp.C      | 49 ++++++++++---
 src/parallel/decompose/scotchDecomp/Make/options   | 17 ++++-
 src/parallel/decompose/scotchDecomp/scotchDecomp.C | 42 ++++++++---
 6 files changed, 163 insertions(+), 62 deletions(-)

diff --git a/src/parallel/decompose/metisDecomp/Allwmake b/src/parallel/decompose/metisDecomp/Allwmake
index 2f666ab..9408c3e 100755
--- a/src/parallel/decompose/metisDecomp/Allwmake
+++ b/src/parallel/decompose/metisDecomp/Allwmake
@@ -8,8 +8,15 @@ cd ${0%/*} || exit 1    # Run from this directory
 if settings=`$WM_PROJECT_DIR/bin/foamEtcFile config.sh/metis`
 then
     . $settings
-    echo "using METIS_ARCH_PATH=$METIS_ARCH_PATH"
-    if [ -r $METIS_ARCH_PATH/lib/libmetis.so ]
+    # Fall-back to system-wide installed METIS
+    if [ -d "$METIS_ARCH_PATH" ]
+    then
+        echo "using METIS_ARCH_PATH=$METIS_ARCH_PATH"
+    else
+        echo "using system-wide installed METIS"
+        METIS_ARCH_PATH=/usr
+    fi
+    if [ -r $METIS_ARCH_PATH/lib*/libmetis.so ]
     then
         wmake $targetType
     fi
diff --git a/src/parallel/decompose/metisDecomp/metisDecomp.C b/src/parallel/decompose/metisDecomp/metisDecomp.C
index 3ede45d..98d4629 100644
--- a/src/parallel/decompose/metisDecomp/metisDecomp.C
+++ b/src/parallel/decompose/metisDecomp/metisDecomp.C
@@ -54,26 +54,43 @@ Foam::label Foam::metisDecomp::decompose
     List<label>& finalDecomp
 )
 {
+    if (sizeof(idx_t) < sizeof(label))
+    {
+        WarningInFunction
+            << "Size of idx_t is less than size of label, possible data loss."
+            << endl;
+    }
+    // METIS-typed lists
+    List<idx_t> tadjncy(adjncy.size(), Zero);
+    List<idx_t> txadj(xadj.size(), Zero);
+    forAll(adjncy, i)
+    {
+        tadjncy[i] = static_cast<idx_t>(adjncy[i]);
+    }
+    forAll(xadj, i)
+    {
+        txadj[i] = static_cast<idx_t>(xadj[i]);
+    }
     // Method of decomposition
     // recursive: multi-level recursive bisection (default)
     // k-way: multi-level k-way
     word method("recursive");
 
-    label numCells = xadj.size()-1;
+    idx_t numCells = static_cast<idx_t>(xadj.size() - 1);
 
     // Decomposition options
-    List<label> options(METIS_NOPTIONS);
+    List<idx_t> options(METIS_NOPTIONS);
     METIS_SetDefaultOptions(options.begin());
 
     // Processor weights initialised with no size, only used if specified in
-    // a file
-    Field<scalar> processorWeights;
+    // a file. real_t is used to please system-wide installed METIS.
+    Field<real_t> processorWeights;
 
     // Cell weights (so on the vertices of the dual)
-    List<label> cellWeights;
+    List<idx_t> cellWeights;
 
     // Face weights (so on the edges of the dual)
-    List<label> faceWeights;
+    List<idx_t> faceWeights;
 
 
     // Check for externally provided cellweights and if so initialise weights
@@ -157,55 +174,61 @@ Foam::label Foam::metisDecomp::decompose
         }
     }
 
-    label ncon = 1;
-    label nProcs = nProcessors_;
+    idx_t ncon = 1;
+    idx_t nProcs = static_cast<idx_t>(nProcessors_);
 
     // Output: cell -> processor addressing
-    finalDecomp.setSize(numCells);
+    finalDecomp.setSize(static_cast<label>(numCells));
+    List<idx_t> tfinalDecomp(static_cast<label>(numCells), 0);
 
     // Output: number of cut edges
-    label edgeCut = 0;
+    idx_t edgeCut = 0;
 
     if (method == "recursive")
     {
         METIS_PartGraphRecursive
         (
-            &numCells,          // num vertices in graph
-            &ncon,              // num balancing constraints
-            const_cast<List<label>&>(xadj).begin(),   // indexing into adjncy
-            const_cast<List<label>&>(adjncy).begin(), // neighbour info
-            cellWeights.begin(),// vertexweights
-            nullptr,               // vsize: total communication vol
-            faceWeights.begin(),// edgeweights
-            &nProcs,            // nParts
+            &numCells,                  // num vertices in graph
+            &ncon,                      // num balancing constraints
+            txadj.begin(),              // indexing into adjncy
+            tadjncy.begin(),            // neighbour info
+            cellWeights.begin(),        // vertexweights
+            nullptr,                    // vsize: total communication vol
+            faceWeights.begin(),        // edgeweights
+            &nProcs,                    // nParts
             processorWeights.begin(),   // tpwgts
-            nullptr,               // ubvec: processor imbalance (default)
+            nullptr,                    // ubvec: processor imbalance (default)
             options.begin(),
             &edgeCut,
-            finalDecomp.begin()
+            tfinalDecomp.begin()
         );
     }
     else
     {
         METIS_PartGraphKway
         (
-            &numCells,          // num vertices in graph
-            &ncon,              // num balancing constraints
-            const_cast<List<label>&>(xadj).begin(),   // indexing into adjncy
-            const_cast<List<label>&>(adjncy).begin(), // neighbour info
-            cellWeights.begin(),// vertexweights
-            nullptr,               // vsize: total communication vol
-            faceWeights.begin(),// edgeweights
-            &nProcs,            // nParts
+            &numCells,                  // num vertices in graph
+            &ncon,                      // num balancing constraints
+            txadj.begin(),              // indexing into adjncy
+            tadjncy.begin(),            // neighbour info
+            cellWeights.begin(),        // vertexweights
+            nullptr,                    // vsize: total communication vol
+            faceWeights.begin(),        // edgeweights
+            &nProcs,                    // nParts
             processorWeights.begin(),   // tpwgts
-            nullptr,               // ubvec: processor imbalance (default)
+            nullptr,                    // ubvec: processor imbalance (default)
             options.begin(),
             &edgeCut,
-            finalDecomp.begin()
+            tfinalDecomp.begin()
         );
     }
 
-    return edgeCut;
+    forAll(tfinalDecomp, i)
+    {
+        finalDecomp[i] = static_cast<label>(tfinalDecomp[i]);
+    }
+
+    return static_cast<label>(edgeCut);
 }
 
 
diff --git a/src/parallel/decompose/ptscotchDecomp/Make/options b/src/parallel/decompose/ptscotchDecomp/Make/options
index cb407ec..dc93165 100644
--- a/src/parallel/decompose/ptscotchDecomp/Make/options
+++ b/src/parallel/decompose/ptscotchDecomp/Make/options
@@ -2,16 +2,27 @@ sinclude $(GENERAL_RULES)/mplib$(WM_MPLIB)
 sinclude $(RULES)/mplib$(WM_MPLIB)
 
 EXE_INC = \
+    -Wno-old-style-cast \
     $(PFLAGS) $(PINC) \
-    -I$(SCOTCH_ARCH_PATH)/include/$(FOAM_MPI) \
-    -I$(SCOTCH_ARCH_PATH)/include \
-    -I/usr/include/scotch \
     -I../decompositionMethods/lnInclude
 
 LIB_LIBS = \
-    -L$(SCOTCH_ARCH_PATH)/lib \
-    -L$(FOAM_EXT_LIBBIN)/$(FOAM_MPI) \
+    $(PLIBS) \
     -lptscotch \
     -lptscotcherrexit \
     -lscotch \
     -lrt
+
+ifneq ("$(SCOTCH_ARCH_PATH)","")
+EXE_INC += -I$(SCOTCH_ARCH_PATH)/include/$(FOAM_MPI)
+EXE_INC += -I$(SCOTCH_ARCH_PATH)/include
+LIB_LIBS += -L$(SCOTCH_ARCH_PATH)/lib
+endif
+
+ifneq ("$(wildcard /usr/include/scotch)","")
+EXE_INC += -I/usr/include/scotch
+endif
+
+ifneq ("$(wildcard $(FOAM_EXT_LIBBIN)/$(FOAM_MPI))","")
+LIB_LIBS += -L$(FOAM_EXT_LIBBIN)/$(FOAM_MPI)
+endif
diff --git a/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C b/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C
index dcd2596..ae79f1e 100644
--- a/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C
+++ b/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C
@@ -23,6 +23,8 @@ License
 
 \*---------------------------------------------------------------------------*/
 
+#include <type_traits>
+
 #include "ptscotchDecomp.H"
 #include "addToRunTimeSelectionTable.H"
 #include "Time.H"
@@ -303,6 +305,26 @@ Foam::label Foam::ptscotchDecomp::decompose
         Pout<< "ptscotchDecomp : entering with xadj:" << xadjSize << endl;
     }
 
+    if (sizeof(label) > sizeof(SCOTCH_Num))
+    {
+        WarningInFunction
+            << "Size of SCOTCH_Num is less than size of label, "
+            "possible data loss." << endl;
+    }
+
+    List<SCOTCH_Num> tadjncy(adjncySize, 0);
+    List<SCOTCH_Num> txadj(xadjSize, 0);
+
+    for (label i = 0; i < adjncySize; i++)
+    {
+        tadjncy[i] = static_cast<SCOTCH_Num>(adjncy[i]);
+    }
+
+    for (label i = 0; i < xadjSize; i++)
+    {
+        txadj[i] = static_cast<SCOTCH_Num>(xadj[i]);
+    }
+
     // Dump graph
     if (decompositionDict_.found("scotchCoeffs"))
     {
@@ -388,7 +410,7 @@ Foam::label Foam::ptscotchDecomp::decompose
     // Graph
     // ~~~~~
 
-    List<label> velotab;
+    List<SCOTCH_Num> velotab;
 
 
     // Check for externally provided cellweights and if so initialise weights
@@ -486,19 +508,19 @@ Foam::label Foam::ptscotchDecomp::decompose
             0,                      // baseval, c-style numbering
             xadjSize-1,             // vertlocnbr, nCells
             xadjSize-1,             // vertlocmax
-            const_cast<SCOTCH_Num*>(xadj),
+            txadj.data(),
                                     // vertloctab, start index per cell into
                                     // adjncy
-            const_cast<SCOTCH_Num*>(xadj+1),// vendloctab, end index  ,,
+            txadj.data()+1,         // vendloctab, end index  ,,
 
-            const_cast<SCOTCH_Num*>(velotab.begin()),// veloloctab, vtx weights
-            nullptr,                   // vlblloctab
+            velotab.begin(),        // veloloctab, vtx weights
+            nullptr,                // vlblloctab
 
             adjncySize,             // edgelocnbr, number of arcs
             adjncySize,             // edgelocsiz
-            const_cast<SCOTCH_Num*>(adjncy),         // edgeloctab
-            nullptr,                   // edgegsttab
-            nullptr                    // edlotab, edge weights
+            tadjncy.data(),         // edgeloctab
+            nullptr,                // edgegsttab
+            nullptr                 // edlotab, edge weights
         ),
         "SCOTCH_dgraphBuild"
     );
@@ -522,7 +544,7 @@ Foam::label Foam::ptscotchDecomp::decompose
     SCOTCH_Arch archdat;
     check(SCOTCH_archInit(&archdat), "SCOTCH_archInit");
 
-    List<label> processorWeights;
+    List<SCOTCH_Num> processorWeights;
     if (decompositionDict_.found("scotchCoeffs"))
     {
         const dictionary& scotchCoeffs =
@@ -577,17 +599,22 @@ Foam::label Foam::ptscotchDecomp::decompose
     {
         Pout<< "SCOTCH_dgraphMap" << endl;
     }
+    List<SCOTCH_Num> tfinalDecomp(max(1, xadjSize-1), 0);
     check
     (
         SCOTCH_dgraphMap
         (
             &grafdat,
             &archdat,
-            &stradat,           // const SCOTCH_Strat *
-            finalDecomp.begin() // parttab
+            &stradat,            // const SCOTCH_Strat *
+            tfinalDecomp.begin() // parttab
         ),
         "SCOTCH_graphMap"
     );
+    forAll(tfinalDecomp, i)
+    {
+        finalDecomp[i] = static_cast<label>(tfinalDecomp[i]);
+    }
 
     #ifdef  FE_NOMASK_ENV
     feenableexcept(oldExcepts);
diff --git a/src/parallel/decompose/scotchDecomp/Make/options b/src/parallel/decompose/scotchDecomp/Make/options
index d2cc770..5dd1627 100644
--- a/src/parallel/decompose/scotchDecomp/Make/options
+++ b/src/parallel/decompose/scotchDecomp/Make/options
@@ -7,13 +7,22 @@ sinclude $(RULES)/mplib$(WM_MPLIB)
 
 EXE_INC = \
     $(PFLAGS) $(PINC) \
-    -I$(SCOTCH_ARCH_PATH)/include \
-    -I/usr/include/scotch \
     -I../decompositionMethods/lnInclude
 
 LIB_LIBS = \
-    -L$(SCOTCH_ARCH_PATH)/lib \
-    -L$(FOAM_EXT_LIBBIN) \
     -lscotch \
     -lscotcherrexit \
     -lrt
+
+ifneq ("$(SCOTCH_ARCH_PATH)","")
+EXE_INC += -I$(SCOTCH_ARCH_PATH)/include
+LIB_LIBS += -L$(SCOTCH_ARCH_PATH)/lib
+endif
+
+ifneq ("$(wildcard /usr/include/scotch)","")
+EXE_INC += -I/usr/include/scotch
+endif
+
+ifneq ("$(wildcard $(FOAM_EXT_LIBBIN))","")
+LIB_LIBS += -L$(FOAM_EXT_LIBBIN)
+endif
diff --git a/src/parallel/decompose/scotchDecomp/scotchDecomp.C b/src/parallel/decompose/scotchDecomp/scotchDecomp.C
index 987ef18..5f84373 100644
--- a/src/parallel/decompose/scotchDecomp/scotchDecomp.C
+++ b/src/parallel/decompose/scotchDecomp/scotchDecomp.C
@@ -206,6 +206,26 @@ Foam::label Foam::scotchDecomp::decomposeOneProc
     List<label>& finalDecomp
 )
 {
+    if (sizeof(label) > sizeof(SCOTCH_Num))
+    {
+        WarningInFunction
+            << "Size of SCOTCH_Num is less than size of label, "
+            "possible data loss." << endl;
+    }
+    // Temporary lists to resolve SCOTCH_Num and label differences
+    List<SCOTCH_Num> sadjncy(adjncy.size());
+    List<SCOTCH_Num> sxadj(xadj.size());
+
+    forAll(adjncy, i)
+    {
+        sadjncy[i] = static_cast<SCOTCH_Num>(adjncy[i]);
+    }
+
+    forAll(xadj, i)
+    {
+        sxadj[i] = static_cast<SCOTCH_Num>(xadj[i]);
+    }
+
     // Dump graph
     if (decompositionDict_.found("scotchCoeffs"))
     {
@@ -276,7 +296,7 @@ Foam::label Foam::scotchDecomp::decomposeOneProc
     // Graph
     // ~~~~~
 
-    List<label> velotab;
+    List<SCOTCH_Num> velotab;
 
 
     // Check for externally provided cellweights and if so initialise weights
@@ -334,13 +354,13 @@ Foam::label Foam::scotchDecomp::decomposeOneProc
         (
             &grafdat,
             0,                      // baseval, c-style numbering
-            xadj.size()-1,          // vertnbr, nCells
-            xadj.begin(),           // verttab, start index per cell into adjncy
-            &xadj[1],               // vendtab, end index  ,,
+            sxadj.size()-1,          // vertnbr, nCells
+            sxadj.begin(),           // verttab, start index per cell into adjncy
+            &sxadj[1],               // vendtab, end index  ,,
             velotab.begin(),        // velotab, vertex weights
             nullptr,                   // vlbltab
-            adjncy.size(),          // edgenbr, number of arcs
-            adjncy.begin(),         // edgetab
+            sadjncy.size(),          // edgenbr, number of arcs
+            sadjncy.begin(),         // edgetab
             nullptr                    // edlotab, edge weights
         ),
         "SCOTCH_graphBuild"
@@ -355,7 +375,7 @@ Foam::label Foam::scotchDecomp::decomposeOneProc
     SCOTCH_Arch archdat;
     check(SCOTCH_archInit(&archdat), "SCOTCH_archInit");
 
-    List<label> processorWeights;
+    List<SCOTCH_Num> processorWeights;
     if (decompositionDict_.found("scotchCoeffs"))
     {
         const dictionary& scotchCoeffs =
@@ -438,6 +458,7 @@ Foam::label Foam::scotchDecomp::decomposeOneProc
 
     finalDecomp.setSize(xadj.size()-1);
     finalDecomp = 0;
+    List<SCOTCH_Num> tfinalDecomp(xadj.size()-1, 0);
     check
     (
         SCOTCH_graphMap
@@ -445,17 +466,20 @@ Foam::label Foam::scotchDecomp::decomposeOneProc
             &grafdat,
             &archdat,
             &stradat,           // const SCOTCH_Strat *
-            finalDecomp.begin() // parttab
+            tfinalDecomp.begin() // parttab
         ),
         "SCOTCH_graphMap"
     );
+    forAll(tfinalDecomp, i)
+    {
+        finalDecomp[i] = static_cast<label>(tfinalDecomp[i]);
+    }
 
     #ifdef FE_NOMASK_ENV
     feenableexcept(oldExcepts);
     #endif
 
 
-
     //finalDecomp.setSize(xadj.size()-1);
     //check
     //(
-- 
2.7.4