Panzer: add Tpetra tests for directories other than adapter-stk

packages/panzer/adapters-ioss/test/ioss_connmngr/tIOSSConnManager.cpp

@@ -45,12 +45,6 @@
 #include "Ioss_IOFactory.h"
 #include "Ioss_DatabaseIO.h"
 
-//#ifdef HAVE_MPI
-//   #include "Epetra_MpiComm.h"
-//#else
-//   #include "Epetra_SerialComm.h"
-//#endif
-
 using Teuchos::RCP;
 using Teuchos::rcp;
 

packages/panzer/adapters-stk/test/bcstrategy/bcstrategy_composite_factory.cpp

@@ -43,8 +43,6 @@
 #include "user_app_EquationSetFactory.hpp"
 #include "user_app_ClosureModel_Factory_TemplateBuilder.hpp"
 
-#include "Epetra_MpiComm.h"
-
 namespace panzer {
 
   TEUCHOS_UNIT_TEST(bcstrategy, basic_construction)

packages/panzer/adapters-stk/test/evaluator_tests/gather_orientation.cpp

@@ -36,8 +36,6 @@ using Teuchos::rcp;
 #include "Teuchos_DefaultMpiComm.hpp"
 #include "Teuchos_OpaqueWrapper.hpp"
 
-#include "Epetra_MpiComm.h"
-
 #include "user_app_EquationSetFactory.hpp"
 
 #include <cstdio> // for get char

packages/panzer/adapters-stk/test/stk_interface_test/tPointLocationSearch.cpp

@@ -21,12 +21,6 @@
 
 #include "Shards_BasicTopologies.hpp"
 
-#ifdef HAVE_MPI
-   #include "Epetra_MpiComm.h"
-#else
-   #include "Epetra_SerialComm.h"
-#endif
-
 // STK_search objects
 #include <stk_search/CoarseSearch.hpp>
 #include <stk_search/BoundingBox.hpp>

packages/panzer/adapters-stk/test/stk_interface_test/tUniformRef.cpp

@@ -23,12 +23,6 @@
 
 #include "Shards_BasicTopologies.hpp"
 
-#ifdef HAVE_MPI
-   #include "Epetra_MpiComm.h"
-#else
-   #include "Epetra_SerialComm.h"
-#endif
-
 /*
 #include "stk_mesh/base/GetEntities.hpp"
 #include "stk_mesh/base/Selector.hpp"

packages/panzer/disc-fe/src/Panzer_L2Projection.cpp

@@ -354,8 +354,7 @@ namespace panzer {
 
     {
       PANZER_FUNC_TIME_MONITOR_DIFF("Exporting of mass matrix",ExportMM);
-      auto map = factory.getMap(0);
-      ghostedMatrix->fillComplete(map,map);
+      ghostedMatrix->fillComplete();
       const auto exporter = factory.getGhostedExport(0);
       ownedMatrix->doExport(*ghostedMatrix, *exporter, Tpetra::ADD);
       ownedMatrix->fillComplete();

packages/panzer/disc-fe/src/lof/Panzer_BlockedTpetraLinearObjFactory.hpp

@@ -25,6 +25,7 @@
 #include "Panzer_LinearObjFactory.hpp"
 #include "Panzer_TpetraLinearObjContainer.hpp"
 #include "Panzer_BlockedTpetraLinearObjContainer.hpp"
+#include "Panzer_TpetraLinearObjContainer.hpp"
 #include "Panzer_BlockedDOFManager.hpp"
 #include "Panzer_CloneableEvaluator.hpp"
 #include "Panzer_HashUtils.hpp" // for pair_hash
@@ -49,6 +50,7 @@ template <typename Traits,typename ScalarT,typename LocalOrdinalT,typename Globa
 class BlockedTpetraLinearObjFactory : public LinearObjFactory<Traits>
                                     , public ThyraObjFactory<double> {
 public:
+   typedef TpetraLinearObjContainer<ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT> TLOC;
    typedef BlockedTpetraLinearObjContainer<ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT> BTLOC;
    typedef Tpetra::Vector<ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT> VectorType;
    typedef Tpetra::CrsMatrix<ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT> CrsMatrixType;
@@ -133,32 +135,62 @@ class BlockedTpetraLinearObjFactory : public LinearObjFactory<Traits>
    //! Use preconstructed scatter evaluators
    template <typename EvalT>
    Teuchos::RCP<panzer::CloneableEvaluator> buildScatter() const
-   { return Teuchos::rcp(new ScatterResidual_BlockedTpetra<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT,NodeT>(blockedDOFManager_)); }
+   {
+      if(gidProviders_.front()->containsBlockedDOFManager()) {
+         return Teuchos::rcp(new ScatterResidual_BlockedTpetra<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT,NodeT>(getGlobalIndexer()));
+      }
+      return Teuchos::rcp(new ScatterResidual_Tpetra<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT,NodeT>(getRangeGlobalIndexer()));
+   }
 
    //! Use preconstructed gather evaluators
    template <typename EvalT>
    Teuchos::RCP<panzer::CloneableEvaluator > buildGather() const
-   { return Teuchos::rcp(new GatherSolution_BlockedTpetra<EvalT,Traits,ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT>(blockedDOFManager_)); }
+   {
+      if(gidProviders_.front()->containsBlockedDOFManager()) {
+         return Teuchos::rcp(new GatherSolution_BlockedTpetra<EvalT,Traits,ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT>(getGlobalIndexer()));
+      }
+      return Teuchos::rcp(new GatherSolution_Tpetra<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT,NodeT>(getRangeGlobalIndexer()));
+   }
 
    //! Use preconstructed gather evaluators
    template <typename EvalT>
    Teuchos::RCP<panzer::CloneableEvaluator > buildGatherTangent() const
-   { return Teuchos::rcp(new GatherTangent_BlockedTpetra<EvalT,Traits,ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT>(blockedDOFManager_)); }
+   {
+      if(gidProviders_.front()->containsBlockedDOFManager()) {
+         return Teuchos::rcp(new GatherTangent_BlockedTpetra<EvalT,Traits,ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT>(getGlobalIndexer()));
+      }
+      return Teuchos::rcp(new GatherTangent_Tpetra<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT,NodeT>(getRangeGlobalIndexer()));
+   }
 
    //! Use preconstructed gather evaluators
    template <typename EvalT>
    Teuchos::RCP<panzer::CloneableEvaluator > buildGatherDomain() const
-   { return Teuchos::rcp(new GatherSolution_BlockedTpetra<EvalT,Traits,ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT>(blockedDOFManager_)); }
+   {
+      if(gidProviders_.back()->containsBlockedDOFManager()) {
+         return Teuchos::rcp(new GatherSolution_BlockedTpetra<EvalT,Traits,ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT>(getDomainBlockedIndexer()));
+      }
+      return Teuchos::rcp(new GatherSolution_Tpetra<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT,NodeT>(getDomainGlobalIndexer()));
+   }
 
    //! Use preconstructed gather evaluators
    template <typename EvalT>
    Teuchos::RCP<panzer::CloneableEvaluator > buildGatherOrientation() const
-   { return Teuchos::rcp(new GatherOrientation<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT>(nc2c_vector(blockedDOFManager_->getFieldDOFManagers()))); }
+   {
+      if(gidProviders_.front()->containsBlockedDOFManager()) {
+         return Teuchos::rcp(new GatherOrientation<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT>(nc2c_vector(getGlobalIndexer()->getFieldDOFManagers())));
+      }
+      return Teuchos::rcp(new GatherOrientation<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT>(getRangeGlobalIndexer()));
+   }
 
    //! Use preconstructed dirichlet scatter evaluators
    template <typename EvalT>
    Teuchos::RCP<panzer::CloneableEvaluator> buildScatterDirichlet() const
-   { return Teuchos::rcp(new ScatterDirichletResidual_BlockedTpetra<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT,NodeT>(blockedDOFManager_)); }
+   {
+      if(gidProviders_.front()->containsBlockedDOFManager()) {
+         return Teuchos::rcp(new ScatterDirichletResidual_BlockedTpetra<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT,NodeT>(getGlobalIndexer()));
+      }
+      return Teuchos::rcp(new ScatterDirichletResidual_Tpetra<EvalT,Traits,LocalOrdinalT,GlobalOrdinalT,NodeT>(getRangeGlobalIndexer()));
+   }
 
 /*************** Generic helper functions for container setup *******************/
 
@@ -234,9 +266,11 @@ class BlockedTpetraLinearObjFactory : public LinearObjFactory<Traits>
 
    //! get the map from the matrix
    virtual Teuchos::RCP<const MapType> getMap(int i) const;
+   virtual Teuchos::RCP<const MapType> getColMap(int i) const;
 
    //! get the ghosted map from the matrix
    virtual Teuchos::RCP<const MapType> getGhostedMap(int i) const;
+   virtual Teuchos::RCP<const MapType> getColGhostedMap(int i) const;
 
    //! get the graph of the crs matrix
    virtual Teuchos::RCP<const CrsGraphType> getGraph(int i,int j) const;
@@ -246,9 +280,11 @@ class BlockedTpetraLinearObjFactory : public LinearObjFactory<Traits>
 
    //! get importer for converting an overalapped object to a "normal" object
    virtual Teuchos::RCP<const ImportType> getGhostedImport(int i) const;
+   virtual Teuchos::RCP<const ImportType> getColGhostedImport(int i) const;
 
    //! get exporter for converting an overalapped object to a "normal" object
    virtual Teuchos::RCP<const ExportType> getGhostedExport(int j) const;
+   virtual Teuchos::RCP<const ExportType> getColGhostedExport(int j) const;
 
    Teuchos::RCP<CrsMatrixType> getTpetraMatrix(int i,int j) const;
    Teuchos::RCP<CrsMatrixType> getGhostedTpetraMatrix(int i,int j) const;
@@ -275,39 +311,145 @@ class BlockedTpetraLinearObjFactory : public LinearObjFactory<Traits>
    virtual void endFill(LinearObjContainer & loc) const;
 
    Teuchos::RCP<const panzer::BlockedDOFManager> getGlobalIndexer() const
-   { return blockedDOFManager_; }
+   {
+      TEUCHOS_ASSERT(!gidProviders_.empty());
+      return gidProviders_.front()->getBlockedIndexer();
+   }
 
    //! Get the domain unique global indexer this factory was created with.
    Teuchos::RCP<const panzer::GlobalIndexer> getDomainGlobalIndexer() const
-   { return blockProvider_; }
+   {
+      TEUCHOS_ASSERT(!gidProviders_.empty());
+      return gidProviders_.back()->getGlobalIndexer();
+   }
+
+   //! Get the domain unique global indexer this factory was created with.
+   Teuchos::RCP<const panzer::BlockedDOFManager> getDomainBlockedIndexer() const
+   {
+      TEUCHOS_ASSERT(!gidProviders_.empty());
+      return gidProviders_.back()->getBlockedIndexer();
+   }
 
    //! Get the range unique global indexer this factory was created with.
    Teuchos::RCP<const panzer::GlobalIndexer> getRangeGlobalIndexer() const
-   { return blockProvider_; }
+   {
+      TEUCHOS_ASSERT(!gidProviders_.empty());
+      return gidProviders_.front()->getGlobalIndexer();
+   }
 
 protected:
+   /*************** Utility class for handling blocked and nonblocked DOF managers *******************/
+
+   /** This classes is mean to abstract away the different global indexer types and hide
+    * if this is a blocked data structure or a unblocked one.
+    */
+   class DOFManagerContainer
+   {
+   public:
+      DOFManagerContainer() {}
+      DOFManagerContainer(const Teuchos::RCP<const GlobalIndexer> &ugi)
+      {
+         setGlobalIndexer(ugi);
+      }
+
+      void setGlobalIndexer(const Teuchos::RCP<const GlobalIndexer> &ugi)
+      {
+         using Teuchos::RCP;
+         using Teuchos::rcp_dynamic_cast;
+
+         auto blockedDOFManager = rcp_dynamic_cast<const BlockedDOFManager>(ugi);
+         auto flatDOFManager = rcp_dynamic_cast<const GlobalIndexer>(ugi);
+
+         if (blockedDOFManager != Teuchos::null)
+         {
+            // set BlockedDOFManager
+            blockedDOFManager_ = blockedDOFManager;
+
+            // get all GID providers
+            auto dofManagers = blockedDOFManager_->getFieldDOFManagers();
+            for (auto itr = dofManagers.begin(); itr != dofManagers.end(); ++itr)
+               gidProviders_.push_back(*itr);
+         }
+         else if (flatDOFManager != Teuchos::null)
+         {
+            // for absolute clarity, nullify the blockedDOFManager_
+            blockedDOFManager_ = Teuchos::null;
+
+            // you have only a single GID provider
+            gidProviders_.push_back(flatDOFManager);
+         }
+         else
+         {
+            TEUCHOS_ASSERT(false);
+         }
+      }
+
+      //! Get the number of global indexers (not including the blocked one) contained.
+      int getFieldBlocks() const
+      {
+         return Teuchos::as<int>(gidProviders_.size());
+      }
+
+      /** Return true if this contains a blocked DOFManager as opposed to only a single DOFManager.
+       * If this returns true then <code>getGlobalIndexer</code> will return a <code>BlockedDOFManager<int,GO></code>,
+       * other wise it will return a <code>GlobalIndexer<int,GO></code>.
+       */
+      bool containsBlockedDOFManager() const
+      {
+         return blockedDOFManager_ != Teuchos::null;
+      }
+
+      //! Get the "parent" global indexer (if <code>containsBlockedDOFManager()==false</code> this will throw)
+      Teuchos::RCP<const BlockedDOFManager> getBlockedIndexer() const
+      {
+         TEUCHOS_ASSERT(containsBlockedDOFManager());
+         return blockedDOFManager_;
+      }
+
+      //! Get the "parent" global indexer (if <code>getFieldBlocks()>1</code> this will be blocked, otherwise it may be either)
+      Teuchos::RCP<const GlobalIndexer> getGlobalIndexer() const
+      {
+         if (blockedDOFManager_ != Teuchos::null)
+            return blockedDOFManager_;
+
+         TEUCHOS_ASSERT(gidProviders_.size() == 1);
+         return gidProviders_[0];
+      }
+
+      //! Get DOFManagers associated with the blocks
+      const std::vector<Teuchos::RCP<const GlobalIndexer>> &getFieldDOFManagers() const
+      {
+         return gidProviders_;
+      }
+
+   private:
+      Teuchos::RCP<const BlockedDOFManager> blockedDOFManager_;
+      std::vector<Teuchos::RCP<const GlobalIndexer>> gidProviders_;
+   };
+
 /*************** Generic methods/members *******************/
 
    // Get the global indexer associated with a particular block
    Teuchos::RCP<const GlobalIndexer> getGlobalIndexer(int i) const;
+   Teuchos::RCP<const GlobalIndexer> getColGlobalIndexer(int i) const;
 
    //! Allocate the space in the std::vector objects so we can fill with appropriate Tpetra data
-   void makeRoomForBlocks(std::size_t blockCnt);
+   void makeRoomForBlocks(std::size_t blockCnt, std::size_t colBlockCnt = 0);
+
+   std::vector<Teuchos::RCP<const DOFManagerContainer> > gidProviders_;
 
-   Teuchos::RCP<const GlobalIndexer> blockProvider_;
-   Teuchos::RCP<const BlockedDOFManager> blockedDOFManager_;
-   std::vector<Teuchos::RCP<const GlobalIndexer> > gidProviders_;
+   bool useColGidProviders_;
 
    // which block entries are ignored
   std::unordered_set<std::pair<int,int>,panzer::pair_hash> excludedPairs_;
 
 /*************** Thyra based methods/members *******************/
 
    void ghostToGlobalThyraVector(const Teuchos::RCP<const Thyra::VectorBase<ScalarT> > & in,
-                                 const Teuchos::RCP<Thyra::VectorBase<ScalarT> > & out) const;
+                                 const Teuchos::RCP<Thyra::VectorBase<ScalarT> > & out, bool col) const;
    void ghostToGlobalThyraMatrix(const Thyra::LinearOpBase<ScalarT> & in,Thyra::LinearOpBase<ScalarT> & out) const;
    void globalToGhostThyraVector(const Teuchos::RCP<const Thyra::VectorBase<ScalarT> > & in,
-                                 const Teuchos::RCP<Thyra::VectorBase<ScalarT> > & out) const;
+                                 const Teuchos::RCP<Thyra::VectorBase<ScalarT> > & out, bool col) const;
 
    mutable Teuchos::RCP<Thyra::ProductVectorSpaceBase<ScalarT> > rangeSpace_;
    mutable Teuchos::RCP<Thyra::ProductVectorSpaceBase<ScalarT> > domainSpace_;
@@ -323,13 +465,16 @@ class BlockedTpetraLinearObjFactory : public LinearObjFactory<Traits>
                                      const Teuchos::Ptr<CrsMatrixType> & A,
                                      bool zeroVectorRows) const;
 
-   void ghostToGlobalTpetraVector(int i,const VectorType & in,VectorType & out) const;
+   void ghostToGlobalTpetraVector(int i,const VectorType & in,VectorType & out, bool col) const;
    void ghostToGlobalTpetraMatrix(int blockRow,const CrsMatrixType & in,CrsMatrixType & out) const;
-   void globalToGhostTpetraVector(int i,const VectorType & in,VectorType & out) const;
+   void globalToGhostTpetraVector(int i,const VectorType & in,VectorType & out, bool col) const;
 
    // get the map from the matrix
    virtual Teuchos::RCP<const MapType> buildTpetraMap(int i) const;
+   virtual Teuchos::RCP<const MapType> buildColTpetraMap(int i) const;
+
    virtual Teuchos::RCP<const MapType> buildTpetraGhostedMap(int i) const;
+   virtual Teuchos::RCP<const MapType> buildColTpetraGhostedMap(int i) const;
 
    // get the graph of the crs matrix
    virtual Teuchos::RCP<const CrsGraphType> buildTpetraGraph(int i,int j) const;
@@ -338,12 +483,22 @@ class BlockedTpetraLinearObjFactory : public LinearObjFactory<Traits>
    // storage for Tpetra graphs and maps
    Teuchos::RCP<const Teuchos::MpiComm<int> > comm_;
    mutable std::vector<Teuchos::RCP<const MapType> > maps_;
+   mutable std::vector<Teuchos::RCP<const MapType> > colMaps_;
+
    mutable std::vector<Teuchos::RCP<const MapType> > ghostedMaps_;
+   mutable std::vector<Teuchos::RCP<const MapType> > colGhostedMaps_;
+
    mutable std::unordered_map<std::pair<int,int>,Teuchos::RCP<const CrsGraphType>,panzer::pair_hash> graphs_ ;
+   mutable std::unordered_map<std::pair<int,int>,Teuchos::RCP<const CrsGraphType>,panzer::pair_hash> colGraphs_ ;
+
    mutable std::unordered_map<std::pair<int,int>,Teuchos::RCP<const CrsGraphType>,panzer::pair_hash> ghostedGraphs_;
+   mutable std::unordered_map<std::pair<int,int>,Teuchos::RCP<const CrsGraphType>,panzer::pair_hash> colGhostedGraphs_;
 
    mutable std::vector<Teuchos::RCP<const ImportType> > importers_;
+   mutable std::vector<Teuchos::RCP<const ImportType> > colImporters_;
+
    mutable std::vector<Teuchos::RCP<const ExportType> > exporters_;
+   mutable std::vector<Teuchos::RCP<const ExportType> > colExporters_;
 };
 
 }