"Fossies" - the Fresh Open Source Software Archive

Member "mvapich2-2.3.2/src/mpid/ch3/channels/common/src/detect/hca/mv2_hca_detect.c" (8 Aug 2019, 22828 Bytes) of package /linux/misc/mvapich2-2.3.2.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "mv2_hca_detect.c" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 2.3.1_vs_2.3.2.

    1 /* Copyright (c) 2001-2019, The Ohio State University. All rights
    2  * reserved.
    3  * Copyright (c) 2016, Intel, Inc. All rights reserved.
    4  *
    5  * This file is part of the MVAPICH2 software package developed by the
    6  * team members of The Ohio State University's Network-Based Computing
    7  * Laboratory (NBCL), headed by Professor Dhabaleswar K. (DK) Panda.
    8  *
    9  * For detailed copyright and licensing information, please refer to the
   10  * copyright file COPYRIGHT in the top level MVAPICH2 directory.
   11  *
   12  */
   13 
   14 #include <stdio.h>
   15 #include <string.h>
   16 #include <stdlib.h>
   17 
   18 #include "mpichconf.h"
   19 
   20 #if defined(HAVE_LIBIBUMAD)
   21 #include <infiniband/umad.h>
   22 #endif
   23 
   24 #include "mv2_arch_hca_detect.h"
   25 #include "upmi.h"
   26 #include "debug_utils.h"
   27 
   28 #include "upmi.h"
   29 #include "mpi.h"
   30 #if ENABLE_PVAR_MV2 && CHANNEL_MRAIL
   31 #include "rdma_impl.h"
   32 #include "mv2_mpit_cvars.h"
   33 #endif
   34 
   35 /*
   36 === BEGIN_MPI_T_CVAR_INFO_BLOCK ===
   37 
   38 cvars:
   39     - name        : FORCE_HCA_TYPE
   40       category    : CH3
   41       type        : int
   42       default     : 0
   43       class       : none
   44       verbosity   : MPI_T_VERBOSITY_USER_BASIC
   45       scope       : MPI_T_SCOPE_ALL_EQ
   46       description : >-
   47         This parameter forces the HCA type.
   48 
   49 === END_MPI_T_CVAR_INFO_BLOCK ===
   50 */
   51 
   52 extern int g_mv2_num_cpus;
   53 static mv2_multirail_info_type g_mv2_multirail_info = mv2_num_rail_unknown;
   54 
   55 #define MV2_STR_MLX          "mlx"
   56 #define MV2_STR_MLX4         "mlx4"
   57 #define MV2_STR_MLX5         "mlx5"
   58 #define MV2_STR_MTHCA        "mthca"
   59 #define MV2_STR_IPATH        "ipath"
   60 #define MV2_STR_QIB          "qib"
   61 #define MV2_STR_HFI1         "hfi1"
   62 #define MV2_STR_EHCA         "ehca"
   63 #define MV2_STR_CXGB3        "cxgb3"
   64 #define MV2_STR_CXGB4        "cxgb4"
   65 #define MV2_STR_NES0         "nes0"
   66 
   67 #if ENABLE_PVAR_MV2 && CHANNEL_MRAIL
   68 MPI_T_cvar_handle mv2_force_hca_type_handle = NULL;
   69 extern int mv2_set_force_hca_type();
   70 extern void mv2_free_hca_handle ();
   71 void mv2_free_hca_handle () {
   72     if (mv2_force_hca_type_handle) {
   73         MPIU_Free(mv2_force_hca_type_handle);
   74         mv2_force_hca_type_handle = NULL;
   75     }
   76 }
   77 #endif
   78 
   79 typedef struct _mv2_hca_types_log_t{
   80     mv2_hca_type hca_type;
   81     char *hca_name;
   82 }mv2_hca_types_log_t;
   83 
   84 #define MV2_HCA_LAST_ENTRY -1
   85 static mv2_hca_types_log_t mv2_hca_types_log[] = {
   86 
   87     /* Mellanox Cards */
   88     {MV2_HCA_MLX_PCI_EX_SDR,"MV2_HCA_MLX_PCI_EX_SDR"},
   89     {MV2_HCA_MLX_PCI_EX_DDR,"MV2_HCA_MLX_PCI_EX_DDR"},
   90     {MV2_HCA_MLX_CX_SDR,    "MV2_HCA_MLX_CX_SDR"},
   91     {MV2_HCA_MLX_CX_DDR,    "MV2_HCA_MLX_CX_DDR"},
   92     {MV2_HCA_MLX_CX_QDR,    "MV2_HCA_MLX_CX_QDR"},
   93     {MV2_HCA_MLX_CX_FDR,    "MV2_HCA_MLX_CX_FDR"},
   94     {MV2_HCA_MLX_CX_EDR,    "MV2_HCA_MLX_CX_EDR"},
   95     {MV2_HCA_MLX_CX_HDR,    "MV2_HCA_MLX_CX_HDR"},
   96     {MV2_HCA_MLX_CX_CONNIB, "MV2_HCA_MLX_CX_CONNIB"},
   97     {MV2_HCA_MLX_PCI_X,     "MV2_HCA_MLX_PCI_X"},
   98 
   99     /* Qlogic Cards */
  100     {MV2_HCA_QLGIC_PATH_HT, "MV2_HCA_QLGIC_PATH_HT"},
  101     {MV2_HCA_QLGIC_QIB,     "MV2_HCA_QLGIC_QIB"},
  102 
  103     /* IBM Cards */
  104     {MV2_HCA_IBM_EHCA,      "MV2_HCA_IBM_EHCA"},
  105 
  106     /* Intel Cards */
  107     {MV2_HCA_INTEL_HFI1,    "MV2_HCA_INTEL_HFI1"},
  108     
  109     /* Chelsio Cards */
  110     {MV2_HCA_CHELSIO_T3,    "MV2_HCA_CHELSIO_T3"},
  111     {MV2_HCA_CHELSIO_T4,    "MV2_HCA_CHELSIO_T4"},
  112 
  113     /* Intel iWarp Cards */
  114     {MV2_HCA_INTEL_NE020,   "MV2_HCA_INTEL_NE020"},
  115 
  116     /*Unknown */
  117     {MV2_HCA_UNKWN,         "MV2_HCA_UNKWN"},
  118     {MV2_HCA_LAST_ENTRY,    "MV2_HCA_LAST_ENTRY"},
  119 };
  120 
  121 char* mv2_get_hca_name(mv2_hca_type hca_type)
  122 {
  123     int i=0;
  124     if (hca_type == MV2_HCA_ANY) {
  125         return("MV2_HCA_ANY");
  126     }
  127     while(mv2_hca_types_log[i].hca_type != MV2_HCA_LAST_ENTRY){
  128 
  129         if(mv2_hca_types_log[i].hca_type == hca_type){
  130             return(mv2_hca_types_log[i].hca_name);
  131         }
  132         i++;
  133     }
  134     return("MV2_HCA_UNKWN");
  135 }
  136 
  137 #if defined(HAVE_LIBIBUMAD)
  138 static int get_rate(umad_ca_t *umad_ca)
  139 {
  140     int i;
  141     char *value;
  142 
  143     if ((value = getenv("MV2_DEFAULT_PORT")) != NULL) {
  144         int default_port = atoi(value);
  145         
  146         if(default_port <= umad_ca->numports){
  147             if (IBV_PORT_ACTIVE == umad_ca->ports[default_port]->state) {
  148                 return umad_ca->ports[default_port]->rate;
  149             }
  150         }
  151     }
  152 
  153     for (i = 1; i <= umad_ca->numports; i++) {
  154         if (IBV_PORT_ACTIVE == umad_ca->ports[i]->state) {
  155             return umad_ca->ports[i]->rate;
  156         }
  157     }    
  158     return 0;
  159 }
  160 #endif
  161 
  162 static const int get_link_width(uint8_t width)
  163 {
  164     switch (width) {
  165     case 1:  return 1;
  166     case 2:  return 4;
  167     case 4:  return 8;
  168     case 8:  return 12;
  169     /* Links on Frontera are returning 16 as link width for now.
  170      * This is a temporary work around for that. */
  171     case 16:  return 2;
  172     default:
  173         PRINT_ERROR("Invalid link width %u\n", width);
  174         return 0;
  175     }
  176 }
  177 
  178 static const float get_link_speed(uint8_t speed)
  179 {
  180     switch (speed) {
  181     case 1:  return 2.5;  /* SDR */
  182     case 2:  return 5.0;  /* DDR */
  183 
  184     case 4:  /* fall through */
  185     case 8:  return 10.0; /* QDR */
  186 
  187     case 16: return 14.0; /* FDR */
  188     case 32: return 25.0; /* EDR */
  189     case 64: return 50.0; /* EDR */
  190     default:
  191         PRINT_ERROR("Invalid link speed %u\n", speed);
  192         return 0;    /* Invalid speed */
  193     }
  194 }
  195 
  196 int mv2_check_hca_type(mv2_hca_type type, int rank)
  197 {
  198     if (type <= MV2_HCA_LIST_START        || type >= MV2_HCA_LIST_END        ||
  199         type == MV2_HCA_IB_TYPE_START     || type == MV2_HCA_IB_TYPE_END     ||
  200         type == MV2_HCA_MLX_START         || type == MV2_HCA_MLX_END         ||
  201         type == MV2_HCA_IWARP_TYPE_START  || type == MV2_HCA_IWARP_TYPE_END  ||
  202         type == MV2_HCA_CHLSIO_START      || type == MV2_HCA_CHLSIO_END      ||
  203         type == MV2_HCA_INTEL_IWARP_START || type == MV2_HCA_INTEL_IWARP_END ||
  204         type == MV2_HCA_QLGIC_START       || type == MV2_HCA_QLGIC_END       ||
  205         type == MV2_HCA_INTEL_START       || type == MV2_HCA_INTEL_END) {
  206 
  207         PRINT_INFO((rank==0), "Wrong value specified for MV2_FORCE_HCA_TYPE\n");
  208         PRINT_INFO((rank==0), "Value must be greater than %d and less than %d \n",
  209                     MV2_HCA_LIST_START, MV2_HCA_LIST_END);
  210         PRINT_INFO((rank==0), "For IB Cards: Please enter value greater than %d and less than %d\n",
  211                     MV2_HCA_MLX_START, MV2_HCA_MLX_END);
  212         PRINT_INFO((rank==0), "For IBM Cards: Please enter value greater than %d and less than %d\n",
  213                     MV2_HCA_IBM_START, MV2_HCA_IBM_END);
  214         PRINT_INFO((rank==0), "For Intel IWARP Cards: Please enter value greater than %d and less than %d\n",
  215                     MV2_HCA_INTEL_IWARP_START, MV2_HCA_INTEL_IWARP_END);
  216         PRINT_INFO((rank==0), "For Chelsio IWARP Cards: Please enter value greater than %d and less than %d\n",
  217                     MV2_HCA_CHLSIO_START, MV2_HCA_CHLSIO_END);
  218         PRINT_INFO((rank==0), "For QLogic Cards: Please enter value greater than %d and less than %d\n",
  219                     MV2_HCA_QLGIC_START, MV2_HCA_QLGIC_END);
  220         PRINT_INFO((rank==0), "For Intel Cards: Please enter value greater than %d and less than %d\n",
  221                     MV2_HCA_INTEL_START, MV2_HCA_INTEL_END);
  222         return 1;
  223     }
  224     return 0;
  225 }
  226 
  227 #if defined(HAVE_LIBIBVERBS)
  228 mv2_hca_type mv2_new_get_hca_type(struct ibv_context *ctx,
  229                                     struct ibv_device *ib_dev,
  230                                     uint64_t *guid)
  231 {
  232     int rate=0;
  233     int my_rank = -1;
  234     char *value = NULL;
  235     char *dev_name = NULL;
  236     struct ibv_device_attr device_attr;
  237     int max_ports = 0;
  238     mv2_hca_type hca_type = MV2_HCA_UNKWN;
  239 
  240     UPMI_GET_RANK(&my_rank);
  241 
  242 #if ENABLE_PVAR_MV2 && CHANNEL_MRAIL
  243     int cvar_forced = mv2_set_force_hca_type();
  244     if (cvar_forced) {
  245         return mv2_MPIDI_CH3I_RDMA_Process.arch_hca_type;
  246     }
  247 #endif /*ENABLE_PVAR_MV2 && CHANNEL_MRAIL*/
  248 
  249     if ((value = getenv("MV2_FORCE_HCA_TYPE")) != NULL) {
  250         hca_type = atoi(value);
  251         int retval = mv2_check_hca_type(hca_type, my_rank);
  252         if (retval) {
  253             PRINT_INFO((my_rank==0), "Falling back to Automatic HCA detection\n");
  254             hca_type = MV2_HCA_UNKWN;
  255         } else {
  256             return hca_type;
  257         }
  258     }
  259 
  260     dev_name = (char*) ibv_get_device_name( ib_dev );
  261 
  262     if (!dev_name) {
  263         PRINT_INFO((my_rank==0), "**********************WARNING***********************\n");
  264         PRINT_INFO((my_rank==0), "Failed to automatically detect the HCA architecture.\n");
  265         PRINT_INFO((my_rank==0), "This may lead to subpar communication performance.\n");
  266         PRINT_INFO((my_rank==0), "****************************************************\n");
  267         return MV2_HCA_UNKWN;
  268     }
  269 
  270     memset(&device_attr, 0, sizeof(struct ibv_device_attr));
  271     if(!ibv_query_device(ctx, &device_attr)){
  272         max_ports = device_attr.phys_port_cnt;
  273         *guid = device_attr.node_guid;
  274     }
  275 
  276     if (!strncmp(dev_name, MV2_STR_MLX, 3)
  277         || !strncmp(dev_name, MV2_STR_MTHCA, 5)) {
  278 
  279         hca_type = MV2_HCA_MLX_PCI_X;
  280 
  281         int query_port = 1;
  282         struct ibv_port_attr port_attr;
  283 
  284         /* honor MV2_DEFAULT_PORT, if set */
  285         if ((value = getenv("MV2_DEFAULT_PORT")) != NULL) {
  286 
  287             int default_port = atoi(value);
  288             query_port = (default_port <= max_ports) ? default_port : 1;
  289         }
  290 
  291         if (!ibv_query_port(ctx, query_port, &port_attr)) {
  292             rate = (int) (get_link_width(port_attr.active_width)
  293                     * get_link_speed(port_attr.active_speed));
  294             PRINT_DEBUG(DEBUG_INIT_verbose, "rate : %d\n", rate);
  295         }
  296         /* mlx4, mlx5 */ 
  297         switch(rate) {
  298             case 200:
  299                 hca_type = MV2_HCA_MLX_CX_HDR;
  300                 break;
  301 
  302             case 100:
  303                 hca_type = MV2_HCA_MLX_CX_EDR;
  304                 break;
  305 
  306             case 56:
  307                 hca_type = MV2_HCA_MLX_CX_FDR;
  308                 break;
  309 
  310             case 40:
  311                 hca_type = MV2_HCA_MLX_CX_QDR;
  312                 break;
  313 
  314             case 20:
  315                 hca_type = MV2_HCA_MLX_CX_DDR;
  316                 break;
  317 
  318             case 10:
  319                 hca_type = MV2_HCA_MLX_CX_SDR;
  320                 break;
  321 
  322             default:
  323                 hca_type = MV2_HCA_MLX_CX_FDR;
  324                 break;
  325         }
  326         if (!strncmp(dev_name, MV2_STR_MLX5, 4) && rate == 56)
  327                 hca_type = MV2_HCA_MLX_CX_CONNIB; 
  328     } else if(!strncmp(dev_name, MV2_STR_IPATH, 5)) {
  329         hca_type = MV2_HCA_QLGIC_PATH_HT;
  330 
  331     } else if(!strncmp(dev_name, MV2_STR_QIB, 3)) {
  332         hca_type = MV2_HCA_QLGIC_QIB;
  333 
  334     } else if(!strncmp(dev_name, MV2_STR_HFI1, 4)) {
  335         hca_type = MV2_HCA_INTEL_HFI1;
  336 
  337     } else if(!strncmp(dev_name, MV2_STR_EHCA, 4)) {
  338         hca_type = MV2_HCA_IBM_EHCA;
  339 
  340     } else if (!strncmp(dev_name, MV2_STR_CXGB3, 5)) {
  341         hca_type = MV2_HCA_CHELSIO_T3;
  342 
  343     } else if (!strncmp(dev_name, MV2_STR_CXGB4, 5)) {
  344         hca_type = MV2_HCA_CHELSIO_T4;
  345 
  346     } else if (!strncmp(dev_name, MV2_STR_NES0, 4)) {
  347         hca_type = MV2_HCA_INTEL_NE020;
  348 
  349     } else {
  350         hca_type = MV2_HCA_UNKWN;
  351     }    
  352 
  353     if (hca_type == MV2_HCA_UNKWN) {
  354         PRINT_INFO((my_rank==0), "**********************WARNING***********************\n");
  355         PRINT_INFO((my_rank==0), "Failed to automatically detect the HCA architecture.\n");
  356         PRINT_INFO((my_rank==0), "This may lead to subpar communication performance.\n");
  357         PRINT_INFO((my_rank==0), "****************************************************\n");
  358     }
  359 
  360     return hca_type;
  361 }
  362 
  363 mv2_hca_type mv2_get_hca_type( struct ibv_device *dev )
  364 {
  365     int rate=0;
  366     char *value = NULL;
  367     char *dev_name;
  368     int my_rank = -1;
  369     mv2_hca_type hca_type = MV2_HCA_UNKWN;
  370 
  371     UPMI_GET_RANK(&my_rank);
  372     
  373 #if ENABLE_PVAR_MV2 && CHANNEL_MRAIL
  374     int cvar_forced = mv2_set_force_hca_type();
  375     if (cvar_forced) {
  376         return mv2_MPIDI_CH3I_RDMA_Process.arch_hca_type;
  377     }
  378 #endif /*ENABLE_PVAR_MV2 && CHANNEL_MRAIL*/
  379 
  380     if ((value = getenv("MV2_FORCE_HCA_TYPE")) != NULL) {
  381         hca_type = atoi(value);
  382         int retval = mv2_check_hca_type(hca_type, my_rank);
  383         if (retval) {
  384             PRINT_INFO((my_rank==0), "Falling back to Automatic HCA detection\n");
  385             hca_type = MV2_HCA_UNKWN;
  386         } else {
  387             return hca_type;
  388         }
  389     }
  390 
  391     dev_name = (char*) ibv_get_device_name( dev );
  392 
  393     if (!dev_name) {
  394         PRINT_INFO((my_rank==0), "**********************WARNING***********************\n");
  395         PRINT_INFO((my_rank==0), "Failed to automatically detect the HCA architecture.\n");
  396         PRINT_INFO((my_rank==0), "This may lead to subpar communication performance.\n");
  397         PRINT_INFO((my_rank==0), "****************************************************\n");
  398         return MV2_HCA_UNKWN;
  399     }
  400 
  401 #ifdef HAVE_LIBIBUMAD
  402     static char last_name[UMAD_CA_NAME_LEN] = { '\0' };
  403     static mv2_hca_type last_type = MV2_HCA_UNKWN;
  404     if (!strncmp(dev_name, last_name, UMAD_CA_NAME_LEN)) {
  405         return last_type;
  406     } else {
  407         strncpy(last_name, dev_name, UMAD_CA_NAME_LEN);
  408     }
  409 #endif /* #ifdef HAVE_LIBIBUMAD */
  410 
  411     if (!strncmp(dev_name, MV2_STR_MLX4, 4)
  412         || !strncmp(dev_name, MV2_STR_MLX5, 4) 
  413         || !strncmp(dev_name, MV2_STR_MTHCA, 5)) {
  414 
  415         hca_type = MV2_HCA_UNKWN;
  416 #if !defined(HAVE_LIBIBUMAD)
  417         int query_port = 1;
  418         struct ibv_context *ctx= NULL;
  419         struct ibv_port_attr port_attr;
  420 
  421 
  422         ctx = ibv_open_device(dev);
  423         if (!ctx) {
  424             return MV2_HCA_UNKWN;
  425         }
  426 
  427         /* honor MV2_DEFAULT_PORT, if set */
  428         if ((value = getenv("MV2_DEFAULT_PORT")) != NULL) {
  429 
  430             int max_ports = 1;
  431             struct ibv_device_attr device_attr;
  432             int default_port = atoi(value);
  433             
  434             memset(&device_attr, 0, sizeof(struct ibv_device_attr));
  435             if(!ibv_query_device(ctx, &device_attr)){
  436                 max_ports = device_attr.phys_port_cnt;
  437             }
  438             query_port = (default_port <= max_ports) ? default_port : 1;
  439         }
  440         
  441         if (!ibv_query_port(ctx, query_port, &port_attr) &&
  442             (port_attr.state == IBV_PORT_ACTIVE)) {
  443             rate = (int) (get_link_width(port_attr.active_width)
  444                     * get_link_speed(port_attr.active_speed));
  445             PRINT_DEBUG(DEBUG_INIT_verbose, "rate : %d\n", rate);
  446         }
  447 #else
  448         umad_ca_t umad_ca;
  449         if (umad_init() < 0) {
  450             last_type = hca_type;
  451             return hca_type;
  452         }
  453 
  454         memset(&umad_ca, 0, sizeof(umad_ca_t));
  455 
  456         if (umad_get_ca(dev_name, &umad_ca) < 0) {
  457             last_type = hca_type;
  458             return hca_type;
  459         }
  460 
  461         if (!getenv("MV2_USE_RoCE")) {
  462             rate = get_rate(&umad_ca);
  463             if (!rate) {
  464                 umad_release_ca(&umad_ca);
  465                 umad_done();
  466                 last_type = hca_type;
  467                 return hca_type;
  468             }
  469         }
  470 
  471         umad_release_ca(&umad_ca);
  472         umad_done();
  473 
  474         if (!strncmp(dev_name, MV2_STR_MTHCA, 5)) {
  475             hca_type = MV2_HCA_MLX_PCI_X;
  476 
  477 
  478             if (!strncmp(umad_ca.ca_type, "MT25", 4)) {
  479                 switch (rate) {
  480                     case 20:
  481                         hca_type = MV2_HCA_MLX_PCI_EX_DDR;
  482                         break;
  483 
  484                     case 10:
  485                         hca_type = MV2_HCA_MLX_PCI_EX_SDR;
  486                         break;
  487 
  488                     default:
  489                         hca_type = MV2_HCA_MLX_PCI_EX_SDR;
  490                         break;
  491                 }
  492 
  493             } else if (!strncmp(umad_ca.ca_type, "MT23", 4)) {
  494                 hca_type = MV2_HCA_MLX_PCI_X;
  495 
  496             } else {
  497                 hca_type = MV2_HCA_MLX_PCI_EX_SDR; 
  498             }
  499         } else 
  500 #endif
  501         { /* mlx4, mlx5 */ 
  502             switch(rate) {
  503                 case 200:
  504                     hca_type = MV2_HCA_MLX_CX_HDR;
  505                     break;
  506 
  507                 case 100:
  508                     hca_type = MV2_HCA_MLX_CX_EDR;
  509                     break;
  510 
  511                 case 56:
  512                     hca_type = MV2_HCA_MLX_CX_FDR;
  513                     break;
  514 
  515                 case 40:
  516                     hca_type = MV2_HCA_MLX_CX_QDR;
  517                     break;
  518 
  519                 case 20:
  520                     hca_type = MV2_HCA_MLX_CX_DDR;
  521                     break;
  522 
  523                 case 10:
  524                     hca_type = MV2_HCA_MLX_CX_SDR;
  525                     break;
  526 
  527                 default:
  528                     hca_type = MV2_HCA_MLX_CX_SDR;
  529                     break;
  530             }
  531             if (!strncmp(dev_name, MV2_STR_MLX5, 4) && rate == 56)
  532                     hca_type = MV2_HCA_MLX_CX_CONNIB; 
  533         }
  534 
  535     } else if(!strncmp(dev_name, MV2_STR_IPATH, 5)) {
  536         hca_type = MV2_HCA_QLGIC_PATH_HT;
  537 
  538     } else if(!strncmp(dev_name, MV2_STR_QIB, 3)) {
  539         hca_type = MV2_HCA_QLGIC_QIB;
  540 
  541     } else if (!strncmp(dev_name, MV2_STR_HFI1, 4)) {
  542         hca_type = MV2_HCA_INTEL_HFI1;
  543 
  544     } else if(!strncmp(dev_name, MV2_STR_EHCA, 4)) {
  545         hca_type = MV2_HCA_IBM_EHCA;
  546 
  547     } else if (!strncmp(dev_name, MV2_STR_CXGB3, 5)) {
  548         hca_type = MV2_HCA_CHELSIO_T3;
  549 
  550     } else if (!strncmp(dev_name, MV2_STR_CXGB4, 5)) {
  551         hca_type = MV2_HCA_CHELSIO_T4;
  552 
  553     } else if (!strncmp(dev_name, MV2_STR_NES0, 4)) {
  554         hca_type = MV2_HCA_INTEL_NE020;
  555 
  556     } else {
  557         hca_type = MV2_HCA_UNKWN;
  558     }    
  559 #ifdef HAVE_LIBIBUMAD
  560     last_type = hca_type;
  561 #endif /* #ifdef HAVE_LIBIBUMAD */
  562     if (hca_type == MV2_HCA_UNKWN) {
  563         PRINT_INFO((my_rank==0), "**********************WARNING***********************\n");
  564         PRINT_INFO((my_rank==0), "Failed to automatically detect the HCA architecture.\n");
  565         PRINT_INFO((my_rank==0), "This may lead to subpar communication performance.\n");
  566         PRINT_INFO((my_rank==0), "****************************************************\n");
  567     }
  568     return hca_type;
  569 }
  570 #else
  571 mv2_hca_type mv2_get_hca_type(void *dev)
  572 {
  573     int my_rank = -1;
  574     char *value = NULL;
  575     mv2_hca_type hca_type = MV2_HCA_UNKWN;
  576 
  577     UPMI_GET_RANK(&my_rank);
  578 
  579 #if ENABLE_PVAR_MV2 && CHANNEL_MRAIL
  580     int cvar_forced = mv2_set_force_hca_type();
  581     if (cvar_forced) {
  582         return mv2_MPIDI_CH3I_RDMA_Process.arch_hca_type;
  583     }
  584 #endif /*ENABLE_PVAR_MV2 && CHANNEL_MRAIL*/
  585 
  586     if ((value = getenv("MV2_FORCE_HCA_TYPE")) != NULL) {
  587         hca_type = atoi(value);
  588         int retval = mv2_check_hca_type(hca_type, my_rank);
  589         if (retval) {
  590             PRINT_INFO((my_rank==0), "Falling back to Automatic HCA detection\n");
  591             hca_type = MV2_HCA_UNKWN;
  592         } else {
  593             return hca_type;
  594         }
  595     }
  596 
  597 #ifdef HAVE_LIBPSM2
  598     hca_type = MV2_HCA_INTEL_HFI1;
  599 #elif HAVE_LIBPSM_INFINIPATH
  600     hca_type = MV2_HCA_QLGIC_QIB;
  601 #else
  602     hca_type = MV2_HCA_UNKWN;
  603 #endif
  604 
  605     return hca_type;
  606 }
  607 #endif
  608 
  609 #if defined(HAVE_LIBIBVERBS)
  610 mv2_arch_hca_type mv2_new_get_arch_hca_type (mv2_hca_type hca_type)
  611 {
  612     mv2_arch_hca_type arch_hca = mv2_get_arch_type();
  613     arch_hca = arch_hca << 16 | hca_type;
  614     arch_hca = arch_hca << 16 | (mv2_arch_num_cores) g_mv2_num_cpus;
  615     return arch_hca;
  616 }
  617 
  618 mv2_arch_hca_type mv2_get_arch_hca_type (struct ibv_device *dev)
  619 {
  620     mv2_arch_hca_type arch_hca = mv2_get_arch_type();
  621     arch_hca = arch_hca << 16 | mv2_get_hca_type(dev);
  622     arch_hca = arch_hca << 16 | (mv2_arch_num_cores) g_mv2_num_cpus;
  623     return arch_hca;
  624 }
  625 #else 
  626 mv2_arch_hca_type mv2_get_arch_hca_type (void *dev)
  627 {
  628     mv2_arch_hca_type arch_hca = mv2_get_arch_type();
  629     arch_hca = arch_hca << 16 | mv2_get_hca_type(dev);
  630     arch_hca = arch_hca << 16 | (mv2_arch_num_cores) g_mv2_num_cpus;
  631     return arch_hca;
  632 }
  633 #endif
  634 
  635 #if defined(HAVE_LIBIBVERBS)
  636 mv2_multirail_info_type mv2_get_multirail_info()
  637 {
  638     if ( mv2_num_rail_unknown == g_mv2_multirail_info ) {
  639         int num_devices;
  640         struct ibv_device **dev_list = NULL;
  641 
  642         /* Get the number of rails */
  643         dev_list = ibv_get_device_list(&num_devices);
  644 
  645         switch (num_devices){
  646             case 1:
  647                 g_mv2_multirail_info = mv2_num_rail_1;
  648                 break;
  649             case 2:
  650                 g_mv2_multirail_info = mv2_num_rail_2;
  651                 break;
  652             case 3:
  653                 g_mv2_multirail_info = mv2_num_rail_3;
  654                 break;
  655             case 4:
  656                 g_mv2_multirail_info = mv2_num_rail_4;
  657                 break;
  658             default:
  659                 g_mv2_multirail_info = mv2_num_rail_unknown;
  660                 break;
  661         }
  662         if (dev_list) {
  663             ibv_free_device_list(dev_list);
  664         }
  665     }
  666     return g_mv2_multirail_info;
  667 }
  668 #else
  669 mv2_multirail_info_type mv2_get_multirail_info()
  670 {
  671     return mv2_num_rail_unknown;
  672 }
  673 
  674 #endif
  675 
  676 #if ENABLE_PVAR_MV2 && CHANNEL_MRAIL
  677 int mv2_set_force_hca_type()
  678 {
  679     int mpi_errno = MPI_SUCCESS;
  680     int cvar_index = 0;
  681     int skip_setting = 0;
  682     int read_value = 0;
  683 
  684     /* Get CVAR index by name */
  685     MPIR_CVAR_GET_INDEX_impl(MPIR_CVAR_FORCE_HCA_TYPE, cvar_index);
  686     if (cvar_index < 0) {
  687         mpi_errno = MPI_ERR_INTERN;
  688         goto fn_fail;
  689     }
  690     mv2_mpit_cvar_access_t wrapper;
  691     wrapper.cvar_name = "MPIR_CVAR_FORCE_HCA_TYPE";
  692     wrapper.cvar_index = cvar_index;
  693     wrapper.cvar_handle = mv2_force_hca_type_handle;
  694     wrapper.default_cvar_value = MV2_HCA_UNKWN;
  695     wrapper.skip_if_default_has_set = 1;
  696     wrapper.error_type = MV2_CVAR_FATAL_ERR;
  697     wrapper.check4_associate_env_conflict = 1;
  698     wrapper.env_name = "MV2_FORCE_HCA_TYPE";
  699     wrapper.env_conflict_error_msg = "the CVAR will set up to default";
  700     wrapper.check_max = 1;
  701     wrapper.max_value = MV2_HCA_LIST_END-1;
  702     wrapper.check_min = 1;
  703     wrapper.min_value = MV2_HCA_LIST_START+1;
  704     wrapper.boundary_error_msg = "Wrong value specified for MPIR_CVAR_FORCE_HCA_TYPE";
  705     wrapper.skip = &skip_setting;
  706     wrapper.value = &read_value;
  707     mpi_errno = mv2_read_and_check_cvar(wrapper);
  708     if (mpi_errno != MPI_SUCCESS){
  709         goto fn_fail;
  710     }
  711     /* Choose algorithm based on CVAR */
  712     if (!skip_setting) {
  713         mv2_hca_type hca_type = read_value;
  714         int retval = mv2_check_hca_type(hca_type,  MPIDI_Process.my_pg_rank);
  715         if (retval) {
  716             PRINT_INFO( (MPIDI_Process.my_pg_rank==0), 
  717                 "### cvar func### Falling back to Automatic HCA detection\n");
  718             hca_type = MV2_HCA_UNKWN;
  719         } 
  720         else {
  721             mv2_MPIDI_CH3I_RDMA_Process.hca_type = hca_type;
  722             mv2_arch_hca_type arch_hca = mv2_get_arch_type();
  723             mv2_MPIDI_CH3I_RDMA_Process.arch_hca_type = 
  724                 (((arch_hca << 16 | hca_type) << 16) | g_mv2_num_cpus);
  725             goto fn_change;
  726         }
  727     }
  728 
  729     fn_fail:
  730     fn_exit:
  731         return mpi_errno;
  732     fn_change:
  733         return 1;
  734 }
  735 #endif