"Fossies" - the Fresh Open Source Software Archive  

Source code changes of the file "src/mpid/ch3/channels/mrail/src/gen2/rdma_iba_priv.c" between
mvapich2-2.3.1.tar.gz and mvapich2-2.3.2.tar.gz

About: MVAPICH2 offers "MPI over InfiniBand, 10GigE/iWARP and RDMA over Converged Ethernet (RoCE)" to achieve best performance, scalability and fault tolerance for high-end computing systems and servers.

rdma_iba_priv.c  (mvapich2-2.3.1):rdma_iba_priv.c  (mvapich2-2.3.2)
skipping to change at line 351 skipping to change at line 351
* *
* Return: * Return:
* Success: MPI_SUCCESS. * Success: MPI_SUCCESS.
* Failure: ERROR (-1). * Failure: ERROR (-1).
*/ */
#undef FUNCNAME #undef FUNCNAME
#define FUNCNAME rdma_find_network_type #define FUNCNAME rdma_find_network_type
#undef FCNAME #undef FCNAME
#define FCNAME MPL_QUOTE(FUNCNAME) #define FCNAME MPL_QUOTE(FUNCNAME)
int rdma_find_network_type(struct ibv_device **dev_list, int num_devices, int rdma_find_network_type(struct ibv_device **dev_list, int num_devices,
int *num_usable_hcas) struct ibv_device **usable_dev_list, int *num_usable_ hcas)
{ {
int i = 0; int i = 0;
int j = 0;
int hca_type = 0; int hca_type = 0;
int network_type = MV2_NETWORK_CLASS_UNKNOWN; int network_type = MV2_HCA_UNKWN;
int num_ib_cards = 0; int num_ib_cards = 0;
int num_iwarp_cards = 0; int num_iwarp_cards = 0;
int num_unknwn_cards = 0; int num_unknwn_cards = 0;
dev_list = ibv_get_device_list(&num_devices);
for (i = 0; i < num_devices; ++i) { for (i = 0; i < num_devices; ++i) {
hca_type = mv2_get_hca_type(dev_list[i]); hca_type = mv2_get_hca_type(dev_list[i]);
PRINT_DEBUG(DEBUG_INIT_verbose>1, "HCA %s type = %s\n", dev_list[i]->
name,
mv2_get_hca_name(hca_type));
if (network_type <= hca_type) {
network_type=hca_type;
}
}
for (i = 0; i < num_devices; ++i) {
hca_type = mv2_get_hca_type(dev_list[i]);
if (network_type != hca_type) {
continue;
}
usable_dev_list[j] = dev_list[i];
PRINT_DEBUG(DEBUG_INIT_verbose>1, "Usable HCA %d = %s. Type = %s\n",
j, dev_list[i]->name, mv2_get_hca_name(hca_type));
j++;
if (MV2_IS_IB_CARD(hca_type)) { if (MV2_IS_IB_CARD(hca_type)) {
num_ib_cards++; num_ib_cards++;
} else if (MV2_IS_IWARP_CARD(hca_type)) { } else if (MV2_IS_IWARP_CARD(hca_type)) {
num_iwarp_cards++; num_iwarp_cards++;
} else { } else {
num_unknwn_cards++; num_unknwn_cards++;
} }
if (MV2_IS_QLE_CARD(hca_type)) { if (MV2_IS_QLE_CARD(hca_type) || MV2_IS_INTEL_CARD(hca_type)) {
PRINT_ERROR("QLogic IB card detected in system\n"); PRINT_ERROR("QLogic IB cards or Intel Omni-Path detected in system.\
n");
PRINT_ERROR("Please re-configure the library with the" PRINT_ERROR("Please re-configure the library with the"
" '--with-device=ch3:psm' configure option" " '--with-device=ch3:psm' configure option"
" for best performance\n"); " for best performance and functionality.\n");
} }
} }
if (num_ib_cards && (num_ib_cards >= num_iwarp_cards)) { if (num_ib_cards && (num_ib_cards >= num_iwarp_cards)) {
network_type = MV2_NETWORK_CLASS_IB;
*num_usable_hcas = num_ib_cards; *num_usable_hcas = num_ib_cards;
} else if (num_iwarp_cards && (num_ib_cards < num_iwarp_cards)) { } else if (num_iwarp_cards && (num_ib_cards < num_iwarp_cards)) {
network_type = MV2_NETWORK_CLASS_IWARP;
*num_usable_hcas = num_iwarp_cards; *num_usable_hcas = num_iwarp_cards;
} else { } else {
network_type = MV2_NETWORK_CLASS_UNKNOWN;
*num_usable_hcas = num_unknwn_cards; *num_usable_hcas = num_unknwn_cards;
} }
ibv_free_device_list(dev_list);
return network_type; return network_type;
} }
/* /*
* Function: rdma_skip_network_card * Function: rdma_skip_network_card
* *
* Description: * Description:
* On a multi-rail network, skip the HCA that does not match with the * On a multi-rail network, skip the HCA that does not match with the
* network type * network type
* *
skipping to change at line 418 skipping to change at line 427
* Failure: ERROR (-1). * Failure: ERROR (-1).
*/ */
#undef FUNCNAME #undef FUNCNAME
#define FUNCNAME rdma_skip_network_card #define FUNCNAME rdma_skip_network_card
#undef FCNAME #undef FCNAME
#define FCNAME MPL_QUOTE(FUNCNAME) #define FCNAME MPL_QUOTE(FUNCNAME)
int rdma_skip_network_card(mv2_iba_network_classes network_type, int rdma_skip_network_card(mv2_iba_network_classes network_type,
struct ibv_device *ib_dev) struct ibv_device *ib_dev)
{ {
int skip = 0; int skip = 0;
int hca_type = 0;
hca_type = mv2_get_hca_type(ib_dev); if (network_type != mv2_get_hca_type(ib_dev)) {
if ((network_type == MV2_NETWORK_CLASS_IB) && (MV2_IS_IWARP_CARD(hca_type)))
{
skip = 1;
} else if ((network_type == MV2_NETWORK_CLASS_IWARP) &&
(MV2_IS_IB_CARD(hca_type))) {
skip = 1; skip = 1;
} else {
skip = 0;
} }
return skip; return skip;
} }
void ring_rdma_close_hca(struct mv2_MPIDI_CH3I_RDMA_Process_t *proc) void ring_rdma_close_hca(struct mv2_MPIDI_CH3I_RDMA_Process_t *proc)
{ {
int err; int err;
proc->boot_device = NULL; proc->boot_device = NULL;
err = ibv_dealloc_pd(proc->boot_ptag); err = ibv_dealloc_pd(proc->boot_ptag);
skipping to change at line 516 skipping to change at line 517
} }
continue; continue;
} }
is_device_opened = 1; is_device_opened = 1;
break; break;
} }
fn_exit: fn_exit:
/* Clean up before exit */ /* Clean up before exit */
ibv_free_device_list(dev_list); if (dev_list) {
ibv_free_device_list(dev_list);
}
return is_device_opened; return is_device_opened;
} }
/* /*
* Function: rdma_open_hca * Function: rdma_open_hca
* *
* Description: * Description:
* Opens the HCA and allocates protection domain for it. * Opens the HCA and allocates protection domain for it.
* *
* Input: * Input:
skipping to change at line 546 skipping to change at line 549
#define FCNAME MPL_QUOTE(FUNCNAME) #define FCNAME MPL_QUOTE(FUNCNAME)
int rdma_open_hca(struct mv2_MPIDI_CH3I_RDMA_Process_t *proc) int rdma_open_hca(struct mv2_MPIDI_CH3I_RDMA_Process_t *proc)
{ {
int i = 0, j = 0; int i = 0, j = 0;
int num_devices = 0; int num_devices = 0;
int num_usable_hcas = 0; int num_usable_hcas = 0;
int first_attempt_to_bind_failed = 0; int first_attempt_to_bind_failed = 0;
int mpi_errno = MPI_SUCCESS; int mpi_errno = MPI_SUCCESS;
struct ibv_device *ib_dev = NULL; struct ibv_device *ib_dev = NULL;
struct ibv_device **dev_list = NULL; struct ibv_device **dev_list = NULL;
#ifdef RDMA_CM struct ibv_device **usable_dev_list = MPIU_Malloc(sizeof(struct ibv_device * )*MAX_NUM_HCAS);
int network_type = MV2_NETWORK_CLASS_UNKNOWN; int network_type = MV2_NETWORK_CLASS_UNKNOWN;
#endif /*RDMA_CM*/
#ifdef CRC_CHECK #ifdef CRC_CHECK
gen_crc_table(); gen_crc_table();
#endif #endif
rdma_num_hcas = 0; rdma_num_hcas = 0;
dev_list = ibv_get_device_list(&num_devices); dev_list = ibv_get_device_list(&num_devices);
#ifdef RDMA_CM
network_type = rdma_find_network_type(dev_list, num_devices, network_type = rdma_find_network_type(dev_list, num_devices,
&num_usable_hcas); usable_dev_list, &num_usable_hcas);
if (network_type == MV2_NETWORK_CLASS_UNKNOWN) { if (network_type == MV2_HCA_UNKWN) {
if (num_usable_hcas) { if (num_usable_hcas) {
PRINT_ERROR("Unknown HCA type: this build of MVAPICH2 does not" PRINT_ERROR("Unknown HCA type: this build of MVAPICH2 does not"
"fully support the HCA found on the system (try with" "fully support the HCA found on the system (try with"
" other build options)\n"); " other build options)\n");
} else { } else {
if ((MPIDI_Get_num_nodes() == 1) && MPIDI_CH3I_Process.has_dpm) { if ((MPIDI_Get_num_nodes() == 1) && MPIDI_CH3I_Process.has_dpm) {
PRINT_ERROR("HCA not found on the system. " PRINT_ERROR("HCA not found on the system. "
"DPM functionaly requires an active HCA.\n"); "DPM functionaly requires an active HCA.\n");
} }
MPIR_ERR_SETFATALANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**fail", MPIR_ERR_SETFATALANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**fail",
"**fail %s", "No IB device found"); "**fail %s", "No IB device found");
} }
} }
#else
num_usable_hcas = num_devices;
#endif /*RDMA_CM*/
retry_hca_open: PRINT_DEBUG(DEBUG_INIT_verbose, "Selected HCA type = %s, Usable HCAs = %d\n"
for (i = 0; i < num_devices; i++) { ,
#ifdef RDMA_CM mv2_get_hca_name(network_type), num_usable_hcas);
if (rdma_skip_network_card(network_type, dev_list[i])) { for (i = 0; i < num_usable_hcas; i++) {
if (rdma_skip_network_card(network_type, usable_dev_list[i])) {
/* Skip HCA's that don't match with network type */ /* Skip HCA's that don't match with network type */
PRINT_DEBUG(DEBUG_INIT_verbose, "1. Skipping HCA %s since type does
not match."
"Selected: %s; Current: %s\n",
usable_dev_list[i]->name, mv2_get_hca_name(network_type)
,
mv2_get_hca_name(mv2_get_hca_type(usable_dev_list[i])));
continue; continue;
} }
#endif /*RDMA_CM*/
if ((rdma_multirail_usage_policy == MV2_MRAIL_BINDING) && if (rdma_multirail_usage_policy == MV2_MRAIL_BINDING) {
(!first_attempt_to_bind_failed)) {
/* Bind a process to a HCA */ /* Bind a process to a HCA */
if (mrail_use_default_mapping) { if (mrail_use_default_mapping) {
mrail_user_defined_p2r_mapping = mrail_user_defined_p2r_mapping =
rdma_local_id % num_usable_hcas; rdma_local_id % num_usable_hcas;
} }
ib_dev = dev_list[mrail_user_defined_p2r_mapping]; ib_dev = usable_dev_list[mrail_user_defined_p2r_mapping];
} else if (!strncmp(rdma_iba_hcas[i], RDMA_IBA_NULL_HCA, 32)) { } else if (!strncmp(rdma_iba_hcas[i], RDMA_IBA_NULL_HCA, 32)) {
/* User hasn't specified any HCA name /* User hasn't specified any HCA name
* We will use the first available HCA(s) */ * We will use the first available HCA(s) */
ib_dev = dev_list[i]; ib_dev = usable_dev_list[i];
} else { } else {
/* User specified HCA(s), try to look for it */ /* User specified HCA(s), try to look for it */
j = 0; j = 0;
while (dev_list[j]) { while (usable_dev_list[j]) {
if (!strncmp(ibv_get_device_name(dev_list[j]), if (!strncmp(ibv_get_device_name(usable_dev_list[j]),
rdma_iba_hcas[rdma_num_hcas], 32)) { rdma_iba_hcas[rdma_num_hcas], 32)) {
ib_dev = dev_list[j]; ib_dev = usable_dev_list[j];
break; break;
} }
j++; j++;
} }
} }
if (!ib_dev) { if (!ib_dev) {
/* Clean up before exit */ /* Clean up before exit */
ibv_free_device_list(dev_list); if (dev_list) {
ibv_free_device_list(dev_list);
}
MPIR_ERR_SETFATALANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**fail", MPIR_ERR_SETFATALANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**fail",
"**fail %s", "No IB device found"); "**fail %s", "No IB device found");
} }
proc->ib_dev[rdma_num_hcas] = ib_dev; proc->ib_dev[rdma_num_hcas] = ib_dev;
proc->nic_context[rdma_num_hcas] = ibv_open_device(ib_dev); proc->nic_context[rdma_num_hcas] = ibv_open_device(ib_dev);
if (!proc->nic_context[rdma_num_hcas]) { if (!proc->nic_context[rdma_num_hcas]) {
/* Clean up before exit */ /* Clean up before exit */
ibv_free_device_list(dev_list); if (dev_list) {
ibv_free_device_list(dev_list);
}
MPIR_ERR_SETFATALANDJUMP2(mpi_errno, MPI_ERR_OTHER, "**fail", MPIR_ERR_SETFATALANDJUMP2(mpi_errno, MPI_ERR_OTHER, "**fail",
"%s %d", "Failed to open HCA number", "%s %d", "Failed to open HCA number",
rdma_num_hcas); rdma_num_hcas);
} }
if (ERROR == rdma_find_active_port(proc->nic_context[rdma_num_hcas], if (ERROR == rdma_find_active_port(proc->nic_context[rdma_num_hcas],
proc->ib_dev[rdma_num_hcas])) { proc->ib_dev[rdma_num_hcas])) {
/* No active port, skip HCA */ /* No active port, skip HCA */
ibv_close_device(proc->nic_context[rdma_num_hcas]); ibv_close_device(proc->nic_context[rdma_num_hcas]);
PRINT_DEBUG(DEBUG_INIT_verbose,"Skipping HCA %s since it does not ha
ve an active port\n",
usable_dev_list[i]->name);
continue; continue;
} }
proc->ptag[rdma_num_hcas] = proc->ptag[rdma_num_hcas] =
ibv_alloc_pd(proc->nic_context[rdma_num_hcas]); ibv_alloc_pd(proc->nic_context[rdma_num_hcas]);
if (!proc->ptag[rdma_num_hcas]) { if (!proc->ptag[rdma_num_hcas]) {
/* Clean up before exit */ /* Clean up before exit */
ibv_free_device_list(dev_list); if (dev_list) {
ibv_free_device_list(dev_list);
}
MPIR_ERR_SETFATALANDJUMP2(mpi_errno, MPI_ERR_OTHER, MPIR_ERR_SETFATALANDJUMP2(mpi_errno, MPI_ERR_OTHER,
"**fail", "%s%d", "**fail", "%s%d",
"Failed to alloc pd number ", "Failed to alloc pd number ",
rdma_num_hcas); rdma_num_hcas);
} }
PRINT_DEBUG(DEBUG_INIT_verbose, "HCA %d/%d = %s\n", rdma_num_hcas+1, PRINT_DEBUG(DEBUG_INIT_verbose,"HCA %d/%d = %s\n", rdma_num_hcas+1,
rdma_num_req_hcas, proc->ib_dev[rdma_num_hcas]->name); rdma_num_req_hcas, proc->ib_dev[rdma_num_hcas]->name);
rdma_num_hcas++; rdma_num_hcas++;
if ((rdma_multirail_usage_policy == MV2_MRAIL_BINDING) || if ((rdma_multirail_usage_policy == MV2_MRAIL_BINDING) ||
(rdma_num_req_hcas == rdma_num_hcas)) { (rdma_num_req_hcas == rdma_num_hcas)) {
/* If usage policy is binding, or if we have found enough /* If usage policy is binding, or if we have found enough
* number of HCAs asked for by the user */ * number of HCAs asked for by the user */
break; break;
} }
} }
if (unlikely(rdma_num_hcas == 0)) { if (unlikely(rdma_num_hcas == 0)) {
if (!first_attempt_to_bind_failed) {
first_attempt_to_bind_failed = 1;
goto retry_hca_open;
}
MPIR_ERR_SETFATALANDJUMP2(mpi_errno, MPI_ERR_OTHER, MPIR_ERR_SETFATALANDJUMP2(mpi_errno, MPI_ERR_OTHER,
"**fail", "%s %d", "**fail", "%s %d",
"No active HCAs found on the system!!!", "No active HCAs found on the system!!!",
rdma_num_hcas); rdma_num_hcas);
} }
fn_exit: fn_exit:
/* Clean up before exit */ /* Clean up before exit */
ibv_free_device_list(dev_list); MPIU_Free(usable_dev_list);
if (dev_list) {
ibv_free_device_list(dev_list);
}
return mpi_errno; return mpi_errno;
fn_fail: fn_fail:
goto fn_exit; goto fn_exit;
} }
#undef FUNCNAME #undef FUNCNAME
#define FUNCNAME rdma_iba_hca_init_noqp #define FUNCNAME rdma_iba_hca_init_noqp
#undef FCNAME #undef FCNAME
#define FCNAME MPL_QUOTE(FUNCNAME) #define FCNAME MPL_QUOTE(FUNCNAME)
skipping to change at line 1568 skipping to change at line 1575
#ifdef _ENABLE_XRC_ #ifdef _ENABLE_XRC_
if (USE_XRC) { if (USE_XRC) {
if (VC_XST_ISSET(vc, XF_INIT_DONE)) if (VC_XST_ISSET(vc, XF_INIT_DONE))
return; return;
else else
VC_XST_SET(vc, XF_INIT_DONE); VC_XST_SET(vc, XF_INIT_DONE);
} }
PRINT_DEBUG(DEBUG_XRC_verbose > 0, "MRAILI_Init_vc %d\n", vc->pg_rank); PRINT_DEBUG(DEBUG_XRC_verbose > 0, "MRAILI_Init_vc %d\n", vc->pg_rank);
#endif #endif
#ifdef _ENABLE_UD_ #ifdef _ENABLE_UD_
if (vc->mrail.state & MRAILI_UD_CONNECTED) { if(rdma_enable_hybrid)
MRAILI_RC_Enable(vc); {
return; if (vc->mrail.state & MRAILI_UD_CONNECTED) {
MRAILI_RC_Enable(vc);
return;
}
} }
#endif #endif
UPMI_GET_SIZE(&pg_size); UPMI_GET_SIZE(&pg_size);
vc->mrail.rfp.phead_RDMA_send = 0; vc->mrail.rfp.phead_RDMA_send = 0;
vc->mrail.rfp.ptail_RDMA_send = 0; vc->mrail.rfp.ptail_RDMA_send = 0;
vc->mrail.rfp.p_RDMA_recv = 0; vc->mrail.rfp.p_RDMA_recv = 0;
vc->mrail.rfp.p_RDMA_recv_tail = 0; vc->mrail.rfp.p_RDMA_recv_tail = 0;
vc->mrail.rfp.rdma_failed = 0; vc->mrail.rfp.rdma_failed = 0;
 End of changes. 37 change blocks. 
55 lines changed or deleted 70 lines changed or added

Home  |  About  |  Features  |  All  |  Newest  |  Dox  |  Diffs  |  RSS Feeds  |  Screenshots  |  Comments  |  Imprint  |  Privacy  |  HTTP(S)