"Fossies" - the Fresh Open Source Software Archive  

Source code changes of the file "Kernel/Operation_Broadcast.cpp" between
getdp-3.4.0-source.tgz and getdp-3.5.0-source.tgz

About: GetDP is a general finite element solver using mixed elements to discretize de Rham-type complexes in one, two and three dimensions.

[ To the main getdp source changes report ]

Operation_Broadcast.cpp  (getdp-3.4.0-source.tgz):Operation_Broadcast.cpp  (getdp-3.5.0-source.tgz)
// GetDP - Copyright (C) 1997-2021 P. Dular and C. Geuzaine, University of Liege // GetDP - Copyright (C) 1997-2022 P. Dular and C. Geuzaine, University of Liege
// //
// See the LICENSE.txt file for license information. Please report all // See the LICENSE.txt file for license information. Please report all
// issues on https://gitlab.onelab.info/getdp/getdp/issues. // issues on https://gitlab.onelab.info/getdp/getdp/issues.
#include <vector> #include <vector>
#include <string> #include <string>
#include <sstream> #include <sstream>
#include <cstring>
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include "GetDPConfig.h" #include "GetDPConfig.h"
#include "ProData.h" #include "ProData.h"
#include "SolvingOperations.h" #include "SolvingOperations.h"
#include "Cal_Quantity.h" #include "Cal_Quantity.h"
#include "Message.h" #include "Message.h"
#include "Cal_Value.h" #include "Cal_Value.h"
#if defined(HAVE_MPI) && !defined(HAVE_PETSC) #if defined(HAVE_MPI) && !defined(HAVE_PETSC)
#include <mpi.h> #include <mpi.h>
#endif #endif
#if defined(HAVE_GMSH) #if defined(HAVE_GMSH)
#include <gmsh.h> #include <gmsh.h>
#endif #endif
#define STRING_SIZE 256 #define STRING_SIZE 256
int Operation_CheckVariables(struct Resolution *Resolution_P, int Operation_CheckVariables(struct Resolution *Resolution_P,
struct Operation *Operation_P, struct Operation *Operation_P,
struct DofData *DofData_P0, struct DofData *DofData_P0,
struct GeoData *GeoData_P0) struct GeoData *GeoData_P0)
{ {
int commsize = Message::GetCommSize(); int commsize = Message::GetCommSize();
int commrank = Message::GetCommRank(); int commrank = Message::GetCommRank();
std::map<std::string, struct Value> &values = Get_AllValueSaved(); std::map<std::string, struct Value> &values = Get_AllValueSaved();
std::vector<std::string> names; std::vector<std::string> names;
for(int rank = 0; rank < commsize; rank++){ for(int rank = 0; rank < commsize; rank++) {
if(rank == commrank){ if(rank == commrank) {
if(!List_Nbr(Operation_P->Case.CheckVariables.Names)){ if(!List_Nbr(Operation_P->Case.CheckVariables.Names)) {
for(std::map<std::string, struct Value>::iterator it = values.begin(); for(std::map<std::string, struct Value>::iterator it = values.begin();
it != values.end(); it++) it != values.end(); it++)
names.push_back(it->first); names.push_back(it->first);
} }
else{ else {
for(unsigned int i = 0; i < List_Nbr(Operation_P->Case.CheckVariables.Na for(unsigned int i = 0;
mes); i++){ i < List_Nbr(Operation_P->Case.CheckVariables.Names); i++) {
char *s; char *s;
List_Read(Operation_P->Case.CheckVariables.Names, i, &s); List_Read(Operation_P->Case.CheckVariables.Names, i, &s);
if (!List_Nbr(Operation_P->Case.CheckVariables.id)) { if(!List_Nbr(Operation_P->Case.CheckVariables.id)) {
if(values.find(s) != values.end()) if(values.find(s) != values.end())
names.push_back(s); names.push_back(s);
else else
Message::Warning("Unknown runtime variable '%s'", s); Message::Warning("Unknown runtime variable '%s'", s);
} }
else{ else {
for(unsigned int j = 0; j < List_Nbr(Operation_P->Case.CheckVariable for(unsigned int j = 0;
s.id); j++){ j < List_Nbr(Operation_P->Case.CheckVariables.id); j++) {
char sidj[STRING_SIZE]; char sidj[STRING_SIZE];
strncpy(sidj, s, sizeof(sidj)); strncpy(sidj, s, sizeof(sidj));
double idj; double idj;
List_Read(Operation_P->Case.GatherVariables.id, j, &idj); List_Read(Operation_P->Case.GatherVariables.id, j, &idj);
char cidj[STRING_SIZE]; char cidj[STRING_SIZE];
snprintf(cidj, sizeof(cidj), "_%g", idj); snprintf(cidj, sizeof(cidj), "_%g", idj);
strcat(sidj, cidj); strcat(sidj, cidj);
if(values.find(sidj) != values.end()) if(values.find(sidj) != values.end())
names.push_back(sidj); names.push_back(sidj);
else else
skipping to change at line 80 skipping to change at line 82
} }
} }
} }
} }
} }
#if defined(HAVE_MPI) #if defined(HAVE_MPI)
MPI_Barrier(MPI_COMM_WORLD); MPI_Barrier(MPI_COMM_WORLD);
#endif #endif
} }
for(int rank = 0; rank < commsize; rank++){ for(int rank = 0; rank < commsize; rank++) {
if (( Operation_P->Case.CheckVariables.from == -1 || if((Operation_P->Case.CheckVariables.from == -1 ||
rank == Operation_P->Case.CheckVariables.from ) && rank == Operation_P->Case.CheckVariables.from) &&
( rank == commrank) ){ (rank == commrank)) {
for(unsigned int i = 0; i < names.size(); i++){ for(unsigned int i = 0; i < names.size(); i++) {
char key[STRING_SIZE]; char key[STRING_SIZE];
strncpy(key, names[i].c_str(), sizeof(key)); strncpy(key, names[i].c_str(), sizeof(key));
struct Value &v = values[key]; struct Value &v = values[key];
std::string stringvalue= Print_Value_ToString(&v); std::string stringvalue = Print_Value_ToString(&v);
printf("On rank %d: %s = %s\n", rank, key, stringvalue.c_str()); printf("On rank %d: %s = %s\n", rank, key, stringvalue.c_str());
} }
} }
#if defined(HAVE_MPI) #if defined(HAVE_MPI)
MPI_Barrier(MPI_COMM_WORLD); MPI_Barrier(MPI_COMM_WORLD);
#endif #endif
} }
return 0; return 0;
} }
#if !defined(HAVE_MPI) #if !defined(HAVE_MPI)
int Operation_BroadcastFieldsGeneric(struct Resolution *Resolution_P, int Operation_BroadcastFieldsGeneric(struct Resolution *Resolution_P,
struct Operation *Operation_P, struct Operation *Operation_P,
struct DofData *DofData_P0, struct DofData *DofData_P0,
struct GeoData *GeoData_P0) struct GeoData *GeoData_P0)
{ {
Message::Info("BroacastFields is noop without MPI"); Message::Info("BroacastFields is noop without MPI");
return 0; return 0;
} }
int Operation_BroadcastVariables(struct Resolution *Resolution_P, int Operation_BroadcastVariables(struct Resolution *Resolution_P,
struct Operation *Operation_P, struct Operation *Operation_P,
struct DofData *DofData_P0, struct DofData *DofData_P0,
struct GeoData *GeoData_P0) struct GeoData *GeoData_P0)
{ {
Message::Info("BroacastVariables is noop without MPI"); Message::Info("BroacastVariables is noop without MPI");
return 0; return 0;
} }
int Operation_GatherVariables(struct Resolution *Resolution_P, int Operation_GatherVariables(struct Resolution *Resolution_P,
struct Operation *Operation_P, struct Operation *Operation_P,
struct DofData *DofData_P0, struct DofData *DofData_P0,
struct GeoData *GeoData_P0) struct GeoData *GeoData_P0)
{ {
Message::Info("GatherVariables is noop without MPI"); Message::Info("GatherVariables is noop without MPI");
return 0; return 0;
} }
int Operation_ScatterVariables(struct Resolution *Resolution_P, int Operation_ScatterVariables(struct Resolution *Resolution_P,
struct Operation *Operation_P, struct Operation *Operation_P,
struct DofData *DofData_P0, struct DofData *DofData_P0,
struct GeoData *GeoData_P0) struct GeoData *GeoData_P0)
{ {
Message::Info("ScatterVariables is noop without MPI"); Message::Info("ScatterVariables is noop without MPI");
return 0; return 0;
} }
#else #else
int Operation_BroadcastFieldsGeneric(struct Resolution *Resolution_P, int Operation_BroadcastFieldsGeneric(struct Resolution *Resolution_P,
struct Operation *Operation_P, struct Operation *Operation_P,
struct DofData *DofData_P0, struct DofData *DofData_P0,
struct GeoData *GeoData_P0) struct GeoData *GeoData_P0)
{ {
// Operation_BroadCastFields was originally designed to work exclusively // Operation_BroadCastFields was originally designed to work exclusively
// within the context of Operation_IterativeLinearSolver; that version is // within the context of Operation_IterativeLinearSolver; that version is
// quite sophisticated, as it allows to broadcast only to certain processors // quite sophisticated, as it allows to broadcast only to certain processors
// (usually the neighbors in a domain decomposition method). This new version // (usually the neighbors in a domain decomposition method). This new version
// is generic, and uses the new Gmsh API. // is generic, and uses the new Gmsh API.
int commsize = Message::GetCommSize(); int commsize = Message::GetCommSize();
int commrank = Message::GetCommRank(); int commrank = Message::GetCommRank();
if(commsize == 1){ if(commsize == 1) {
Message::Info("Nothing to broacast: communicator size is 1"); Message::Info("Nothing to broacast: communicator size is 1");
return 0; return 0;
} }
#if !defined(HAVE_GMSH) #if !defined(HAVE_GMSH)
Message::Error("GetDP must be compiled with Gmsh support for BroadcastFields") Message::Error(
; "GetDP must be compiled with Gmsh support for BroadcastFields");
#else #else
std::vector<int> tags; std::vector<int> tags;
for(int i = 0; i < List_Nbr(Operation_P->Case.BroadcastFields.ViewTags); i++){ for(int i = 0; i < List_Nbr(Operation_P->Case.BroadcastFields.ViewTags);
i++) {
double j; double j;
List_Read(Operation_P->Case.BroadcastFields.ViewTags, i, &j); List_Read(Operation_P->Case.BroadcastFields.ViewTags, i, &j);
tags.push_back((int)j); tags.push_back((int)j);
} }
if(tags.empty()) // broadcast all views if(tags.empty()) // broadcast all views
gmsh::view::getTags(tags); gmsh::view::getTags(tags);
Message::Info(0, "BroadcastFields: %d fields from rank %d (comm. size %d)", Message::Info(0, "BroadcastFields: %d fields from rank %d (comm. size %d)",
tags.size(), commrank, commsize); tags.size(), commrank, commsize);
// TODO: this should probably be implemented using MPI_Allgatherv // TODO: this should probably be implemented using MPI_Allgatherv
for(int rank = 0; rank < commsize; rank++){ for(int rank = 0; rank < commsize; rank++) {
if(rank == commrank){ if(rank == commrank) {
int numTags = tags.size(); int numTags = tags.size();
MPI_Bcast(&numTags, 1, MPI_INT, rank, MPI_COMM_WORLD); MPI_Bcast(&numTags, 1, MPI_INT, rank, MPI_COMM_WORLD);
for(int t = 0; t < numTags; t++){ for(int t = 0; t < numTags; t++) {
int tag = tags[t]; int tag = tags[t];
MPI_Bcast(&tag, 1, MPI_INT, rank, MPI_COMM_WORLD); MPI_Bcast(&tag, 1, MPI_INT, rank, MPI_COMM_WORLD);
std::vector<std::vector<double> > data; std::vector<std::vector<double> > data;
std::vector<std::string> dataTypes; std::vector<std::string> dataTypes;
std::vector<int> numElements; std::vector<int> numElements;
gmsh::view::getListData(tag, dataTypes, numElements, data); gmsh::view::getListData(tag, dataTypes, numElements, data);
int numDataTypes = dataTypes.size(); int numDataTypes = dataTypes.size();
MPI_Bcast(&numDataTypes, 1, MPI_INT, rank, MPI_COMM_WORLD); MPI_Bcast(&numDataTypes, 1, MPI_INT, rank, MPI_COMM_WORLD);
for(unsigned int i = 0; i < dataTypes.size(); i++){ for(unsigned int i = 0; i < dataTypes.size(); i++) {
char dataType[2] = {dataTypes[i][0], dataTypes[i][1]}; char dataType[2] = {dataTypes[i][0], dataTypes[i][1]};
MPI_Bcast(dataType, 2, MPI_SIGNED_CHAR, rank, MPI_COMM_WORLD); MPI_Bcast(dataType, 2, MPI_SIGNED_CHAR, rank, MPI_COMM_WORLD);
MPI_Bcast(&numElements[i], 1, MPI_INT, rank, MPI_COMM_WORLD); MPI_Bcast(&numElements[i], 1, MPI_INT, rank, MPI_COMM_WORLD);
int numData = data[i].size(); int numData = data[i].size();
MPI_Bcast(&numData, 1, MPI_INT, rank, MPI_COMM_WORLD); MPI_Bcast(&numData, 1, MPI_INT, rank, MPI_COMM_WORLD);
MPI_Bcast(&data[i][0], data[i].size(), MPI_DOUBLE, rank, MPI_COMM_WORL MPI_Bcast(&data[i][0], data[i].size(), MPI_DOUBLE, rank,
D); MPI_COMM_WORLD);
} }
} }
} }
else{ else {
int numTags; int numTags;
MPI_Bcast(&numTags, 1, MPI_INT, rank, MPI_COMM_WORLD); MPI_Bcast(&numTags, 1, MPI_INT, rank, MPI_COMM_WORLD);
for(int t = 0; t < numTags; t++){ for(int t = 0; t < numTags; t++) {
int tag; int tag;
MPI_Bcast(&tag, 1, MPI_INT, rank, MPI_COMM_WORLD); MPI_Bcast(&tag, 1, MPI_INT, rank, MPI_COMM_WORLD);
std::stringstream stream; std::stringstream stream;
stream << "copy from rank " << rank << " to on rank " << commrank; stream << "copy from rank " << rank << " to on rank " << commrank;
gmsh::view::add(stream.str(), tag); gmsh::view::add(stream.str(), tag);
int numDataTypes; int numDataTypes;
MPI_Bcast(&numDataTypes, 1, MPI_INT, rank, MPI_COMM_WORLD); MPI_Bcast(&numDataTypes, 1, MPI_INT, rank, MPI_COMM_WORLD);
for(int i = 0; i < numDataTypes; i++){ for(int i = 0; i < numDataTypes; i++) {
char dataType[2]; char dataType[2];
MPI_Bcast(dataType, 2, MPI_SIGNED_CHAR, rank, MPI_COMM_WORLD); MPI_Bcast(dataType, 2, MPI_SIGNED_CHAR, rank, MPI_COMM_WORLD);
int numElements; int numElements;
MPI_Bcast(&numElements, 1, MPI_INT, rank, MPI_COMM_WORLD); MPI_Bcast(&numElements, 1, MPI_INT, rank, MPI_COMM_WORLD);
int numData; int numData;
MPI_Bcast(&numData, 1, MPI_INT, rank, MPI_COMM_WORLD); MPI_Bcast(&numData, 1, MPI_INT, rank, MPI_COMM_WORLD);
std::vector<double> data(numData); std::vector<double> data(numData);
MPI_Bcast(&data[0], data.size(), MPI_DOUBLE, rank, MPI_COMM_WORLD); MPI_Bcast(&data[0], data.size(), MPI_DOUBLE, rank, MPI_COMM_WORLD);
gmsh::view::addListData(tag, dataType, numElements, data); gmsh::view::addListData(tag, dataType, numElements, data);
gmsh::fltk::run(); gmsh::fltk::run();
} }
} }
} }
} }
#endif #endif
return 0; return 0;
} }
int Operation_GatherVariables(struct Resolution *Resolution_P, int Operation_GatherVariables(struct Resolution *Resolution_P,
struct Operation *Operation_P, struct Operation *Operation_P,
struct DofData *DofData_P0, struct DofData *DofData_P0,
struct GeoData *GeoData_P0) struct GeoData *GeoData_P0)
{ {
int commsize = Message::GetCommSize(); int commsize = Message::GetCommSize();
int commrank = Message::GetCommRank(); int commrank = Message::GetCommRank();
Message::Info("GatherVariables: rank %d (size %d)", commrank, commsize); Message::Info("GatherVariables: rank %d (size %d)", commrank, commsize);
int Allgather=1; int Allgather = 1;
if (Operation_P->Case.GatherVariables.to<-1 || if(Operation_P->Case.GatherVariables.to < -1 ||
Operation_P->Case.GatherVariables.to>commsize-1){ Operation_P->Case.GatherVariables.to > commsize - 1) {
Message::Warning("GatherVariables: impossible to Gather towards rank %d whic Message::Warning("GatherVariables: impossible to Gather towards rank %d "
h does not exist " "which does not exist "
"=> GatherVariables is aborted", "=> GatherVariables is aborted",
Operation_P->Case.GatherVariables.to); Operation_P->Case.GatherVariables.to);
return 0; return 0;
} }
else if (Operation_P->Case.GatherVariables.to!=-1) else if(Operation_P->Case.GatherVariables.to != -1)
Allgather=0; Allgather = 0;
std::map<std::string, struct Value> &values = Get_AllValueSaved(); std::map<std::string, struct Value> &values = Get_AllValueSaved();
//////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////
///// ///////
// Identify Valid Variable names to gather ... // Identify Valid Variable names to gather ...
//////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////
///// ///////
std::vector<std::string> names; std::vector<std::string> names;
for(unsigned int i = 0; i < List_Nbr(Operation_P->Case.GatherVariables.Names); for(unsigned int i = 0; i < List_Nbr(Operation_P->Case.GatherVariables.Names);
i++){ i++) {
char *s; char *s;
List_Read(Operation_P->Case.GatherVariables.Names, i, &s); List_Read(Operation_P->Case.GatherVariables.Names, i, &s);
if (!List_Nbr(Operation_P->Case.GatherVariables.id)) { if(!List_Nbr(Operation_P->Case.GatherVariables.id)) {
if(values.find(s) != values.end()) if(values.find(s) != values.end())
names.push_back(s); names.push_back(s);
else else
Message::Warning("GatherVariables: Unknown variable %s", s); Message::Warning("GatherVariables: Unknown variable %s", s);
} }
else { else {
for(unsigned int j = 0; j < List_Nbr(Operation_P->Case.GatherVariables.id) for(unsigned int j = 0;
; j++){ j < List_Nbr(Operation_P->Case.GatherVariables.id); j++) {
char sidj[STRING_SIZE]; char sidj[STRING_SIZE];
strncpy(sidj, s, sizeof(sidj)); strncpy(sidj, s, sizeof(sidj));
double idj; double idj;
List_Read(Operation_P->Case.GatherVariables.id, j, &idj); List_Read(Operation_P->Case.GatherVariables.id, j, &idj);
char cidj[STRING_SIZE]; char cidj[STRING_SIZE];
snprintf(cidj, sizeof(cidj), "_%g", idj); snprintf(cidj, sizeof(cidj), "_%g", idj);
strcat(sidj, cidj); strcat(sidj, cidj);
if(values.find(sidj) != values.end()) if(values.find(sidj) != values.end())
names.push_back(sidj); names.push_back(sidj);
else else
Message::Warning("GatherVariables: Unknown variable %s", sidj); Message::Warning("GatherVariables: Unknown variable %s", sidj);
} }
} }
} }
if(names.empty()) if(names.empty())
Message::Warning("GatherVariables: No known variable given to gather for ran Message::Warning(
k %d",commrank); "GatherVariables: No known variable given to gather for rank %d",
commrank);
int numVar = names.size(); int numVar = names.size();
//Message::Warning("GatherVariables: number of variables= %d", numVar); // Message::Warning("GatherVariables: number of variables= %d", numVar);
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
// Build vectors: vnames, vtypes, vvals ... // Build vectors: vnames, vtypes, vvals ...
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
int vnamessize = numVar*STRING_SIZE; int vnamessize = numVar * STRING_SIZE;
char *vnames; char *vnames;
vnames= new char[vnamessize]; vnames = new char[vnamessize];
int *vtypes; int *vtypes;
vtypes = new int[numVar]; vtypes = new int[numVar];
double *vvals; double *vvals;
vvals= new double[numVar*NBR_MAX_HARMONIC * MAX_DIM]; vvals = new double[numVar * NBR_MAX_HARMONIC * MAX_DIM];
for(unsigned int i = 0; i < numVar; i++){ for(unsigned int i = 0; i < numVar; i++) {
strncpy(&(vnames[i*STRING_SIZE]), names[i].c_str(), STRING_SIZE); strncpy(&(vnames[i * STRING_SIZE]), names[i].c_str(), STRING_SIZE);
struct Value &v = values[&(vnames[i*STRING_SIZE])]; struct Value &v = values[&(vnames[i * STRING_SIZE])];
vtypes[i]= v.Type; vtypes[i] = v.Type;
for(unsigned int k = 0; k < NBR_MAX_HARMONIC * MAX_DIM; k++) for(unsigned int k = 0; k < NBR_MAX_HARMONIC * MAX_DIM; k++)
vvals[i*NBR_MAX_HARMONIC * MAX_DIM + k]=v.Val[k]; vvals[i * NBR_MAX_HARMONIC * MAX_DIM + k] = v.Val[k];
} }
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
// Prepare Gather ... // Prepare Gather ...
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
int numVartot, *vnumVar_gathered; int numVartot, *vnumVar_gathered;
//int *vnamessizes; //(v1) //vnamessizes could be gathered (v1) or deduced fro // int *vnamessizes; //(v1) //vnamessizes could be gathered (v1) or deduced
m vnumVar_gathered and STRING_SIZE (v2) // from vnumVar_gathered and STRING_SIZE (v2)
if (Allgather || commrank==Operation_P->Case.GatherVariables.to) { if(Allgather || commrank == Operation_P->Case.GatherVariables.to) {
vnumVar_gathered = new int[commsize]; vnumVar_gathered = new int[commsize];
//vnamessizes = new int[commsize]; // (v1) // vnamessizes = new int[commsize]; // (v1)
} }
if (Allgather) { if(Allgather) {
MPI_Allreduce(&numVar, &numVartot,1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); MPI_Allreduce(&numVar, &numVartot, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
MPI_Allgather(&numVar, 1, MPI_INT, vnumVar_gathered,1,MPI_INT,MPI_COMM_WORLD MPI_Allgather(&numVar, 1, MPI_INT, vnumVar_gathered, 1, MPI_INT,
); MPI_COMM_WORLD);
//MPI_Allgather(&vnamessize,1, MPI_INT, vnamessizes, 1, MPI_INT, MPI_COMM_WO // MPI_Allgather(&vnamessize,1, MPI_INT, vnamessizes, 1, MPI_INT,
RLD); // (v1) // MPI_COMM_WORLD); // (v1)
} }
else { else {
MPI_Reduce(&numVar,&numVartot,1,MPI_INT, MPI_SUM, Operation_P->Case.GatherVa MPI_Reduce(&numVar, &numVartot, 1, MPI_INT, MPI_SUM,
riables.to, MPI_COMM_WORLD); Operation_P->Case.GatherVariables.to, MPI_COMM_WORLD);
MPI_Gather( &numVar,1,MPI_INT, vnumVar_gathered, 1, MPI_INT, Operation_P->Ca MPI_Gather(&numVar, 1, MPI_INT, vnumVar_gathered, 1, MPI_INT,
se.GatherVariables.to, MPI_COMM_WORLD); Operation_P->Case.GatherVariables.to, MPI_COMM_WORLD);
//MPI_Gather(&vnamessize,1,MPI_INT,vnamessizes,1,MPI_INT,Operation_P->Case.G // MPI_Gather(&vnamessize,1,MPI_INT,vnamessizes,1,MPI_INT,Operation_P->Case.
atherVariables.to,MPI_COMM_WORLD); // (v1) GatherVariables.to,MPI_COMM_WORLD);
// // (v1)
} }
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
// Gather all vectors vnames in vector vnames_gathered ... // Gather all vectors vnames in vector vnames_gathered ...
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
char *vnames_gathered; char *vnames_gathered;
int *vnamessizes, // (v2) int *vnamessizes, // (v2)
*vnamesdispls; *vnamesdispls;
if (Allgather || commrank==Operation_P->Case.GatherVariables.to){ if(Allgather || commrank == Operation_P->Case.GatherVariables.to) {
vnames_gathered = new char[numVartot*STRING_SIZE]; vnames_gathered = new char[numVartot * STRING_SIZE];
vnamessizes = new int[commsize]; // (v2) vnamessizes = new int[commsize]; // (v2)
vnamesdispls = new int[commsize]; vnamesdispls = new int[commsize];
for(unsigned int i = 0; i < commsize; i++) // (v2) for(unsigned int i = 0; i < commsize; i++) // (v2)
vnamessizes[i]=vnumVar_gathered[i]*STRING_SIZE; // (v2) vnamessizes[i] = vnumVar_gathered[i] * STRING_SIZE; // (v2)
vnamesdispls[0]=0; vnamesdispls[0] = 0;
for(unsigned int i = 1; i < commsize; i++) for(unsigned int i = 1; i < commsize; i++)
vnamesdispls[i]=vnamesdispls[i-1]+vnamessizes[i-1]; vnamesdispls[i] = vnamesdispls[i - 1] + vnamessizes[i - 1];
} }
if (Allgather) if(Allgather)
MPI_Allgatherv(vnames,vnamessize,MPI_SIGNED_CHAR,vnames_gathered, vnamessize MPI_Allgatherv(vnames, vnamessize, MPI_SIGNED_CHAR, vnames_gathered,
s, vnamesdispls,MPI_SIGNED_CHAR,MPI_COMM_WORLD); // Build vnames_gathered (needs vnamessizes, vnamesdispls, MPI_SIGNED_CHAR,
vnames (ok), vnamessizes, vnamesdispls) !!! MPI_COMM_WORLD); // Build vnames_gathered (needs vnames (ok),
// vnamessizes, vnamesdispls) !!!
else else
MPI_Gatherv(vnames,vnamessize,MPI_SIGNED_CHAR,vnames_gathered, vnamessizes, MPI_Gatherv(vnames, vnamessize, MPI_SIGNED_CHAR, vnames_gathered,
vnamesdispls,MPI_SIGNED_CHAR,Operation_P->Case.GatherVariables.to,MPI_COMM_WORLD vnamessizes, vnamesdispls, MPI_SIGNED_CHAR,
); Operation_P->Case.GatherVariables.to, MPI_COMM_WORLD);
if (Allgather || commrank==Operation_P->Case.GatherVariables.to) { if(Allgather || commrank == Operation_P->Case.GatherVariables.to) {
delete [] vnames; delete[] vnames;
delete [] vnamessizes; delete[] vnamessizes;
delete [] vnamesdispls; delete[] vnamesdispls;
} }
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
// Gather all vectors vvals in vector vvals_gathered ... // Gather all vectors vvals in vector vvals_gathered ...
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
double *vvals_gathered; double *vvals_gathered;
int *vvalssizes, int *vvalssizes, *vvalsdispls;
*vvalsdispls; if(Allgather || commrank == Operation_P->Case.GatherVariables.to) {
if (Allgather || commrank==Operation_P->Case.GatherVariables.to) { vvals_gathered = new double[numVartot * NBR_MAX_HARMONIC * MAX_DIM];
vvals_gathered = new double[numVartot*NBR_MAX_HARMONIC * MAX_DIM]; vvalssizes = new int[commsize];
vvalssizes = new int[commsize]; vvalsdispls = new int[commsize];
vvalsdispls = new int[commsize];
for(unsigned int i = 0; i < commsize; i++) for(unsigned int i = 0; i < commsize; i++)
vvalssizes[i]=vnumVar_gathered[i]*NBR_MAX_HARMONIC * MAX_DIM; vvalssizes[i] = vnumVar_gathered[i] * NBR_MAX_HARMONIC * MAX_DIM;
vvalsdispls[0]=0; vvalsdispls[0] = 0;
for(unsigned int i = 1; i < commsize; i++) for(unsigned int i = 1; i < commsize; i++)
vvalsdispls[i]=vvalsdispls[i-1]+vvalssizes[i-1]; vvalsdispls[i] = vvalsdispls[i - 1] + vvalssizes[i - 1];
} }
if (Allgather) if(Allgather)
MPI_Allgatherv(vvals,numVar*NBR_MAX_HARMONIC * MAX_DIM, MPI_DOUBLE, vvals_ga MPI_Allgatherv(vvals, numVar * NBR_MAX_HARMONIC * MAX_DIM, MPI_DOUBLE,
thered, vvalssizes, vvalsdispls, MPI_DOUBLE, MPI_COMM_WORLD); vvals_gathered, vvalssizes, vvalsdispls, MPI_DOUBLE,
MPI_COMM_WORLD);
else else
MPI_Gatherv(vvals,numVar*NBR_MAX_HARMONIC * MAX_DIM, MPI_DOUBLE, vvals_gathe MPI_Gatherv(vvals, numVar * NBR_MAX_HARMONIC * MAX_DIM, MPI_DOUBLE,
red, vvalssizes, vvalsdispls, MPI_DOUBLE, Operation_P->Case.GatherVariables.to, vvals_gathered, vvalssizes, vvalsdispls, MPI_DOUBLE,
MPI_COMM_WORLD); Operation_P->Case.GatherVariables.to, MPI_COMM_WORLD);
if (Allgather || commrank==Operation_P->Case.GatherVariables.to){ if(Allgather || commrank == Operation_P->Case.GatherVariables.to) {
delete [] vvals; delete[] vvals;
delete [] vvalssizes; delete[] vvalssizes;
delete [] vvalsdispls; delete[] vvalsdispls;
} }
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
// Gather all vectors vtypes in vector vtypes_gathered ... // Gather all vectors vtypes in vector vtypes_gathered ...
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
int *vtypes_gathered, int *vtypes_gathered, *vtypesdispls;
*vtypesdispls; if(Allgather || commrank == Operation_P->Case.GatherVariables.to) {
if (Allgather || commrank==Operation_P->Case.GatherVariables.to){
vtypes_gathered = new int[numVartot]; vtypes_gathered = new int[numVartot];
//vtypessizes = vnumVar_gathered which is already known at this stage // vtypessizes = vnumVar_gathered which is already known at this stage
vtypesdispls = new int[commsize]; vtypesdispls = new int[commsize];
vtypesdispls[0]=0; vtypesdispls[0] = 0;
for(unsigned int i = 1; i < commsize; i++) for(unsigned int i = 1; i < commsize; i++)
vtypesdispls[i]=vtypesdispls[i-1]+vnumVar_gathered[i-1]; vtypesdispls[i] = vtypesdispls[i - 1] + vnumVar_gathered[i - 1];
} }
if (Allgather) if(Allgather)
MPI_Allgatherv(vtypes,numVar,MPI_INT, vtypes_gathered, vnumVar_gathered, vty MPI_Allgatherv(vtypes, numVar, MPI_INT, vtypes_gathered, vnumVar_gathered,
pesdispls, MPI_INT, MPI_COMM_WORLD); vtypesdispls, MPI_INT, MPI_COMM_WORLD);
else else
MPI_Gatherv(vtypes,numVar,MPI_INT, vtypes_gathered, vnumVar_gathered, vtypes MPI_Gatherv(vtypes, numVar, MPI_INT, vtypes_gathered, vnumVar_gathered,
displs, MPI_INT, Operation_P->Case.GatherVariables.to,MPI_COMM_WORLD); vtypesdispls, MPI_INT, Operation_P->Case.GatherVariables.to,
MPI_COMM_WORLD);
if (Allgather || commrank==Operation_P->Case.GatherVariables.to){
delete [] vtypes; if(Allgather || commrank == Operation_P->Case.GatherVariables.to) {
delete [] vnumVar_gathered; delete[] vtypes;
delete [] vtypesdispls; delete[] vnumVar_gathered;
delete[] vtypesdispls;
} }
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
// Update Variables from gathered vectors // Update Variables from gathered vectors
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
if (Allgather || commrank==Operation_P->Case.GatherVariables.to){ if(Allgather || commrank == Operation_P->Case.GatherVariables.to) {
for(unsigned int i = 0; i < numVartot; i++){ for(unsigned int i = 0; i < numVartot; i++) {
struct Value v; struct Value v;
v.Type = vtypes_gathered[i]; v.Type = vtypes_gathered[i];
for(unsigned int k = 0; k < NBR_MAX_HARMONIC * MAX_DIM; k++) for(unsigned int k = 0; k < NBR_MAX_HARMONIC * MAX_DIM; k++)
v.Val[k]=vvals_gathered[i*NBR_MAX_HARMONIC * MAX_DIM+k]; v.Val[k] = vvals_gathered[i * NBR_MAX_HARMONIC * MAX_DIM + k];
values[&(vnames_gathered[i*STRING_SIZE])]=v; values[&(vnames_gathered[i * STRING_SIZE])] = v;
} }
delete [] vnames_gathered; delete[] vnames_gathered;
delete [] vtypes_gathered; delete[] vtypes_gathered;
delete [] vvals_gathered; delete[] vvals_gathered;
} }
return 0; return 0;
} }
int Operation_ScatterVariables(struct Resolution *Resolution_P, int Operation_ScatterVariables(struct Resolution *Resolution_P,
struct Operation *Operation_P, struct Operation *Operation_P,
struct DofData *DofData_P0, struct DofData *DofData_P0,
struct GeoData *GeoData_P0) struct GeoData *GeoData_P0)
{ {
int commsize = Message::GetCommSize(); int commsize = Message::GetCommSize();
int commrank = Message::GetCommRank(); int commrank = Message::GetCommRank();
Message::Info("ScatterVariables: rank %d (size %d)", commrank, commsize); Message::Info("ScatterVariables: rank %d (size %d)", commrank, commsize);
if (Operation_P->Case.ScatterVariables.from<0 || if(Operation_P->Case.ScatterVariables.from < 0 ||
Operation_P->Case.ScatterVariables.from>commsize-1){ Operation_P->Case.ScatterVariables.from > commsize - 1) {
Message::Warning("ScatterVariables: impossible to Scatter from rank %d which Message::Warning("ScatterVariables: impossible to Scatter from rank %d "
does not exist " "which does not exist "
"=> ScatterVariables is aborted", "=> ScatterVariables is aborted",
Operation_P->Case.ScatterVariables.from); Operation_P->Case.ScatterVariables.from);
return 0; return 0;
} }
int numid = List_Nbr(Operation_P->Case.ScatterVariables.id); int numid = List_Nbr(Operation_P->Case.ScatterVariables.id);
int vid[numid]; // vector of id numbers of variable associated to a rank (vid) int vid[numid]; // vector of id numbers of variable associated to a rank (vid)
for(unsigned int j = 0; j < numid; j++){ for(unsigned int j = 0; j < numid; j++) {
double idj; double idj;
List_Read(Operation_P->Case.ScatterVariables.id, j, &idj); List_Read(Operation_P->Case.ScatterVariables.id, j, &idj);
vid[j]=(int) idj; vid[j] = (int)idj;
} }
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
// (Before Scatter): Gather all numid in the vector vnumid_gathered in the roo // (Before Scatter): Gather all numid in the vector vnumid_gathered in the
t rank and get numidtot // root rank and get numidtot
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
int numidtot, *vnumid_gathered ; int numidtot, *vnumid_gathered;
if (commrank==Operation_P->Case.ScatterVariables.from) if(commrank == Operation_P->Case.ScatterVariables.from)
vnumid_gathered = new int[commsize]; vnumid_gathered = new int[commsize];
MPI_Reduce(&numid,&numidtot,1,MPI_INT, MPI_SUM, Operation_P->Case.ScatterVaria MPI_Reduce(&numid, &numidtot, 1, MPI_INT, MPI_SUM,
bles.from, MPI_COMM_WORLD); Operation_P->Case.ScatterVariables.from, MPI_COMM_WORLD);
MPI_Gather( &numid,1,MPI_INT, vnumid_gathered, 1, MPI_INT, Operation_P->Case.S MPI_Gather(&numid, 1, MPI_INT, vnumid_gathered, 1, MPI_INT,
catterVariables.from, MPI_COMM_WORLD); Operation_P->Case.ScatterVariables.from, MPI_COMM_WORLD);
/* /*
if (commrank==Operation_P->Case.ScatterVariables.from && numidtot==0){ if (commrank==Operation_P->Case.ScatterVariables.from && numidtot==0){
Message::Warning("ScatterVariables: missing information on how to scatter th Message::Warning("ScatterVariables: missing information on how to scatter
e id numbers from root %d towards other ranks\n" the id numbers from root %d towards other ranks\n"
"=> ScatterVariables cannot be applied", commrank); "=> ScatterVariables cannot be applied", commrank);
} }
*/ */
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
// (Before Scatter): Gather all vid vectors in vid_gathered in the root rank // (Before Scatter): Gather all vid vectors in vid_gathered in the root rank
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
int *vid_gathered, *viddispls; int *vid_gathered, *viddispls;
if (commrank==Operation_P->Case.ScatterVariables.from){ if(commrank == Operation_P->Case.ScatterVariables.from) {
vid_gathered = new int[numidtot]; vid_gathered = new int[numidtot];
viddispls = new int[commsize]; viddispls = new int[commsize];
viddispls[0]=0; viddispls[0] = 0;
for(unsigned int i = 1; i < commsize; i++) for(unsigned int i = 1; i < commsize; i++)
viddispls[i]=viddispls[i-1]+vnumid_gathered[i-1]; viddispls[i] = viddispls[i - 1] + vnumid_gathered[i - 1];
} }
MPI_Gatherv(vid,numid,MPI_INT, vid_gathered, vnumid_gathered, viddispls, MPI_I MPI_Gatherv(vid, numid, MPI_INT, vid_gathered, vnumid_gathered, viddispls,
NT, Operation_P->Case.ScatterVariables.from,MPI_COMM_WORLD); MPI_INT, Operation_P->Case.ScatterVariables.from, MPI_COMM_WORLD);
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
// (Before Scatter): Identify Valid Variable names to Scatter from the root ra // (Before Scatter): Identify Valid Variable names to Scatter from the root
nk ... // rank ...
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
std::map<std::string, struct Value> &values = Get_AllValueSaved(); std::map<std::string, struct Value> &values = Get_AllValueSaved();
std::vector<std::string> names; std::vector<std::string> names;
int numVartot, *vnumVar_toscatter; int numVartot, *vnumVar_toscatter;
if (commrank==Operation_P->Case.ScatterVariables.from){ if(commrank == Operation_P->Case.ScatterVariables.from) {
vnumVar_toscatter = new int[commsize]; vnumVar_toscatter = new int[commsize];
for (unsigned int rank=0; rank<commsize; rank++) { for(unsigned int rank = 0; rank < commsize; rank++) {
vnumVar_toscatter[rank] = 0; vnumVar_toscatter[rank] = 0;
for(unsigned int i = 0; i < List_Nbr(Operation_P->Case.ScatterVariables.Na for(unsigned int i = 0;
mes); i++){ i < List_Nbr(Operation_P->Case.ScatterVariables.Names); i++) {
char *s; char *s;
List_Read(Operation_P->Case.ScatterVariables.Names, i, &s); List_Read(Operation_P->Case.ScatterVariables.Names, i, &s);
{ {
for(unsigned int k = 0; k < vnumid_gathered[rank]; k++){ for(unsigned int k = 0; k < vnumid_gathered[rank]; k++) {
char sidj[STRING_SIZE]; char sidj[STRING_SIZE];
strncpy(sidj, s, sizeof(sidj)); strncpy(sidj, s, sizeof(sidj));
char cidj[STRING_SIZE]; char cidj[STRING_SIZE];
snprintf(cidj, sizeof(cidj), "_%d", vid_gathered[viddispls[rank]+k]) snprintf(cidj, sizeof(cidj), "_%d",
; vid_gathered[viddispls[rank] + k]);
strcat(sidj, cidj); strcat(sidj, cidj);
if(values.find(sidj) != values.end()) { if(values.find(sidj) != values.end()) {
names.push_back(sidj); names.push_back(sidj);
vnumVar_toscatter[rank]+=1; vnumVar_toscatter[rank] += 1;
} }
else else
Message::Warning("ScatterVariables: Unknown variable %s", sidj); Message::Warning("ScatterVariables: Unknown variable %s", sidj);
} }
} }
} }
} }
if(names.empty()) if(names.empty())
Message::Warning("ScatterVariables: No known variable found to scatter fro Message::Warning(
m rank %d ",commrank); "ScatterVariables: No known variable found to scatter from rank %d ",
commrank);
numVartot = names.size(); numVartot = names.size();
delete [] vid_gathered; delete[] vid_gathered;
delete [] vnumid_gathered; delete[] vnumid_gathered;
delete [] viddispls; delete[] viddispls;
} }
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
// Build vectors: vnames_toscatter, vtypes_toscatter, vvals_toscatter in the r // Build vectors: vnames_toscatter, vtypes_toscatter, vvals_toscatter in the
oot rank // root rank
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
char *vnames_toscatter; char *vnames_toscatter;
int *vtypes_toscatter; int *vtypes_toscatter;
double *vvals_toscatter; double *vvals_toscatter;
if (commrank==Operation_P->Case.ScatterVariables.from){ if(commrank == Operation_P->Case.ScatterVariables.from) {
int vnamessize_toscatter = numVartot*STRING_SIZE; int vnamessize_toscatter = numVartot * STRING_SIZE;
vnames_toscatter= new char[vnamessize_toscatter]; vnames_toscatter = new char[vnamessize_toscatter];
vtypes_toscatter = new int[numVartot]; vtypes_toscatter = new int[numVartot];
vvals_toscatter= new double[numVartot*NBR_MAX_HARMONIC * MAX_DIM]; vvals_toscatter = new double[numVartot * NBR_MAX_HARMONIC * MAX_DIM];
for(unsigned int i = 0; i < numVartot; i++){ for(unsigned int i = 0; i < numVartot; i++) {
strncpy(&(vnames_toscatter[i*STRING_SIZE]), names[i].c_str(), STRING_SIZE) strncpy(&(vnames_toscatter[i * STRING_SIZE]), names[i].c_str(),
; STRING_SIZE);
struct Value &v = values[&(vnames_toscatter[i*STRING_SIZE])]; struct Value &v = values[&(vnames_toscatter[i * STRING_SIZE])];
vtypes_toscatter[i]= v.Type; vtypes_toscatter[i] = v.Type;
for(unsigned int k = 0; k < NBR_MAX_HARMONIC * MAX_DIM; k++) for(unsigned int k = 0; k < NBR_MAX_HARMONIC * MAX_DIM; k++)
vvals_toscatter[i*NBR_MAX_HARMONIC * MAX_DIM + k]=v.Val[k]; vvals_toscatter[i * NBR_MAX_HARMONIC * MAX_DIM + k] = v.Val[k];
} }
} }
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
// Prepare Scatter: Get numVar that each rank will receive ... // Prepare Scatter: Get numVar that each rank will receive ...
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
int numVar=0; int numVar = 0;
MPI_Scatter(vnumVar_toscatter, 1, MPI_INT, &numVar, 1, MPI_INT, Operation_P->C MPI_Scatter(vnumVar_toscatter, 1, MPI_INT, &numVar, 1, MPI_INT,
ase.ScatterVariables.from, MPI_COMM_WORLD); Operation_P->Case.ScatterVariables.from, MPI_COMM_WORLD);
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
// Scatter vector vnames_toscatter from root to vectors vnames in each rank .. // Scatter vector vnames_toscatter from root to vectors vnames in each rank
. // ...
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
char *vnames; char *vnames;
int *vnamessizes, *vnamesdispls; int *vnamessizes, *vnamesdispls;
vnames = new char[numVar*STRING_SIZE]; vnames = new char[numVar * STRING_SIZE];
if (commrank==Operation_P->Case.ScatterVariables.from) { if(commrank == Operation_P->Case.ScatterVariables.from) {
vnamessizes = new int[commsize]; vnamessizes = new int[commsize];
vnamesdispls = new int[commsize]; vnamesdispls = new int[commsize];
for(unsigned int i = 0; i < commsize; i++) for(unsigned int i = 0; i < commsize; i++)
vnamessizes[i]=vnumVar_toscatter[i]*STRING_SIZE; vnamessizes[i] = vnumVar_toscatter[i] * STRING_SIZE;
vnamesdispls[0]=0; vnamesdispls[0] = 0;
for(unsigned int i = 1; i < commsize; i++) for(unsigned int i = 1; i < commsize; i++)
vnamesdispls[i]=vnamesdispls[i-1]+vnamessizes[i-1]; vnamesdispls[i] = vnamesdispls[i - 1] + vnamessizes[i - 1];
} }
MPI_Scatterv(vnames_toscatter,vnamessizes,vnamesdispls,MPI_SIGNED_CHAR,vnames, MPI_Scatterv(vnames_toscatter, vnamessizes, vnamesdispls, MPI_SIGNED_CHAR,
numVar*STRING_SIZE,MPI_SIGNED_CHAR,Operation_P->Case.ScatterVariables.from,MPI_C vnames, numVar * STRING_SIZE, MPI_SIGNED_CHAR,
OMM_WORLD); Operation_P->Case.ScatterVariables.from, MPI_COMM_WORLD);
if (commrank==Operation_P->Case.ScatterVariables.from) { if(commrank == Operation_P->Case.ScatterVariables.from) {
delete [] vnames_toscatter; delete[] vnames_toscatter;
delete [] vnamessizes; delete[] vnamessizes;
delete [] vnamesdispls; delete[] vnamesdispls;
} }
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
// Scatter vector vvals_toscatter from root to vectors vvals in each rank ... // Scatter vector vvals_toscatter from root to vectors vvals in each rank ...
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
double *vvals; double *vvals;
int *vvalssizes, *vvalsdispls; int *vvalssizes, *vvalsdispls;
vvals = new double[numVar*NBR_MAX_HARMONIC * MAX_DIM]; vvals = new double[numVar * NBR_MAX_HARMONIC * MAX_DIM];
if (commrank==Operation_P->Case.ScatterVariables.from) { if(commrank == Operation_P->Case.ScatterVariables.from) {
vvalssizes = new int[commsize]; vvalssizes = new int[commsize];
vvalsdispls = new int[commsize]; vvalsdispls = new int[commsize];
for(unsigned int i = 0; i < commsize; i++) for(unsigned int i = 0; i < commsize; i++)
vvalssizes[i]=vnumVar_toscatter[i]*NBR_MAX_HARMONIC * MAX_DIM; vvalssizes[i] = vnumVar_toscatter[i] * NBR_MAX_HARMONIC * MAX_DIM;
vvalsdispls[0]=0; vvalsdispls[0] = 0;
for(unsigned int i = 1; i < commsize; i++) for(unsigned int i = 1; i < commsize; i++)
vvalsdispls[i]=vvalsdispls[i-1]+vvalssizes[i-1]; vvalsdispls[i] = vvalsdispls[i - 1] + vvalssizes[i - 1];
} }
MPI_Scatterv(vvals_toscatter,vvalssizes, vvalsdispls, MPI_DOUBLE, vvals, numVa MPI_Scatterv(vvals_toscatter, vvalssizes, vvalsdispls, MPI_DOUBLE, vvals,
r*NBR_MAX_HARMONIC * MAX_DIM, MPI_DOUBLE, Operation_P->Case.ScatterVariables.fro numVar * NBR_MAX_HARMONIC * MAX_DIM, MPI_DOUBLE,
m, MPI_COMM_WORLD); //MPI_Comm comm) Operation_P->Case.ScatterVariables.from,
if (commrank==Operation_P->Case.ScatterVariables.from) { MPI_COMM_WORLD); // MPI_Comm comm)
delete [] vvals_toscatter; if(commrank == Operation_P->Case.ScatterVariables.from) {
delete [] vvalssizes; delete[] vvals_toscatter;
delete [] vvalsdispls; delete[] vvalssizes;
delete[] vvalsdispls;
} }
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
// Scatter vector vtypes_toscatter from root to vectors vtypes in each rank .. // Scatter vector vtypes_toscatter from root to vectors vtypes in each rank
. // ...
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
int *vtypes, *vtypesdispls; int *vtypes, *vtypesdispls;
//int *vtypessizes, = vnumVar_toscatter which is already known at this stage // int *vtypessizes, = vnumVar_toscatter which is already known at this stage
vtypes = new int[numVar]; vtypes = new int[numVar];
if (commrank==Operation_P->Case.ScatterVariables.from) { if(commrank == Operation_P->Case.ScatterVariables.from) {
//vtypessizes = new int[commsize]; // vtypessizes = new int[commsize];
vtypesdispls = new int[commsize]; vtypesdispls = new int[commsize];
//for(unsigned int i = 0; i < commsize; i++) // for(unsigned int i = 0; i < commsize; i++)
// vtypessizes[i]=vnumVar_toscatter[i]; // vtypessizes[i]=vnumVar_toscatter[i];
vtypesdispls[0]=0; vtypesdispls[0] = 0;
for(unsigned int i = 1; i < commsize; i++) for(unsigned int i = 1; i < commsize; i++)
vtypesdispls[i]=vtypesdispls[i-1]+vnumVar_toscatter[i-1]; vtypesdispls[i] = vtypesdispls[i - 1] + vnumVar_toscatter[i - 1];
} }
MPI_Scatterv(vtypes_toscatter, vnumVar_toscatter, vtypesdispls, MPI_INT, vtype MPI_Scatterv(vtypes_toscatter, vnumVar_toscatter, vtypesdispls, MPI_INT,
s, numVar, MPI_INT, Operation_P->Case.ScatterVariables.from, MPI_COMM_WORLD); vtypes, numVar, MPI_INT, Operation_P->Case.ScatterVariables.from,
MPI_COMM_WORLD);
if (commrank==Operation_P->Case.ScatterVariables.from) { if(commrank == Operation_P->Case.ScatterVariables.from) {
delete [] vtypes_toscatter; delete[] vtypes_toscatter;
delete [] vnumVar_toscatter; delete[] vnumVar_toscatter;
delete [] vtypesdispls; delete[] vtypesdispls;
} }
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
// In each rank, Update Variables from vectors vnames, vvals, vtypes obtained // In each rank, Update Variables from vectors vnames, vvals, vtypes obtained
after scattering // after scattering
////////////////////////////////////////////////////////////////////////////// /////// ////////////////////////////////////////////////////////////////////////////// ///////
for(unsigned int i = 0; i < numVar; i++){ for(unsigned int i = 0; i < numVar; i++) {
struct Value v; struct Value v;
v.Type = vtypes[i]; v.Type = vtypes[i];
for(unsigned int k = 0; k < NBR_MAX_HARMONIC * MAX_DIM; k++) for(unsigned int k = 0; k < NBR_MAX_HARMONIC * MAX_DIM; k++)
v.Val[k]=vvals[i*NBR_MAX_HARMONIC * MAX_DIM+k]; v.Val[k] = vvals[i * NBR_MAX_HARMONIC * MAX_DIM + k];
values[&(vnames[i*STRING_SIZE])]=v; values[&(vnames[i * STRING_SIZE])] = v;
} }
delete [] vnames; delete[] vnames;
delete [] vvals; delete[] vvals;
delete [] vtypes; delete[] vtypes;
return 0; return 0;
} }
int Operation_BroadcastVariables(struct Resolution *Resolution_P, int Operation_BroadcastVariables(struct Resolution *Resolution_P,
struct Operation *Operation_P, struct Operation *Operation_P,
struct DofData *DofData_P0, struct DofData *DofData_P0,
struct GeoData *GeoData_P0) struct GeoData *GeoData_P0)
{ {
int commsize = Message::GetCommSize(); int commsize = Message::GetCommSize();
int commrank = Message::GetCommRank(); int commrank = Message::GetCommRank();
Message::Info("BroadcastVariables: rank %d (size %d)", commrank, commsize); Message::Info("BroadcastVariables: rank %d (size %d)", commrank, commsize);
if (Operation_P->Case.BroadcastVariables.from<-1 || if(Operation_P->Case.BroadcastVariables.from < -1 ||
Operation_P->Case.BroadcastVariables.from>commsize-1){ Operation_P->Case.BroadcastVariables.from > commsize - 1) {
Message::Warning("BroadcastVariables: impossible to Broadcast from rank %d w Message::Warning("BroadcastVariables: impossible to Broadcast from rank %d "
hich does not exist ", "which does not exist ",
"=> BroadcastVariables is aborted", "=> BroadcastVariables is aborted",
Operation_P->Case.BroadcastVariables.from); Operation_P->Case.BroadcastVariables.from);
return 0; return 0;
} }
std::map<std::string, struct Value> &values = Get_AllValueSaved(); std::map<std::string, struct Value> &values = Get_AllValueSaved();
std::vector<std::string> names; std::vector<std::string> names;
for(unsigned int i = 0; i < List_Nbr(Operation_P->Case.BroadcastVariables.Name for(unsigned int i = 0;
s); i++){ i < List_Nbr(Operation_P->Case.BroadcastVariables.Names); i++) {
char *s; char *s;
List_Read(Operation_P->Case.BroadcastVariables.Names, i, &s); List_Read(Operation_P->Case.BroadcastVariables.Names, i, &s);
if(values.find(s) != values.end()) if(values.find(s) != values.end())
names.push_back(s); names.push_back(s);
else else
Message::Warning("BroadcastVariables: Unknown variable %s", s); Message::Warning("BroadcastVariables: Unknown variable %s", s);
} }
if(names.empty()){ if(names.empty()) {
for(std::map<std::string, struct Value>::iterator it = values.begin(); for(std::map<std::string, struct Value>::iterator it = values.begin();
it != values.end(); it++) it != values.end(); it++)
names.push_back(it->first); names.push_back(it->first);
} }
// TODO: this should probably be implemented using MPI_Allgatherv // TODO: this should probably be implemented using MPI_Allgatherv
//http://mpitutorial.com/tutorials/mpi-scatter-gather-and-allgather/ // http://mpitutorial.com/tutorials/mpi-scatter-gather-and-allgather/
//http://mpitutorial.com/tutorials/mpi-broadcast-and-collective-communication/ // http://mpitutorial.com/tutorials/mpi-broadcast-and-collective-communication
/
for(int rank = 0; rank < commsize; rank++){ for(int rank = 0; rank < commsize; rank++) {
if (Operation_P->Case.BroadcastVariables.from==-1 || if(Operation_P->Case.BroadcastVariables.from == -1 ||
rank==Operation_P->Case.BroadcastVariables.from ){ rank == Operation_P->Case.BroadcastVariables.from) {
if(rank == commrank){ if(rank == commrank) {
int numValues = names.size(); int numValues = names.size();
MPI_Bcast(&numValues, 1, MPI_INT, rank, MPI_COMM_WORLD); MPI_Bcast(&numValues, 1, MPI_INT, rank, MPI_COMM_WORLD);
for(unsigned int i = 0; i < names.size(); i++){ for(unsigned int i = 0; i < names.size(); i++) {
char key[STRING_SIZE]; char key[STRING_SIZE];
strncpy(key, names[i].c_str(), sizeof(key)); strncpy(key, names[i].c_str(), sizeof(key));
//Message::Warning("I am %d Sending %s from %d", commrank, key, rank); // Message::Warning("I am %d Sending %s from %d", commrank, key,
// rank);
MPI_Bcast(key, sizeof(key), MPI_SIGNED_CHAR, rank, MPI_COMM_WORLD); MPI_Bcast(key, sizeof(key), MPI_SIGNED_CHAR, rank, MPI_COMM_WORLD);
struct Value &v = values[key]; struct Value &v = values[key];
MPI_Bcast(&v.Type, 1, MPI_INT, rank, MPI_COMM_WORLD); MPI_Bcast(&v.Type, 1, MPI_INT, rank, MPI_COMM_WORLD);
MPI_Bcast(&v.Val, NBR_MAX_HARMONIC * MAX_DIM, MPI_DOUBLE, MPI_Bcast(&v.Val, NBR_MAX_HARMONIC * MAX_DIM, MPI_DOUBLE, rank,
rank, MPI_COMM_WORLD); MPI_COMM_WORLD);
} }
} }
else{ else {
int numValues; int numValues;
MPI_Bcast(&numValues, 1, MPI_INT, rank, MPI_COMM_WORLD); MPI_Bcast(&numValues, 1, MPI_INT, rank, MPI_COMM_WORLD);
for(unsigned int i = 0; i < numValues; i++){ for(unsigned int i = 0; i < numValues; i++) {
char key[STRING_SIZE]; char key[STRING_SIZE];
MPI_Bcast(key, sizeof(key), MPI_SIGNED_CHAR, rank, MPI_COMM_WORLD); MPI_Bcast(key, sizeof(key), MPI_SIGNED_CHAR, rank, MPI_COMM_WORLD);
//Message::Warning("I am %d Receving %s from %d", commrank, key, rank) // Message::Warning("I am %d Receving %s from %d", commrank, key,
; // rank);
struct Value v; struct Value v;
MPI_Bcast(&v.Type, 1, MPI_INT, rank, MPI_COMM_WORLD); MPI_Bcast(&v.Type, 1, MPI_INT, rank, MPI_COMM_WORLD);
MPI_Bcast(&v.Val, NBR_MAX_HARMONIC * MAX_DIM, MPI_DOUBLE, MPI_Bcast(&v.Val, NBR_MAX_HARMONIC * MAX_DIM, MPI_DOUBLE, rank,
rank, MPI_COMM_WORLD); MPI_COMM_WORLD);
values[key] = v; values[key] = v;
} }
} }
} }
} }
/* /*
if (0) { if (0) {
for(int rank = 0; rank < commsize; rank++){ for(int rank = 0; rank < commsize; rank++){
if (Operation_P->Case.BroadcastVariables.from==-1 || if (Operation_P->Case.BroadcastVariables.from==-1 ||
 End of changes. 124 change blocks. 
312 lines changed or deleted 320 lines changed or added
To the C++ Programs section of the getdp source changes report or the top of this page
Mainly generated by pkgdiff using rfcdiff
Home  |  About  |  Features  |  All  |  Newest  |  Dox  |  Diffs  |  RSS Feeds  |  Screenshots  |  Comments  |  Imprint  |  Privacy  |  HTTP(S)