openmpi  3.1.6
About: Open MPI is a high performance Message Passing Interface (MPI) library project combining technologies and resources from several other projects (FT-MPI, LA-MPI, LAM/MPI, and PACX-MPI) in order to build the best MPI library available. 3.x series.
  Fossies Dox: openmpi-3.1.6.tar.bz2  ("unofficial" and yet experimental doxygen-generated source code documentation)  

sharedfp_sm_file_open.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
3  * University Research and Technology
4  * Corporation. All rights reserved.
5  * Copyright (c) 2004-2017 The University of Tennessee and The University
6  * of Tennessee Research Foundation. All rights
7  * reserved.
8  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9  * University of Stuttgart. All rights reserved.
10  * Copyright (c) 2004-2005 The Regents of the University of California.
11  * All rights reserved.
12  * Copyright (c) 2013-2017 University of Houston. All rights reserved.
13  * Copyright (c) 2013 Intel, Inc. All rights reserved.
14  * Copyright (c) 2015 Research Organization for Information Science
15  * and Technology (RIST). All rights reserved.
16  * Copyright (c) 2015 Cisco Systems, Inc. All rights reserved.
17  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
18  * $COPYRIGHT$
19  *
20  * Additional copyrights may follow
21  *
22  * $HEADER$
23  */
24 
25 
26 #include "ompi_config.h"
27 
28 #if HAVE_LIBGEN_H
29 #include <libgen.h>
30 #endif
31 #if HAVE_SYS_STAT_H
32 #include <sys/stat.h>
33 #endif
34 
35 #include "sharedfp_sm.h"
36 
37 #include "mpi.h"
38 #include "ompi/constants.h"
39 #include "ompi/group/group.h"
40 #include "ompi/proc/proc.h"
43 
44 #include <semaphore.h>
45 #include <sys/mman.h>
46 #include <libgen.h>
47 #include <unistd.h>
48 
50  const char* filename,
51  int amode,
52  struct opal_info_t *info,
54 {
55  int err = OMPI_SUCCESS;
56  struct mca_sharedfp_base_data_t* sh;
58  mca_io_ompio_file_t * shfileHandle, *ompio_fh;
59  char * filename_basename;
60  char * sm_filename;
61  int sm_filename_length;
62  struct mca_sharedfp_sm_offset * sm_offset_ptr;
63  struct mca_sharedfp_sm_offset sm_offset;
64  mca_io_ompio_data_t *data;
65  int sm_fd;
66  int rank;
67  uint32_t comm_cid;
68  int int_pid;
69  pid_t my_pid;
70 
71  /*----------------------------------------------------*/
72  /*Open the same file again without shared file pointer*/
73  /*----------------------------------------------------*/
74  shfileHandle = (mca_io_ompio_file_t *)malloc(sizeof(mca_io_ompio_file_t));
75  if ( NULL == shfileHandle ) {
76  opal_output(0, "mca_sharedfp_sm_file_open: Error during memory allocation\n");
78  }
79  err = mca_common_ompio_file_open(comm,filename,amode,info,shfileHandle,false);
80  if ( OMPI_SUCCESS != err) {
81  opal_output(0, "mca_sharedfp_sm_file_open: Error during file open\n");
82  free (shfileHandle);
83  return err;
84  }
85  shfileHandle->f_fh = fh->f_fh;
87  ompio_fh = &data->ompio_fh;
88 
89  err = mca_common_ompio_set_view (shfileHandle,
90  ompio_fh->f_disp,
91  ompio_fh->f_etype,
92  ompio_fh->f_orig_filetype,
93  ompio_fh->f_datarep,
94  &(MPI_INFO_NULL->super));
95 
96  /*Memory is allocated here for the sh structure*/
99  "mca_sharedfp_sm_file_open: malloc f_sharedfp_ptr struct\n");
100  }
101 
102  sh = (struct mca_sharedfp_base_data_t*)malloc(sizeof(struct mca_sharedfp_base_data_t));
103  if ( NULL == sh ) {
104  opal_output(0, "mca_sharedfp_sm_file_open: Error, unable to malloc f_sharedfp_ptr struct\n");
105  free(shfileHandle);
107  }
108 
109  /*Populate the sh file structure based on the implementation*/
110  sh->sharedfh = shfileHandle; /* Shared file pointer*/
111  sh->global_offset = 0; /* Global Offset*/
112  sh->comm = comm; /* Communicator*/
114 
115  rank = ompi_comm_rank ( sh->comm );
116 
117  /*Open a shared memory segment which will hold the shared file pointer*/
118  if ( mca_sharedfp_sm_verbose ) {
120  "mca_sharedfp_sm_file_open: allocatge shared memory segment.\n");
121  }
122 
123 
124  sm_data = (struct mca_sharedfp_sm_data*) malloc ( sizeof(struct mca_sharedfp_sm_data));
125  if ( NULL == sm_data ){
126  opal_output(0, "mca_sharedfp_sm_file_open: Error, unable to malloc sm_data struct\n");
127  free(sh);
128  free(shfileHandle);
130  }
132 
133 
134  /* the shared memory segment is identified opening a file
135  ** and then mapping it to memory
136  ** For sharedfp we also want to put the file backed shared memory into the tmp directory
137  */
138  filename_basename = basename(filename);
139  /* format is "%s/%s_cid-%d-%d.sm", see below */
140  sm_filename_length = strlen(ompi_process_info.job_session_dir) + 1 + strlen(filename_basename) + 5 + (3*sizeof(uint32_t)+1) + 4;
141  sm_filename = (char*) malloc( sizeof(char) * sm_filename_length);
142  if (NULL == sm_filename) {
143  opal_output(0, "mca_sharedfp_sm_file_open: Error, unable to malloc sm_filename\n");
144  free(sm_data);
145  free(sh);
146  free(shfileHandle);
148  }
149 
150  comm_cid = ompi_comm_get_cid(comm);
151  if ( 0 == fh->f_rank ) {
152  my_pid = getpid();
153  int_pid = (int) my_pid;
154  }
155  err = comm->c_coll->coll_bcast (&int_pid, 1, MPI_INT, 0, comm, comm->c_coll->coll_bcast_module );
156  if ( OMPI_SUCCESS != err ) {
157  opal_output(0,"mca_sharedfp_sm_file_open: Error in bcast operation \n");
158  free(sm_filename);
159  free(sm_data);
160  free(sh);
161  return err;
162  }
163  snprintf(sm_filename, sm_filename_length, "%s/%s_cid-%d-%d.sm", ompi_process_info.job_session_dir,
164  filename_basename, comm_cid, int_pid);
165 
166  /* open shared memory file, initialize to 0, map into memory */
167  sm_fd = open(sm_filename, O_RDWR | O_CREAT,
168  S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
169  if ( sm_fd == -1){
170  /*error opening file*/
171  opal_output(0,"mca_sharedfp_sm_file_open: Error, unable to open file for mmap: %s\n",sm_filename);
172  free(sm_filename);
173  free(sm_data);
174  free(sh);
175  free(shfileHandle);
176  return OMPI_ERROR;
177  }
178 
180 
181  /*TODO: is it necessary to write to the file first?*/
182  if( 0 == rank ){
183  memset ( &sm_offset, 0, sizeof (struct mca_sharedfp_sm_offset ));
184  write ( sm_fd, &sm_offset, sizeof(struct mca_sharedfp_sm_offset));
185  }
186  comm->c_coll->coll_barrier (comm, comm->c_coll->coll_barrier_module );
187 
188  /*the file has been written to, now we can map*/
189  sm_offset_ptr = mmap(NULL, sizeof(struct mca_sharedfp_sm_offset), PROT_READ | PROT_WRITE,
190  MAP_SHARED, sm_fd, 0);
191 
192  close(sm_fd);
193 
194  if ( sm_offset_ptr==MAP_FAILED){
195  err = OMPI_ERROR;
196  opal_output(0, "mca_sharedfp_sm_file_open: Error, unable to mmap file: %s\n",sm_filename);
197  opal_output(0, "%s\n", strerror(errno));
198  free(sm_filename);
199  free(sm_data);
200  free(sh);
201  free(shfileHandle);
202  return OMPI_ERROR;
203  }
204 
205  /* Initialize semaphore so that is shared between processes. */
206  /* the semaphore is shared by keeping it in the shared memory segment */
207 
208 #if defined(HAVE_SEM_OPEN)
209 
210 #if defined (__APPLE__)
211  sm_data->sem_name = (char*) malloc( sizeof(char) * 32);
212  snprintf(sm_data->sem_name,31,"OMPIO_%s",filename_basename);
213 #else
214  sm_data->sem_name = (char*) malloc( sizeof(char) * 253);
215  snprintf(sm_data->sem_name,252,"OMPIO_%s",filename_basename);
216 #endif
217 
218  if( (sm_data->mutex = sem_open(sm_data->sem_name, O_CREAT, 0644, 1)) != SEM_FAILED ) {
219 #elif defined(HAVE_SEM_INIT)
221  if(sem_init(&sm_offset_ptr->mutex, 1, 1) != -1){
222 #endif
223  /*If opening was successful*/
224  /*Store the new file handle*/
226  /* Assign the sm_data to sh->selected_module_data*/
228  /*remember the shared file handle*/
229  fh->f_sharedfp_data = sh;
230 
231  /*write initial zero*/
232  if(rank==0){
233  MPI_Offset position=0;
234 
235  sem_wait(sm_data->mutex);
236  sm_offset_ptr->offset=position;
237  sem_post(sm_data->mutex);
238  }
239  }else{
240  free(sm_filename);
242  free(sh);
243  free(shfileHandle);
246  }
247 
249 
250 #if defined(HAVE_SEM_OPEN)
251  if ( 0 == rank ) {
252  sem_unlink ( sm_data->sem_name);
253  }
254 #endif
255 
256  return err;
257 }
258 
260 {
261  int err = OMPI_SUCCESS;
262  /*sharedfp data structure*/
263  struct mca_sharedfp_base_data_t *sh=NULL;
264  /*sharedfp sm module data structure*/
265  struct mca_sharedfp_sm_data * file_data=NULL;
266 
267  if( NULL == fh->f_sharedfp_data ){
268  if ( mca_sharedfp_sm_verbose ) {
270  "sharedfp_sm_file_close: shared file pointer structure not initialized\n");
271  }
272  return OMPI_SUCCESS;
273  }
274  sh = fh->f_sharedfp_data;
275 
276  /* Use an MPI Barrier in order to make sure that
277  * all processes are ready to release the
278  * shared file pointer resources
279  */
281 
282  file_data = (sm_data*)(sh->selected_module_data);
283  if (file_data) {
284  /*Close sm handle*/
285  if (file_data->sm_offset_ptr) {
286  /* destroy semaphore */
287 #if defined(HAVE_SEM_OPEN)
288  sem_close ( file_data->mutex);
289  free (file_data->sem_name);
290 #elif defined(HAVE_SEM_INIT)
291  sem_destroy(&file_data->sm_offset_ptr->mutex);
292 #endif
293  /*Release the shared memory segment.*/
294  munmap(file_data->sm_offset_ptr,sizeof(struct mca_sharedfp_sm_offset));
295  /*Q: Do we need to delete the file? */
296  remove(file_data->sm_filename);
297  }
298  /*free our sm data structure*/
299  if(file_data->sm_filename){
300  free(file_data->sm_filename);
301  }
302  free(file_data);
303  }
304 
305  /* Close the main file opened by this component*/
307 
308  /*free shared file pointer data struct*/
309  free(sh);
310 
311  return err;
312 
313 }
mca_sharedfp_sm_data::mutex
sem_t * mutex
Definition: sharedfp_sm.h:123
mca_sharedfp_base_data_t::sharedfh
struct mca_io_ompio_file_t * sharedfh
Definition: sharedfp.h:226
mca_io_ompio_file_t
Back-end structure for MPI_File.
Definition: io_ompio.h:194
munmap
munmap(sm_offset_ptr, sizeof(struct mca_sharedfp_sm_offset))
ompi_file_t::f_io_selected_data
struct mca_io_base_file_t * f_io_selected_data
Allow the selected module to cache data on the file.
Definition: file.h:94
info
static output_desc_t info[64]
Definition: output.c:114
mca_sharedfp_base_data_t::selected_module_data
void * selected_module_data
Definition: sharedfp.h:232
OMPI_ERR_OUT_OF_RESOURCE
@ OMPI_ERR_OUT_OF_RESOURCE
Definition: constants.h:37
mca_common_ompio_set_view
int mca_common_ompio_set_view(mca_io_ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE disp, ompi_datatype_t *etype, ompi_datatype_t *filetype, const char *datarep, opal_info_t *info)
Definition: common_ompio_file_view.c:56
mca_coll_base_comm_coll_t::coll_barrier_module
mca_coll_base_module_2_2_0_t * coll_barrier_module
Definition: coll.h:523
ompi_communicator_t::c_coll
mca_coll_base_comm_coll_t * c_coll
Definition: communicator.h:189
mca_coll_base_comm_coll_t::coll_bcast_module
mca_coll_base_module_2_2_0_t * coll_bcast_module
Definition: coll.h:525
mca_sharedfp_sm_file_open
int mca_sharedfp_sm_file_open(struct ompi_communicator_t *comm, const char *filename, int amode, struct opal_info_t *info, mca_io_ompio_file_t *fh)
Definition: sharedfp_sm_file_open.c:49
mca_io_ompio_file_t::f_fh
struct ompi_file_t * f_fh
Definition: io_ompio.h:197
MAP_FAILED
#define MAP_FAILED
Definition: pshmem.h:26
mca_io_ompio_file_t::f_rank
int f_rank
Definition: io_ompio.h:200
mca_sharedfp_base_data_t
Definition: sharedfp.h:224
mca_common_ompio_file_open
int mca_common_ompio_file_open(ompi_communicator_t *comm, const char *filename, int amode, opal_info_t *info, mca_io_ompio_file_t *ompio_fh, bool use_sharedfp)
Definition: common_ompio_file_open.c:50
OMPI_SUCCESS
@ OMPI_SUCCESS
Definition: constants.h:35
ompi_communicator_t
Definition: communicator.h:134
sharedfp_sm.h
proc.h
ompi_sharedfp_base_framework
mca_base_framework_t ompi_sharedfp_base_framework
err
err
Definition: sharedfp_sm_file_open.c:245
constants.h
mca_sharedfp_sm_verbose
int mca_sharedfp_sm_verbose
Definition: sharedfp_sm_component.c:42
mca_io_ompio_file_t::f_etype
ompi_datatype_t * f_etype
Definition: io_ompio.h:239
mca_sharedfp_sm_data::sm_offset_ptr
struct mca_sharedfp_sm_offset * sm_offset_ptr
Definition: sharedfp_sm.h:117
mca_sharedfp_sm_data::sem_name
char * sem_name
Definition: sharedfp_sm.h:124
snprintf
#define snprintf
Definition: pmix_config_bottom.h:428
mca_coll_base_comm_coll_t::coll_barrier
mca_coll_base_module_barrier_fn_t coll_barrier
Definition: coll.h:522
mca_sharedfp_base_data_t::comm
struct ompi_communicator_t * comm
Definition: sharedfp.h:228
mca_sharedfp_sm_offset::offset
long long offset
Definition: sharedfp_sm.h:109
opal_output
void opal_output(int output_id, const char *format,...)
Definition: output.c:367
sharedfp.h
mca_sharedfp_sm_offset::mutex
sem_t mutex
Definition: sharedfp_sm.h:108
mca_common_ompio_file_close
int mca_common_ompio_file_close(mca_io_ompio_file_t *ompio_fh)
Definition: common_ompio_file_open.c:240
ompi_comm_get_cid
static uint32_t ompi_comm_get_cid(ompi_communicator_t *comm)
Context ID for the communicator, suitable for passing to ompi_comm_lookup for getting the communicato...
Definition: communicator.h:369
mca_sharedfp_sm_file_close
int mca_sharedfp_sm_file_close(mca_io_ompio_file_t *fh)
Definition: sharedfp_sm_file_open.c:259
base.h
ompi_process_info
#define ompi_process_info
Definition: rte_orte.h:84
mca_io_ompio_file_t::f_sharedfp_data
void * f_sharedfp_data
Definition: io_ompio.h:223
group.h
mca_base_framework_t::framework_output
int framework_output
Opal output for this framework (or -1)
Definition: mca_base_framework.h:154
ompi_comm_rank
static int ompi_comm_rank(ompi_communicator_t *comm)
rank w/in the communicator
Definition: communicator.h:343
OMPI_ERROR
@ OMPI_ERROR
Definition: constants.h:36
free
free(sm_data)
mca_coll_base_comm_coll_t::coll_bcast
mca_coll_base_module_bcast_fn_t coll_bcast
Definition: coll.h:524
mca_io_ompio_data_t
Definition: io_ompio.h:285
mca_io_ompio_file_t::f_disp
OMPI_MPI_OFFSET_TYPE f_disp
Definition: io_ompio.h:199
mca_sharedfp_sm_offset
Definition: sharedfp_sm.h:107
mca_io_ompio_file_t::f_orig_filetype
ompi_datatype_t * f_orig_filetype
Definition: io_ompio.h:241
mca_io_ompio_file_t::f_datarep
char * f_datarep
Definition: io_ompio.h:206
MPI_Offset
long long MPI_Offset
Definition: mpio.h:52
ompi_config.h
mca_sharedfp_sm_data::sm_filename
char * sm_filename
Definition: sharedfp_sm.h:119
filename
static char * filename
Definition: sync.c:26
mca_sharedfp_sm_data
Definition: sharedfp_sm.h:116
opal_info_t
Definition: info.h:40
NULL
#define NULL
Copyright (C) 2000-2004 by Etnus, LLC.
Definition: ompi_msgq_dll.c:136
mca_sharedfp_base_data_t::global_offset
OMPI_MPI_OFFSET_TYPE global_offset
Definition: sharedfp.h:227
sm_data
struct mca_sharedfp_sm_data sm_data
Definition: sharedfp_sm.h:127