irods  4.2.8
About: iRODS (the integrated Rule Oriented Data System) is a distributed data-management system for creating data grids, digital libraries, persistent archives, and real-time data systems.
  Fossies Dox: irods-4.2.8.tar.gz  ("unofficial" and yet experimental doxygen-generated source code documentation)  

putUtil.cpp
Go to the documentation of this file.
1 
3 #ifndef windows_platform
4 #include <sys/time.h>
5 #endif
6 #include "rodsPath.h"
7 #include "rodsErrorTable.h"
8 #include "rodsLog.h"
9 #include "putUtil.h"
10 #include "miscUtil.h"
11 #include "rcPortalOpr.h"
12 #include "checksum.hpp"
13 #include "rcGlobalExtern.h"
14 #include <string>
15 #include <boost/filesystem.hpp>
17 #include "irods_path_recursion.hpp"
18 #include "irods_exception.hpp"
19 #include "irods_random.hpp"
20 #include "irods_log.hpp"
21 
22 #include "sockComm.h"
23 #include <boost/filesystem/operations.hpp>
24 #include <boost/lexical_cast.hpp>
25 #include <boost/filesystem/convenience.hpp>
26 
27 
28 /* checkStateForResume - check the state for resume operation
29  * return 0 - skip
30  * return 1 - resume
31  */
32 
33 int
34 chkStateForResume( rcComm_t * conn, rodsRestart_t * rodsRestart,
35  char * targPath, rodsArguments_t * rodsArgs, objType_t objType,
36  keyValPair_t * condInput, int deleteFlag ) {
37  int status;
38 
39  if ( rodsRestart->restartState & MATCHED_RESTART_COLL ) {
40  if ( rodsRestart->curCnt > rodsRestart->doneCnt ) {
42  "chkStateForResume:Restart failed.curCnt %d>doneCnt %d,path %s",
43  rodsRestart->curCnt, rodsRestart->doneCnt, targPath );
44  return RESTART_OPR_FAILED;
45  }
46 
47  if ( rodsRestart->restartState & LAST_PATH_MATCHED ) {
48  if ( objType == DATA_OBJ_T || objType == LOCAL_FILE_T ) {
49  /* a file */
50  if ( rodsArgs->verbose == True ) {
51  printf( "***** RESUMING OPERATION ****\n" );
52  }
53  setStateForResume( conn, rodsRestart, targPath,
54  objType, condInput, deleteFlag );
55  }
56  status = 1;
57  }
58  else if ( strcmp( targPath, rodsRestart->lastDonePath ) == 0 ) {
59  /* will handle this with the next file */
60  rodsRestart->curCnt ++;
61  if ( rodsRestart->curCnt != rodsRestart->doneCnt ) {
63  "chkStateForResume:Restart failed.curCnt %d!=doneCnt %d,path %s",
64  rodsRestart->curCnt, rodsRestart->doneCnt, targPath );
65  return RESTART_OPR_FAILED;
66  }
67  rodsRestart->restartState |= LAST_PATH_MATCHED;
68  status = 0;
69  }
70  else if ( objType == DATA_OBJ_T || objType == LOCAL_FILE_T ) {
71  /* A file. no match - skip this */
72  if ( rodsArgs->verbose == True ) {
73  printf( " ---- Skip file %s ----\n", targPath );
74  }
75  rodsRestart->curCnt ++;
76  status = 0;
77  }
78  else {
79  /* collection - drill down and see */
80  status = 1;
81  }
82  }
83  else if ( rodsRestart->restartState & PATH_MATCHING ) {
84  /* the path does not match. skip */
85  status = 0;
86  }
87  else {
88  status = 1;
89  }
90 
91  return status;
92 }
93 
94 int
95 setStateForResume( rcComm_t * conn, rodsRestart_t * rodsRestart,
96  char * restartPath, objType_t objType, keyValPair_t * condInput,
97  int deleteFlag )
98 {
99  namespace fs = boost::filesystem;
100 
101  if ( restartPath != NULL && deleteFlag > 0 ) {
102  if ( objType == DATA_OBJ_T ) {
103  if ( ( condInput == NULL ||
104  getValByKey( condInput, FORCE_FLAG_KW ) == NULL ) &&
105  ( conn->fileRestart.info.status != FILE_RESTARTED ||
106  strcmp( conn->fileRestart.info.objPath, restartPath ) != 0 ) ) {
107  dataObjInp_t dataObjInp;
108  /* need to remove any partially completed file */
109  /* XXXXX may not be enough for bulk put */
110  memset( &dataObjInp, 0, sizeof( dataObjInp ) );
111  addKeyVal( &dataObjInp.condInput, FORCE_FLAG_KW, "" );
112  rstrcpy( dataObjInp.objPath, restartPath, MAX_NAME_LEN );
113  int status = rcDataObjUnlink( conn, & dataObjInp );
114  if ( status < 0 ) {
115  std::string notice = std::string( "rcDataObjUnlink returned with code: " );
116  notice.append( boost::lexical_cast<std::string>( status ) );
117  irods::log( LOG_NOTICE, notice );
118  }
119  clearKeyVal( &dataObjInp.condInput );
120  }
121  }
122  else if ( objType == LOCAL_FILE_T ) {
123  if ( conn->fileRestart.info.status != FILE_RESTARTED ||
124  strcmp( conn->fileRestart.info.fileName, restartPath ) != 0 ) {
125  fs::path path( restartPath );
126  if ( fs::exists( path ) ) {
127  int status = fs::remove( path );
128  if ( status < 0 ) {
129  irods::log( ERROR( status, "boost:filesystem::remove() failed." ) );
130  }
131  }
132  }
133  }
134  else {
136  "setStateForResume: illegal objType %d for %s",
137  objType, restartPath );
138  }
139  }
140  rodsRestart->restartState = OPR_RESUMED; /* resumed opr */
141 
142  return 0;
143 }
144 
145 
146 int
147 setSessionTicket( rcComm_t *myConn, char *ticket ) {
148  ticketAdminInp_t ticketAdminInp;
149  int status;
150 
151  ticketAdminInp.arg1 = "session";
152  ticketAdminInp.arg2 = ticket;
153  ticketAdminInp.arg3 = "";
154  ticketAdminInp.arg4 = "";
155  ticketAdminInp.arg5 = "";
156  ticketAdminInp.arg6 = "";
157  status = rcTicketAdmin( myConn, &ticketAdminInp );
158  if ( status != 0 ) {
159  printf( "set ticket error %d \n", status );
160  }
161  return status;
162 }
163 
164 int
165 putUtil( rcComm_t **myConn, rodsEnv *myRodsEnv,
166  rodsArguments_t *myRodsArgs, rodsPathInp_t *rodsPathInp ) {
167  int i;
168  int status;
169  int savedStatus = 0;
170  rodsPath_t *targPath = 0;
171  dataObjInp_t dataObjOprInp;
172  bulkOprInp_t bulkOprInp;
173  rodsRestart_t rodsRestart;
174  rcComm_t *conn = *myConn;
175 
176  if ( rodsPathInp == NULL ) {
177  return USER__NULL_INPUT_ERR;
178  }
179 
180  if ( myRodsArgs->ticket == True ) {
181  if ( myRodsArgs->ticketString == NULL ) {
183  "initCondForPut: NULL ticketString error" );
184  return USER__NULL_INPUT_ERR;
185  }
186  else {
187  setSessionTicket( conn, myRodsArgs->ticketString );
188  }
189  }
190 
191  status = initCondForPut( conn, myRodsEnv, myRodsArgs, &dataObjOprInp,
192  &bulkOprInp, &rodsRestart );
193  if ( status < 0 ) {
194  return status;
195  }
196 
197  // Issue 4006: disallow mixed files and directory sources with the
198  // recursive (-r) option.
199  status = irods::disallow_file_dir_mix_on_command_line(myRodsArgs, rodsPathInp );
200  if ( status < 0 ) {
201  return status;
202  }
203 
204  if ( rodsPathInp->resolved == False ) {
205  status = resolveRodsTarget( conn, rodsPathInp, PUT_OPR );
206  if ( status < 0 ) {
208  "putUtil: resolveRodsTarget error, status = %d", status );
209  return status;
210  }
211  rodsPathInp->resolved = True;
212  }
213 
214  // Issue 3988: Scan all physical source directories for loops before
215  // doing any actual file transfers.
216  irods::recursion_map_t pathmap;
217 
218  status = irods::file_system_sanity_check( pathmap, myRodsArgs, rodsPathInp );
219  if (status < 0) {
220  return status;
221  }
222 
223  /* initialize the progress struct */
224  if ( gGuiProgressCB != NULL ) {
225  bzero( &conn->operProgress, sizeof( conn->operProgress ) );
226  for ( i = 0; i < rodsPathInp->numSrc; i++ ) {
227  targPath = &rodsPathInp->targPath[i];
228  if ( targPath->objType == DATA_OBJ_T ) {
229  conn->operProgress.totalNumFiles++;
230  if ( rodsPathInp->srcPath[i].size > 0 ) {
231  conn->operProgress.totalFileSize +=
232  rodsPathInp->srcPath[i].size;
233  }
234  }
235  else {
236  // This function does its own try/catch of irods::exception
237  // so just check for errors
238  status = getDirSizeForProgStat( myRodsArgs, rodsPathInp->srcPath[i].outPath, &conn->operProgress );
239  if (status < 0)
240  {
241  return status;
242  }
243  }
244  }
245  }
246 
247  if ( conn->fileRestart.flags == FILE_RESTART_ON ) {
248  fileRestartInfo_t *info;
249  status = readLfRestartFile( conn->fileRestart.infoFile, &info );
250  if ( status >= 0 ) {
251  status = lfRestartPutWithInfo( conn, info );
252  if ( status >= 0 ) {
253  /* save info so we know what got restarted */
254  rstrcpy( conn->fileRestart.info.objPath, info->objPath,
255  MAX_NAME_LEN );
257  printf( "%s was restarted successfully\n",
258  conn->fileRestart.info.objPath );
259  unlink( conn->fileRestart.infoFile );
260  }
261  free( info );
262  }
263  }
264  for ( i = 0; i < rodsPathInp->numSrc; i++ ) {
265  targPath = &rodsPathInp->targPath[i];
266 
267  if ( targPath->objType == DATA_OBJ_T ) {
268 
269  try {
270  if (! irods::is_path_valid_for_recursion(myRodsArgs, rodsPathInp->srcPath[i].outPath) )
271  {
272  continue;
273  }
274  } catch ( const irods::exception& _e ) {
276  return USER_INPUT_PATH_ERR;
277  }
278 
279  dataObjOprInp.createMode = rodsPathInp->srcPath[i].objMode;
280  status = putFileUtil( conn, rodsPathInp->srcPath[i].outPath,
281  targPath->outPath, rodsPathInp->srcPath[i].size,
282  myRodsArgs, &dataObjOprInp );
283  }
284  else if ( targPath->objType == COLL_OBJ_T ) {
285 
286  // send recursive flag in case the resource plugin needs it
287  addKeyVal(&dataObjOprInp.condInput, RECURSIVE_OPR__KW, "1"); // value doesn't matter, other than for testing
288 
289  setStateForRestart( &rodsRestart, targPath, myRodsArgs );
290  if ( myRodsArgs->bulk == True ) {
291  status = bulkPutDirUtil( myConn,
292  rodsPathInp->srcPath[i].outPath, targPath->outPath,
293  myRodsEnv, myRodsArgs, &dataObjOprInp, &bulkOprInp,
294  &rodsRestart );
295  }
296  else {
297  status = putDirUtil( myConn, rodsPathInp->srcPath[i].outPath,
298  targPath->outPath, myRodsEnv, myRodsArgs, &dataObjOprInp,
299  &bulkOprInp, &rodsRestart, NULL );
301  {
302  return status;
303  }
304  }
305  }
306  else
307  {
308  /* should not be here */
310  "putUtil: invalid put dest objType %d for %s",
311  targPath->objType, targPath->outPath );
312  return USER_INPUT_PATH_ERR;
313  }
314  /* XXXX may need to return a global status */
315  if ( status < 0 ) {
317  "putUtil: put error for %s, status = %d",
318  targPath->outPath, status );
319  savedStatus = status;
320  break;
321  }
322  }
323 
324  if ( rodsRestart.fd > 0 ) {
325  close( rodsRestart.fd );
326  }
327 
328  if ( savedStatus < 0 ) {
329  status = savedStatus;
330  }
331  else if ( status == CAT_NO_ROWS_FOUND ) {
332  status = 0;
333  }
334  if ( status < 0 && myRodsArgs->retries == True ) {
335  int reconnFlag;
336  /* this is recursive. Only do it the first time */
337  myRodsArgs->retries = False;
338  if ( myRodsArgs->reconnect == True ) {
339  reconnFlag = RECONN_TIMEOUT;
340  }
341  else {
342  reconnFlag = NO_RECONN;
343  }
344  while ( myRodsArgs->retriesValue > 0 ) {
345  rErrMsg_t errMsg;
346  bzero( &errMsg, sizeof( errMsg ) );
347  status = rcReconnect( myConn, myRodsEnv->rodsHost, myRodsEnv,
348  reconnFlag );
349  if ( status < 0 ) {
351  "putUtil: rcReconnect error for %s", targPath->outPath );
352  return status;
353  }
354  status = putUtil( myConn, myRodsEnv, myRodsArgs, rodsPathInp );
355  if ( status >= 0 ) {
356  printf( "Retry put successful\n" );
357  break;
358  }
359  else {
361  "putUtil: retry putUtil error" );
362  }
363  myRodsArgs->retriesValue--;
364  }
365  }
366  return status;
367 }
368 
369 int
370 putFileUtil( rcComm_t *conn, char *srcPath, char *targPath, rodsLong_t srcSize,
371  rodsArguments_t *rodsArgs, dataObjInp_t *dataObjOprInp ) {
372  int status;
373  struct timeval startTime, endTime;
374 
375  if ( srcPath == NULL || targPath == NULL ) {
377  "putFileUtil: NULL srcPath or targPath input" );
378  return USER__NULL_INPUT_ERR;
379  }
380 
381  if ( conn->fileRestart.info.status == FILE_RESTARTED &&
382  strcmp( conn->fileRestart.info.objPath, targPath ) == 0 ) {
383  /* it was restarted */
385  return 0;
386  }
387 
388  if ( rodsArgs->verbose == True ) {
389  ( void ) gettimeofday( &startTime, ( struct timezone * )0 );
390  }
391 
392  if ( gGuiProgressCB != NULL ) {
393  rstrcpy( conn->operProgress.curFileName, srcPath, MAX_NAME_LEN );
394  conn->operProgress.curFileSize = srcSize;
395  conn->operProgress.curFileSizeDone = 0;
396  conn->operProgress.flag = 0;
397  gGuiProgressCB( &conn->operProgress );
398  }
399 
400  /* have to take care of checksum here since it needs to be recalculated */
401  if ( rodsArgs->checksum == True ) {
402  // set the expected flag to indicate that we want a server-side
403  // checksum computed and stored in the catalog
404  addKeyVal( &dataObjOprInp->condInput, REG_CHKSUM_KW, "" );
405 
406  }
407  else if ( rodsArgs->verifyChecksum == True ) {
408  rodsEnv env;
409  int ret = getRodsEnv( &env );
410  if ( ret < 0 ) {
411  rodsLog(
412  LOG_ERROR,
413  "putFileUtil - failed to capture rods env %d",
414  ret );
415  return ret;
416  }
417 
418  status = rcChksumLocFile( srcPath,
420  &dataObjOprInp->condInput,
421  env.rodsDefaultHashScheme );
422  if ( status < 0 ) {
424  "putFileUtil: rcChksumLocFile error for %s, status = %d",
425  srcPath, status );
426  return status;
427  }
428  }
429  if ( strlen( targPath ) >= MAX_PATH_ALLOWED - 1 ) {
430  return USER_PATH_EXCEEDS_MAX;
431  }
432  rstrcpy( dataObjOprInp->objPath, targPath, MAX_NAME_LEN );
433  dataObjOprInp->dataSize = srcSize;
434  status = rcDataObjPut( conn, dataObjOprInp, srcPath );
435 
436  if ( status >= 0 ) {
437  if ( rodsArgs->verbose == True ) {
438  ( void ) gettimeofday( &endTime, ( struct timezone * )0 );
439  printTiming( conn, dataObjOprInp->objPath, srcSize, srcPath,
440  &startTime, &endTime );
441  }
442  if ( gGuiProgressCB != NULL ) {
444  conn->operProgress.totalFileSizeDone += srcSize;
445  }
446  }
447 
448  return status;
449 }
450 
451 int
452 initCondForPut( rcComm_t *conn, rodsEnv *myRodsEnv, rodsArguments_t *rodsArgs,
453  dataObjInp_t *dataObjOprInp, bulkOprInp_t *bulkOprInp,
454  rodsRestart_t *rodsRestart )
455 {
456  namespace fs = boost::filesystem;
457 
458  char *tmpStr;
459 
460  if ( rodsArgs == NULL ) { // JMC cppcheck - nullptr
461  rodsLog( LOG_ERROR, "initCondForPut :: NULL rodsArgs" );
462  return -1;
463  }
464 
465  if ( dataObjOprInp == NULL ) {
467  "initCondForPut: NULL dataObjOprInp input" );
468  return USER__NULL_INPUT_ERR;
469  }
470 
471  memset( dataObjOprInp, 0, sizeof( dataObjInp_t ) );
472 
473  if ( rodsArgs->kv_pass ) {
474  addKeyVal(
475  &dataObjOprInp->condInput,
477  rodsArgs->kv_pass_string );
478 
479  }
480 
481  if ( rodsArgs->bulk == True ) {
482  if ( bulkOprInp == NULL ) {
484  "initCondForPut: NULL bulkOprInp input" );
485  return USER__NULL_INPUT_ERR;
486  }
487  bzero( bulkOprInp, sizeof( bulkOprInp_t ) );
488  if ( rodsArgs->checksum == True ) {
489  addKeyVal( &bulkOprInp->condInput, REG_CHKSUM_KW, "" );
490  }
491  else if ( rodsArgs->verifyChecksum == True ) {
492  addKeyVal( &bulkOprInp->condInput, VERIFY_CHKSUM_KW, "" );
493  }
494  initAttriArrayOfBulkOprInp( bulkOprInp );
495  }
496 
497  dataObjOprInp->oprType = PUT_OPR;
498 
499  if ( rodsArgs->all == True ) {
500  addKeyVal( &dataObjOprInp->condInput, ALL_KW, "" );
501  }
502 
503  if ( rodsArgs->dataType == True ) {
504  if ( rodsArgs->dataTypeString == NULL ) {
505  addKeyVal( &dataObjOprInp->condInput, DATA_TYPE_KW, "generic" );
506  }
507  else {
508  if ( strcmp( rodsArgs->dataTypeString, "t" ) == 0 ||
509  strcmp( rodsArgs->dataTypeString, "tar" ) == 0 ) {
510  addKeyVal( &dataObjOprInp->condInput, DATA_TYPE_KW,
511  "tar file" );
512  }
513  else {
514  addKeyVal( &dataObjOprInp->condInput, DATA_TYPE_KW,
515  rodsArgs->dataTypeString );
516  }
517  }
518  }
519  else {
520  addKeyVal( &dataObjOprInp->condInput, DATA_TYPE_KW, "generic" );
521  }
522 
523  if ( rodsArgs->force == True ) {
524  addKeyVal( &dataObjOprInp->condInput, FORCE_FLAG_KW, "" );
525  if ( rodsArgs->bulk == True ) {
526  addKeyVal( &bulkOprInp->condInput, FORCE_FLAG_KW, "" );
527  }
528  }
529 
530 #ifdef windows_platform
531  dataObjOprInp->numThreads = NO_THREADING;
532 #else
533  if ( rodsArgs->number == True ) {
534  if ( rodsArgs->numberValue == 0 ) {
535  dataObjOprInp->numThreads = NO_THREADING;
536  }
537  else {
538  dataObjOprInp->numThreads = rodsArgs->numberValue;
539  }
540  }
541 #endif
542 
543  if ( rodsArgs->physicalPath == True ) {
544  if ( rodsArgs->physicalPathString == NULL ) {
546  "initCondForPut: NULL physicalPathString error" );
547  return USER__NULL_INPUT_ERR;
548  }
549  else {
550  fs::path slash( "/" );
551  std::string preferred_slash = slash.make_preferred().native();
552  if ( preferred_slash[0] != rodsArgs->physicalPathString[0] ) {
553  rodsLog(
554  LOG_ERROR,
555  "initCondForPut: physical path [%s] must be absolute, not relative",
556  rodsArgs->physicalPathString );
557  return USER_INPUT_PATH_ERR;
558  }
559 
560  addKeyVal( &dataObjOprInp->condInput, FILE_PATH_KW,
561  rodsArgs->physicalPathString );
562  }
563  }
564 
565  if ( rodsArgs->resource == True ) {
566  if ( rodsArgs->resourceString == NULL ) {
568  "initCondForPut: NULL resourceString error" );
569  return USER__NULL_INPUT_ERR;
570  }
571  else {
572  addKeyVal( &dataObjOprInp->condInput, DEST_RESC_NAME_KW,
573  rodsArgs->resourceString );
574  if ( rodsArgs->bulk == True ) {
575  addKeyVal( &bulkOprInp->condInput, DEST_RESC_NAME_KW,
576  rodsArgs->resourceString );
577  }
578 
579  }
580  }
581  else if ( myRodsEnv != NULL && strlen( myRodsEnv->rodsDefResource ) > 0 ) {
582  /* use rodsDefResource but set the DEF_RESC_NAME_KW instead.
583  * Used by dataObjCreate. Will only make a new replica only if
584  * DEST_RESC_NAME_KW is set */
585  addKeyVal( &dataObjOprInp->condInput, DEF_RESC_NAME_KW,
586  myRodsEnv->rodsDefResource );
587  if ( rodsArgs->bulk == True ) {
588  addKeyVal( &bulkOprInp->condInput, DEF_RESC_NAME_KW,
589  myRodsEnv->rodsDefResource );
590  }
591  }
592 
593  if ( rodsArgs->replNum == True ) {
594  addKeyVal( &dataObjOprInp->condInput, REPL_NUM_KW,
595  rodsArgs->replNumValue );
596  }
597 
598  if ( rodsArgs->purgeCache == True ) { // JMC - backport 4537
599  addKeyVal( &dataObjOprInp->condInput, PURGE_CACHE_KW, "" );
600  }
601 
602 
603  if ( rodsArgs->rbudp == True ) {
604  /* use -Q for rbudp transfer */
605  addKeyVal( &dataObjOprInp->condInput, RBUDP_TRANSFER_KW, "" );
606  }
607 
608  if ( rodsArgs->veryVerbose == True ) {
609  addKeyVal( &dataObjOprInp->condInput, VERY_VERBOSE_KW, "" );
610  }
611 
612  if ( ( tmpStr = getenv( RBUDP_SEND_RATE_KW ) ) != NULL ) {
613  addKeyVal( &dataObjOprInp->condInput, RBUDP_SEND_RATE_KW, tmpStr );
614  }
615 
616  if ( ( tmpStr = getenv( RBUDP_PACK_SIZE_KW ) ) != NULL ) {
617  addKeyVal( &dataObjOprInp->condInput, RBUDP_PACK_SIZE_KW, tmpStr );
618  }
619 
620  memset( rodsRestart, 0, sizeof( rodsRestart_t ) );
621  if ( rodsArgs->restart == True ) {
622  int status;
623  status = openRestartFile( rodsArgs->restartFileString, rodsRestart );
624  if ( status < 0 ) {
626  "initCondForPut: openRestartFile of %s error",
627  rodsArgs->restartFileString );
628  return status;
629  }
630  if ( rodsRestart->doneCnt == 0 ) {
631  /* brand new */
632  if ( rodsArgs->bulk == True ) {
633  rstrcpy( rodsRestart->oprType, BULK_OPR_KW, NAME_LEN );
634  }
635  else {
636  rstrcpy( rodsRestart->oprType, NON_BULK_OPR_KW, NAME_LEN );
637  }
638  }
639  else {
640  if ( rodsArgs->bulk == True ) {
641  if ( strcmp( rodsRestart->oprType, BULK_OPR_KW ) != 0 ) {
643  "initCondForPut: -b cannot be used to restart this iput" );
645  }
646  }
647  else {
648  if ( strcmp( rodsRestart->oprType, NON_BULK_OPR_KW ) != 0 ) {
650  "initCondForPut: -b must be used to restart this iput" );
652  }
653  }
654  }
655  }
656  if ( rodsArgs->retries == True && rodsArgs->restart == False &&
657  rodsArgs->lfrestart == False ) {
658 
660  "initCondForPut: --retries must be used with -X or --lfrestart option" );
661 
662  return USER_INPUT_OPTION_ERR;
663  }
664 
665  /* Not needed - dataObjOprInp->createMode = 0700; */
666  /* mmap in rbudp needs O_RDWR */
667  dataObjOprInp->openFlags = O_RDWR;
668 
669  if ( rodsArgs->lfrestart == True ) {
670  if ( rodsArgs->bulk == True ) {
672  "initCondForPut: --lfrestart cannot be used with -b option" );
673  }
674  else if ( rodsArgs->rbudp == True ) {
676  "initCondForPut: --lfrestart cannot be used with -Q option" );
677  }
678  else {
681  MAX_NAME_LEN );
682  }
683  }
684 
685  if ( rodsArgs->wlock == True ) { // JMC - backport 4604
686  addKeyVal( &dataObjOprInp->condInput, LOCK_TYPE_KW, WRITE_LOCK_TYPE );
687  }
688  if ( rodsArgs->rlock == True ) { // JMC - backport 4612
689  rodsLog( LOG_ERROR, "initCondForPut: --rlock not supported, changing it to --wlock" );
690  addKeyVal( &dataObjOprInp->condInput, LOCK_TYPE_KW, WRITE_LOCK_TYPE );
691  }
692 
693  if ( rodsArgs->metadata_string != NULL && strlen( rodsArgs->metadata_string ) != 0 ) {
694  addKeyVal( &dataObjOprInp->condInput, METADATA_INCLUDED_KW, rodsArgs->metadata_string );
695  }
696 
697  if ( rodsArgs->acl_string != NULL && strlen( rodsArgs->acl_string ) != 0 ) {
698  addKeyVal( &dataObjOprInp->condInput, ACL_INCLUDED_KW, rodsArgs->acl_string );
699  }
700 
701  return 0;
702 }
703 
704 int
705 putDirUtil( rcComm_t **myConn, char *srcDir, char *targColl,
706  rodsEnv *myRodsEnv, rodsArguments_t *rodsArgs, dataObjInp_t *dataObjOprInp,
707  bulkOprInp_t *bulkOprInp, rodsRestart_t *rodsRestart,
708  bulkOprInfo_t *bulkOprInfo )
709 {
710  namespace fs = boost::filesystem;
711 
712  char srcChildPath[MAX_NAME_LEN], targChildPath[MAX_NAME_LEN];
713  int status = 0;
714 
715  if ( srcDir == NULL || targColl == NULL ) {
716  rodsLog( LOG_ERROR, "putDirUtil: NULL srcDir or targColl input" );
717  return USER__NULL_INPUT_ERR;
718  }
719 
720  try {
721  if (! irods::is_path_valid_for_recursion(rodsArgs, srcDir))
722  {
723  return 0;
724  }
725  } catch ( const irods::exception& _e ) {
727  return USER_INPUT_PATH_ERR;
728  }
729 
730  if ( rodsArgs->recursive != True ) {
732  "putDirUtil: -r option must be used for putting %s directory",
733  srcDir );
734  return USER_INPUT_OPTION_ERR;
735  }
736 
737  if ( rodsArgs->redirectConn == True && rodsArgs->force != True ) {
738  int reconnFlag;
739  if ( rodsArgs->reconnect == True ) {
740  reconnFlag = RECONN_TIMEOUT;
741  }
742  else {
743  reconnFlag = NO_RECONN;
744  }
745  /* reconnect to the resource server */
746  rstrcpy( dataObjOprInp->objPath, targColl, MAX_NAME_LEN );
747  redirectConnToRescSvr( myConn, dataObjOprInp, myRodsEnv, reconnFlag );
748  rodsArgs->redirectConn = 0; /* only do it once */
749  }
750 
751  rcComm_t *conn = *myConn;
752  fs::path srcDirPath( srcDir );
753  if ( !exists( srcDirPath ) || !is_directory( srcDirPath ) ) {
755  "putDirUtil: opendir local dir error for %s, errno = %d\n",
756  srcDir, errno );
757  return USER_INPUT_PATH_ERR;
758  }
759 
760  if ( rodsArgs->verbose == True ) {
761  fprintf( stdout, "C- %s:\n", targColl );
762  }
763 
764  int bulkFlag = NON_BULK_OPR;
765 
766  if ( bulkOprInfo != NULL ) {
767  bulkFlag = bulkOprInfo->flags;
768  }
769 
770  int savedStatus = 0;
771  boost::system::error_code ec;
772  fs::directory_iterator it(srcDirPath, ec), end;
773 
774  if (ec && it == end) {
775  rodsLog( LOG_ERROR, "Error ecountered when processing directory %s for put: %s", srcDirPath.c_str(), ec.message().c_str());
776  savedStatus = ec.value();
777  } else {
778  for ( ; it != end; it.increment(ec) ) {
779  if (ec) {
780  rodsLog( LOG_ERROR, "Error ecountered when processing directory %s for put: %s", srcDirPath.c_str(), ec.message().c_str());
781  continue;
782  }
783 
784  fs::path p = it->path();
785  snprintf( srcChildPath, MAX_NAME_LEN, "%s", p.c_str() );
786 
787  try {
788  if (! irods::is_path_valid_for_recursion(rodsArgs, srcChildPath) )
789  {
790  continue;
791  }
792  } catch ( const irods::exception& _e ) {
794  return USER_INPUT_PATH_ERR;
795  }
796 
797  if ( !exists( p ) ) {
798  rodsLog( LOG_ERROR, "putDirUtil: stat error for %s, errno = %d\n", srcChildPath, errno );
799  return USER_INPUT_PATH_ERR;
800  }
801 
802  // Issue 4013 - moved this snprintf from below, in order to
803  // grab the short filename here, before the symlink is read,
804  // and uses the symlink target filename instead.
805  snprintf( targChildPath, MAX_NAME_LEN, "%s/%s", targColl, p.filename().c_str() );
806 
807  if ( is_symlink( p ) ) {
808  fs::path cp = read_symlink( p );
809  // Issue 4009 (similar to Issue 3663).
810  if (cp.is_relative()) {
811  snprintf( srcChildPath, MAX_NAME_LEN, "%s/%s", srcDir, cp.c_str() );
812  } else {
813  snprintf( srcChildPath, MAX_NAME_LEN, "%s", cp.c_str() );
814  }
815  p = fs::path( srcChildPath );
816  }
817 
818  rodsLong_t dataSize = 0;
819  dataObjOprInp->createMode = getPathStMode( p.c_str() );
820  objType_t childObjType;
821 
822  if ( is_regular_file( p ) ) {
823  childObjType = DATA_OBJ_T;
824  dataSize = file_size( p );
825  }
826  else if ( is_directory( p ) ) {
827  childObjType = COLL_OBJ_T;
828  }
829  else {
830  rodsLog( LOG_ERROR, "putDirUtil: unknown local path type for %s", srcChildPath );
831  savedStatus = USER_INPUT_PATH_ERR;
832  continue;
833  }
834 
835  if ( childObjType == DATA_OBJ_T ) {
836  const auto f_size = file_size(p);
837 
838  if ( bulkFlag == BULK_OPR_SMALL_FILES && f_size > MAX_BULK_OPR_FILE_SIZE ) {
839  continue;
840  }
841  else if ( bulkFlag == BULK_OPR_LARGE_FILES && f_size <= MAX_BULK_OPR_FILE_SIZE ) {
842  continue;
843  }
844  }
845 
846  status = chkStateForResume( conn, rodsRestart, targChildPath,
847  rodsArgs, childObjType, &dataObjOprInp->condInput, 1 );
848 
849  if ( status < 0 ) {
850  /* restart failed */
851  return status;
852  }
853 
854  if ( status == 0 ) {
855  if ( bulkFlag == BULK_OPR_SMALL_FILES &&
856  ( rodsRestart->restartState & LAST_PATH_MATCHED ) != 0 ) {
857  /* enable foreFlag one time */
858  setForceFlagForRestart( bulkOprInp, bulkOprInfo );
859  }
860  continue;
861  }
862 
863  if ( childObjType == DATA_OBJ_T ) { /* a file */
864  if ( bulkFlag == BULK_OPR_SMALL_FILES ) {
865  status = bulkPutFileUtil( conn, srcChildPath, targChildPath,
866  dataSize, dataObjOprInp->createMode, rodsArgs,
867  bulkOprInp, bulkOprInfo );
868  }
869  else {
870  /* normal put */
871  try {
872  status = putFileUtil( conn, srcChildPath, targChildPath,
873  dataSize, rodsArgs, dataObjOprInp );
874  } catch ( const fs::filesystem_error& e ) {
875  rodsLog( LOG_ERROR, e.what() );
876  status = e.code().value();
877  }
878  }
879  if ( rodsRestart->fd > 0 ) {
880  if ( status >= 0 ) {
881  if ( bulkFlag == BULK_OPR_SMALL_FILES ) {
882  if ( status > 0 ) {
883  /* status is the number of files bulk loaded */
884  rodsRestart->curCnt += status;
885  status = writeRestartFile( rodsRestart,
886  targChildPath );
887  }
888  }
889  else {
890  /* write the restart file */
891  rodsRestart->curCnt ++;
892  status = writeRestartFile( rodsRestart, targChildPath );
893  }
894  }
895  }
896  }
897  else { /* a directory */
898  status = mkColl( conn, targChildPath );
899  if ( status < 0 ) {
900  rodsLogError( LOG_ERROR, status, "putDirUtil: mkColl error for %s", targChildPath );
902  {
903  return status;
904  }
905  }
906  status = putDirUtil( myConn, srcChildPath, targChildPath,
907  myRodsEnv, rodsArgs, dataObjOprInp, bulkOprInp,
908  rodsRestart, bulkOprInfo );
909 
910  }
911 
912  if ( status < 0 && status != CAT_NO_ROWS_FOUND ) {
913  rodsLogError( LOG_ERROR, status, "putDirUtil: put %s failed. status = %d", srcChildPath, status );
914  savedStatus = status;
915  if ( rodsRestart->fd > 0 ) {
916  break;
917  }
918  }
919  }
920  }
921  return savedStatus;
922 }
923 
924 int
925 bulkPutDirUtil( rcComm_t **myConn, char *srcDir, char *targColl,
926  rodsEnv *myRodsEnv, rodsArguments_t *rodsArgs, dataObjInp_t *dataObjOprInp,
927  bulkOprInp_t *bulkOprInp, rodsRestart_t *rodsRestart ) {
928  int status;
929  bulkOprInfo_t bulkOprInfo;
930 
931  /* do large files first */
932  bzero( &bulkOprInfo, sizeof( bulkOprInfo ) );
933  bulkOprInfo.flags = BULK_OPR_LARGE_FILES;
934 
935  status = putDirUtil( myConn, srcDir, targColl, myRodsEnv, rodsArgs,
936  dataObjOprInp, bulkOprInp, rodsRestart, &bulkOprInfo );
937 
938  if ( status < 0 ) {
940  "bulkPutDirUtil: Large files bulkPut error for %s", srcDir );
941  return status;
942  }
943 
944  /* now bulk put the small files */
945  bzero( &bulkOprInfo, sizeof( bulkOprInfo ) );
946  bulkOprInfo.flags = BULK_OPR_SMALL_FILES;
947  rstrcpy( dataObjOprInp->objPath, targColl, MAX_NAME_LEN );
948  rstrcpy( bulkOprInp->objPath, targColl, MAX_NAME_LEN );
949  bulkOprInfo.bytesBuf.len = 0;
950  bulkOprInfo.bytesBuf.buf = malloc( BULK_OPR_BUF_SIZE );
951 
952  status = putDirUtil( myConn, srcDir, targColl, myRodsEnv, rodsArgs,
953  dataObjOprInp, bulkOprInp, rodsRestart, &bulkOprInfo );
954 
955  if ( status < 0 ) {
957  "bulkPutDirUtil: Small files bulkPut error for %s", srcDir );
958  return status;
959  }
960 
961  if ( bulkOprInfo.count > 0 ) {
962  status = sendBulkPut( *myConn, bulkOprInp, &bulkOprInfo, rodsArgs );
963  if ( status >= 0 ) {
964  if ( rodsRestart->fd > 0 ) {
965  rodsRestart->curCnt += bulkOprInfo.count;
966  /* last entry. */
967  writeRestartFile( rodsRestart, bulkOprInfo.cachedTargPath );
968  }
969  }
970  else {
972  "bulkPutDirUtil: tarAndBulkPut error for %s",
973  bulkOprInfo.phyBunDir );
974  }
975  clearBulkOprInfo( &bulkOprInfo );
976  if ( bulkOprInfo.bytesBuf.buf != NULL ) {
977  free( bulkOprInfo.bytesBuf.buf );
978  bulkOprInfo.bytesBuf.buf = NULL;
979  }
980 
981  }
982  return status;
983 }
984 
985 int
986 getPhyBunDir( char *phyBunRootDir, char *userName, char *outPhyBunDir )
987 {
988  namespace fs = boost::filesystem;
989 
990  while ( 1 ) {
991  snprintf( outPhyBunDir, MAX_NAME_LEN, "%s/%s.phybun.%u", phyBunRootDir,
992  userName, irods::getRandom<unsigned int>() );
993  fs::path p( outPhyBunDir );
994  if ( !exists( p ) ) {
995  break;
996  }
997  }
998  return 0;
999 }
1000 
1001 int
1002 bulkPutFileUtil( rcComm_t *conn, char *srcPath, char *targPath,
1003  rodsLong_t srcSize, int createMode,
1004  rodsArguments_t *rodsArgs, bulkOprInp_t *bulkOprInp,
1005  bulkOprInfo_t *bulkOprInfo ) {
1006  int status;
1007  int in_fd;
1008  int bytesRead = 0;
1009  /*#ifndef windows_platform
1010  char *bufPtr = bulkOprInfo->bytesBuf.buf + bulkOprInfo->size;
1011  #else*/ /* make change for Windows only */
1012  char *bufPtr = ( char * )( bulkOprInfo->bytesBuf.buf ) + bulkOprInfo->size;
1013  /*#endif*/
1014 
1015 #ifdef windows_platform
1016  in_fd = iRODSNt_bopen( srcPath, O_RDONLY, 0 );
1017 #else
1018  in_fd = open( srcPath, O_RDONLY, 0 );
1019 #endif
1020  if ( in_fd < 0 ) { /* error */
1021  status = USER_FILE_DOES_NOT_EXIST - errno;
1022  rodsLog( LOG_ERROR,
1023  "bulkPutFileUtilcannot open file %s, status = %d", srcPath, status );
1024  return status;
1025  }
1026 
1027  status = myRead( in_fd, bufPtr, srcSize, &bytesRead, NULL );
1028  if ( status != srcSize ) {
1029  if ( status >= 0 ) {
1030  status = SYS_COPY_LEN_ERR - errno;
1031  rodsLogError( LOG_ERROR, status, "bulkPutFileUtil: bytesRead %ju does not match srcSize %ju for %s",
1032  ( uintmax_t )bytesRead, ( uintmax_t )srcSize, srcPath );
1033  }
1034  else {
1035  status = USER_INPUT_PATH_ERR - errno;
1036  }
1037  return status;
1038  }
1039  close( in_fd );
1040  rstrcpy( bulkOprInfo->cachedTargPath, targPath, MAX_NAME_LEN );
1041  bulkOprInfo->count++;
1042  bulkOprInfo->size += srcSize;
1043  bulkOprInfo->bytesBuf.len = bulkOprInfo->size;
1044 
1045  if ( getValByKey( &bulkOprInp->condInput, REG_CHKSUM_KW ) != NULL ||
1046  getValByKey( &bulkOprInp->condInput, VERIFY_CHKSUM_KW ) != NULL ) {
1047  char chksumStr[NAME_LEN];
1048 
1049  rodsEnv env;
1050  int ret = getRodsEnv( &env );
1051  if ( ret < 0 ) {
1052  rodsLog(
1053  LOG_ERROR,
1054  "bulkPutFileUtil: error getting rods env %d",
1055  ret );
1056  return ret;
1057  }
1058  status = chksumLocFile( srcPath, chksumStr, env.rodsDefaultHashScheme );
1059  if ( status < 0 ) {
1060  rodsLog( LOG_ERROR,
1061  "bulkPutFileUtil: chksumLocFile error for %s ", srcPath );
1062  return status;
1063  }
1064  status = fillAttriArrayOfBulkOprInp( targPath, createMode, chksumStr,
1065  bulkOprInfo->size, bulkOprInp );
1066  }
1067  else {
1068  status = fillAttriArrayOfBulkOprInp( targPath, createMode, NULL,
1069  bulkOprInfo->size, bulkOprInp );
1070  }
1071  if ( status < 0 ) {
1073  "bulkPutFileUtil: fillAttriArrayOfBulkOprInp error for %s",
1074  srcPath );
1075  return status;
1076  }
1077  if ( bulkOprInfo->count >= MAX_NUM_BULK_OPR_FILES ||
1078  bulkOprInfo->size >= BULK_OPR_BUF_SIZE -
1080  /* tar send it */
1081  status = sendBulkPut( conn, bulkOprInp, bulkOprInfo, rodsArgs );
1082  if ( status >= 0 ) {
1083  /* return the count */
1084  status = bulkOprInfo->count;
1085  }
1086  else {
1088  "bulkPutFileUtil: tarAndBulkPut error for %s", srcPath );
1089  }
1090  clearBulkOprInfo( bulkOprInfo );
1091  }
1092  return status;
1093 }
1094 
1095 int
1096 sendBulkPut( rcComm_t *conn, bulkOprInp_t *bulkOprInp,
1097  bulkOprInfo_t *bulkOprInfo, rodsArguments_t *rodsArgs ) {
1098  struct timeval startTime, endTime;
1099  int status = 0;
1100 
1101  if ( bulkOprInfo == NULL || bulkOprInfo->count <= 0 ) {
1102  return 0;
1103  }
1104 
1105  if ( rodsArgs->verbose == True ) {
1106  ( void ) gettimeofday( &startTime, ( struct timezone * )0 );
1107  }
1108 
1109  /* send it */
1110  if ( bulkOprInfo->bytesBuf.buf != NULL ) {
1111  status = rcBulkDataObjPut( conn, bulkOprInp, &bulkOprInfo->bytesBuf );
1112  }
1113  /* reset the row count */
1114  bulkOprInp->attriArray.rowCnt = 0;
1115  if ( bulkOprInfo->forceFlagAdded == 1 ) {
1116  rmKeyVal( &bulkOprInp->condInput, FORCE_FLAG_KW );
1117  bulkOprInfo->forceFlagAdded = 0;
1118  }
1119  if ( status >= 0 ) {
1120  if ( rodsArgs->verbose == True ) {
1121  printf( "Bulk upload %d files.\n", bulkOprInfo->count );
1122  ( void ) gettimeofday( &endTime, ( struct timezone * )0 );
1123  printTiming( conn, bulkOprInfo->cachedTargPath,
1124  bulkOprInfo->size, bulkOprInfo->cachedTargPath,
1125  &startTime, &endTime );
1126  }
1127  if ( gGuiProgressCB != NULL ) {
1129  bulkOprInfo->cachedTargPath, MAX_NAME_LEN );
1130  conn->operProgress.totalNumFilesDone += bulkOprInfo->count;
1131  conn->operProgress.totalFileSizeDone += bulkOprInfo->size;
1132  gGuiProgressCB( &conn->operProgress );
1133  }
1134  }
1135 
1136  return status;
1137 }
1138 
1139 int
1141  if ( bulkOprInfo == NULL || bulkOprInfo->count <= 0 ) {
1142  return 0;
1143  }
1144  bulkOprInfo->bytesBuf.len = 0;
1145  bulkOprInfo->count = bulkOprInfo->size = 0;
1146  *bulkOprInfo->cachedTargPath = '\0';
1147 
1148  return 0;
1149 }
RBUDP_SEND_RATE_KW
#define RBUDP_SEND_RATE_KW
Definition: rodsKeyWdDef.h:78
rodsArguments_t::restartFileString
char * restartFileString
Definition: parseCommandLine.h:117
rodsLog
void rodsLog(int level, const char *formatStr,...)
Definition: rodsLog.cpp:86
getValByKey
char * getValByKey(const keyValPair_t *condInput, const char *keyWord)
Definition: rcMisc.cpp:675
DataObjInp::objPath
char objPath[(1024+64)]
Definition: dataObjInpOut.h:66
writeRestartFile
int writeRestartFile(rodsRestart_t *rodsRestart, char *lastDonePath)
Definition: rcMisc.cpp:2677
LOCK_TYPE_KW
#define LOCK_TYPE_KW
Definition: rodsKeyWdDef.h:115
rcComm_t
Definition: rcConnect.h:95
NULL
#define NULL
Definition: rodsDef.h:70
BULK_OPR_KW
#define BULK_OPR_KW
Definition: rodsKeyWdDef.h:99
getDirSizeForProgStat
int getDirSizeForProgStat(rodsArguments_t *rodsArgs, char *srcDir, operProgress_t *operProgress)
Definition: miscUtil.cpp:1967
fileRestart_t::info
fileRestartInfo_t info
Definition: rcConnect.h:85
True
#define True
Definition: parseCommandLine.h:11
addKeyVal
int addKeyVal(keyValPair_t *condInput, const char *keyWord, const char *value)
Definition: rcMisc.cpp:789
BULK_OPR_SMALL_FILES
#define BULK_OPR_SMALL_FILES
Definition: putUtil.h:18
myRead
int myRead(int sock, void *buf, int len, int *bytesRead, struct timeval *tv)
Definition: rcMisc.cpp:4651
BULK_OPR_BUF_SIZE
#define BULK_OPR_BUF_SIZE
Definition: rodsDef.h:138
rodsPath.h
rodsArguments_t::ticketString
char * ticketString
Definition: parseCommandLine.h:99
rodsArguments_t::all
int all
Definition: parseCommandLine.h:18
DataObjInp::openFlags
int openFlags
Definition: dataObjInpOut.h:68
getRodsEnv
int getRodsEnv(rodsEnv *myRodsEnv)
Definition: getRodsEnv.cpp:112
rErrMsg_t
Definition: rodsError.h:19
bulkOprInp_t
Definition: bulkDataObjPut.h:17
OperProgress::flag
int flag
Definition: guiProgressCallback.h:9
irods_server_properties.hpp
rodsArguments_t
Definition: parseCommandLine.h:14
RodsPathInp::resolved
int resolved
Definition: rodsPath.h:39
rodsRestart_t::doneCnt
int doneCnt
Definition: rodsDef.h:320
RESTART_OPR_FAILED
@ RESTART_OPR_FAILED
Definition: rodsErrorTable.h:262
irods::disallow_file_dir_mix_on_command_line
int disallow_file_dir_mix_on_command_line(rodsArguments_t const *const rodsArgs, rodsPathInp_t const *const rodsPathInp)
Definition: irods_path_recursion.cpp:354
fileRestart_t::infoFile
char infoFile[(1024+64)]
Definition: rcConnect.h:84
BytesBuf::buf
void * buf
Definition: rodsDef.h:199
initAttriArrayOfBulkOprInp
int initAttriArrayOfBulkOprInp(bulkOprInp_t *bulkOprInp)
Definition: rcMisc.cpp:3716
rodsRestart_t::restartState
int restartState
Definition: rodsDef.h:325
rodsArguments_t::retriesValue
int retriesValue
Definition: parseCommandLine.h:122
rodsArguments_t::rbudp
int rbudp
Definition: parseCommandLine.h:83
rcBulkDataObjPut
int rcBulkDataObjPut(rcComm_t *conn, bulkOprInp_t *bulkOprInp, bytesBuf_t *bulkOprInpBBuf)
Definition: rcBulkDataObjPut.cpp:143
NON_BULK_OPR
#define NON_BULK_OPR
Definition: putUtil.h:17
setForceFlagForRestart
int setForceFlagForRestart(bulkOprInp_t *bulkOprInp, bulkOprInfo_t *bulkOprInfo)
Definition: rcMisc.cpp:3978
RBUDP_TRANSFER_KW
#define RBUDP_TRANSFER_KW
Definition: rodsKeyWdDef.h:76
RodsPathInp
Definition: rodsPath.h:33
DataObjInp::dataSize
rodsLong_t dataSize
Definition: dataObjInpOut.h:70
rcTicketAdmin
int rcTicketAdmin(rcComm_t *conn, ticketAdminInp_t *ticketAdminInp)
Definition: rcTicketAdmin.cpp:9
rodsArguments_t::verbose
int verbose
Definition: parseCommandLine.h:107
MAX_NUM_BULK_OPR_FILES
#define MAX_NUM_BULK_OPR_FILES
Definition: rodsDef.h:136
OperProgress::curFileSize
rodsLong_t curFileSize
Definition: guiProgressCallback.h:15
irods_exception.hpp
NON_BULK_OPR_KW
#define NON_BULK_OPR_KW
Definition: rodsKeyWdDef.h:100
env
Definition: restructs.hpp:226
rodsArguments_t::dataTypeString
char * dataTypeString
Definition: parseCommandLine.h:55
OperProgress::curFileSizeDone
rodsLong_t curFileSizeDone
Definition: guiProgressCallback.h:16
irods::experimental::administration::client::v1::exists
auto exists(rcComm_t &conn, const user &user) -> bool
Definition: user_administration.cpp:359
readLfRestartFile
int readLfRestartFile(char *infoFile, fileRestartInfo_t **info)
Definition: rcPortalOpr.cpp:1475
pid_age.p
p
Definition: pid_age.py:13
resolveRodsTarget
int resolveRodsTarget(rcComm_t *conn, rodsPathInp_t *rodsPathInp, int oprType)
Definition: miscUtil.cpp:2342
fileRestartInfo_t::objPath
char objPath[(1024+64)]
Definition: rcConnect.h:74
OperProgress::totalFileSize
rodsLong_t totalFileSize
Definition: guiProgressCallback.h:11
PUT_OPR
#define PUT_OPR
Definition: dataObjInpOut.h:167
bulkOprInp_t::condInput
keyValPair_t condInput
Definition: bulkDataObjPut.h:20
checksum.hpp
OperProgress::totalFileSizeDone
rodsLong_t totalFileSizeDone
Definition: guiProgressCallback.h:13
USER_FILE_DOES_NOT_EXIST
@ USER_FILE_DOES_NOT_EXIST
Definition: rodsErrorTable.h:241
clearBulkOprInfo
int clearBulkOprInfo(bulkOprInfo_t *bulkOprInfo)
Definition: putUtil.cpp:1140
bulkOprInfo_t::size
int size
Definition: putUtil.h:27
rodsArguments_t::rlock
int rlock
Definition: parseCommandLine.h:58
rodsArguments_t::bulk
int bulk
Definition: parseCommandLine.h:27
REPL_NUM_KW
#define REPL_NUM_KW
Definition: rodsKeyWdDef.h:30
gGuiProgressCB
guiProgressCallback gGuiProgressCB
Definition: rcGlobal.cpp:207
ALL_KW
#define ALL_KW
Definition: rodsKeyWdDef.h:10
rodsArguments_t::physicalPathString
char * physicalPathString
Definition: parseCommandLine.h:71
setSessionTicket
int setSessionTicket(rcComm_t *myConn, char *ticket)
Definition: putUtil.cpp:147
rodsArguments_t::resource
int resource
Definition: parseCommandLine.h:86
rodsRestart_t::lastDonePath
char lastDonePath[(1024+64)]
Definition: rodsDef.h:322
rcGlobalExtern.h
bulkOprInfo_t::cachedTargPath
char cachedTargPath[(1024+64)]
Definition: putUtil.h:30
getPathStMode
int getPathStMode(const char *p)
Definition: rcMisc.cpp:4354
RodsPathInp::srcPath
rodsPath_t * srcPath
Definition: rodsPath.h:35
rodsArguments_t::kv_pass_string
char * kv_pass_string
Definition: parseCommandLine.h:141
LOG_ERROR
#define LOG_ERROR
Definition: rodsLog.h:43
RodsPath::objType
objType_t objType
Definition: rodsPath.h:19
RECURSIVE_OPR__KW
#define RECURSIVE_OPR__KW
Definition: rodsKeyWdDef.h:66
putDirUtil
int putDirUtil(rcComm_t **myConn, char *srcDir, char *targColl, rodsEnv *myRodsEnv, rodsArguments_t *rodsArgs, dataObjInp_t *dataObjOprInp, bulkOprInp_t *bulkOprInp, rodsRestart_t *rodsRestart, bulkOprInfo_t *bulkOprInfo)
Definition: putUtil.cpp:705
NO_THREADING
#define NO_THREADING
Definition: rodsDef.h:100
ticketAdminInp_t::arg1
char * arg1
Definition: ticketAdmin.h:7
SYS_INVALID_INPUT_PARAM
@ SYS_INVALID_INPUT_PARAM
Definition: rodsErrorTable.h:195
OperProgress::curFileName
char curFileName[MAX_NAME_LEN]
Definition: guiProgressCallback.h:14
PURGE_CACHE_KW
#define PURGE_CACHE_KW
Definition: rodsKeyWdDef.h:110
DATA_TYPE_KW
#define DATA_TYPE_KW
Definition: rodsKeyWdDef.h:23
FILE_RESTART_ON
@ FILE_RESTART_ON
Definition: rcConnect.h:64
BytesBuf::len
int len
Definition: rodsDef.h:198
irods::experimental::filesystem::client::end
auto end(const collection_iterator &) noexcept -> const collection_iterator
Definition: collection_iterator.hpp:88
mkColl
int mkColl(rcComm_t *conn, char *collection)
Definition: miscUtil.cpp:31
OperProgress::totalNumFilesDone
rodsLong_t totalNumFilesDone
Definition: guiProgressCallback.h:12
REG_CHKSUM_KW
#define REG_CHKSUM_KW
Definition: rodsKeyWdDef.h:15
COLL_OBJ_T
@ COLL_OBJ_T
Definition: rodsType.h:39
RECONN_TIMEOUT
#define RECONN_TIMEOUT
Definition: rcConnect.h:36
RodsPathInp::targPath
rodsPath_t * targPath
Definition: rodsPath.h:37
ticketAdminInp_t::arg6
char * arg6
Definition: ticketAdmin.h:12
rodsArguments_t::checksum
int checksum
Definition: parseCommandLine.h:52
DEST_RESC_NAME_KW
#define DEST_RESC_NAME_KW
Definition: rodsKeyWdDef.h:20
rodsArguments_t::metadata_string
char * metadata_string
Definition: parseCommandLine.h:138
rodsArguments_t::redirectConn
int redirectConn
Definition: parseCommandLine.h:51
bulkOprInfo_t::forceFlagAdded
int forceFlagAdded
Definition: putUtil.h:26
rodsArguments_t::ticket
int ticket
Definition: parseCommandLine.h:98
setStateForResume
int setStateForResume(rcComm_t *conn, rodsRestart_t *rodsRestart, char *restartPath, objType_t objType, keyValPair_t *condInput, int deleteFlag)
Definition: putUtil.cpp:95
rodsArguments_t::verifyChecksum
int verifyChecksum
Definition: parseCommandLine.h:53
bulkOprInfo_t::bytesBuf
bytesBuf_t bytesBuf
Definition: putUtil.h:33
bulkOprInfo_t::phyBunDir
char phyBunDir[(1024+64)]
Definition: putUtil.h:29
OPR_RESUMED
#define OPR_RESUMED
Definition: rodsDef.h:308
SYS_COPY_LEN_ERR
@ SYS_COPY_LEN_ERR
Definition: rodsErrorTable.h:95
MAX_PATH_ALLOWED
#define MAX_PATH_ALLOWED
Definition: rodsDef.h:58
redirectConnToRescSvr
int redirectConnToRescSvr(rcComm_t **conn, dataObjInp_t *dataObjInp, rodsEnv *myEnv, int reconnFlag)
Definition: sockComm.cpp:1520
rodsArguments_t::acl_string
char * acl_string
Definition: parseCommandLine.h:139
sendBulkPut
int sendBulkPut(rcComm_t *conn, bulkOprInp_t *bulkOprInp, bulkOprInfo_t *bulkOprInfo, rodsArguments_t *rodsArgs)
Definition: putUtil.cpp:1096
VERY_VERBOSE_KW
#define VERY_VERBOSE_KW
Definition: rodsKeyWdDef.h:77
putUtil
int putUtil(rcComm_t **myConn, rodsEnv *myRodsEnv, rodsArguments_t *myRodsArgs, rodsPathInp_t *rodsPathInp)
Definition: putUtil.cpp:165
setStateForRestart
int setStateForRestart(rodsRestart_t *rodsRestart, rodsPath_t *targPath, rodsArguments_t *rodsArgs)
Definition: rcMisc.cpp:2714
OperProgress::totalNumFiles
rodsLong_t totalNumFiles
Definition: guiProgressCallback.h:10
rodsArguments_t::purgeCache
int purgeCache
Definition: parseCommandLine.h:78
rodsArguments_t::veryVerbose
int veryVerbose
Definition: parseCommandLine.h:108
LAST_PATH_MATCHED
#define LAST_PATH_MATCHED
Definition: rodsDef.h:306
ACL_INCLUDED_KW
#define ACL_INCLUDED_KW
Definition: rodsKeyWdDef.h:35
DataObjInp
Definition: dataObjInpOut.h:65
irods::file_system_sanity_check
int file_system_sanity_check(irods::recursion_map_t &, rodsArguments_t const *const, rodsPathInp_t const *const)
Definition: irods_path_recursion.cpp:282
fileRestartInfo_t::status
fileRestartStatus_t status
Definition: rcConnect.h:76
putFileUtil
int putFileUtil(rcComm_t *conn, char *srcPath, char *targPath, rodsLong_t srcSize, rodsArguments_t *rodsArgs, dataObjInp_t *dataObjOprInp)
Definition: putUtil.cpp:370
MAX_NAME_LEN
#define MAX_NAME_LEN
Definition: rodsDef.h:61
terminate_irods_processes.e
e
Definition: terminate_irods_processes.py:19
ticketAdminInp_t::arg2
char * arg2
Definition: ticketAdmin.h:8
bulkPutFileUtil
int bulkPutFileUtil(rcComm_t *conn, char *srcPath, char *targPath, rodsLong_t srcSize, int createMode, rodsArguments_t *rodsArgs, bulkOprInp_t *bulkOprInp, bulkOprInfo_t *bulkOprInfo)
Definition: putUtil.cpp:1002
RBUDP_PACK_SIZE_KW
#define RBUDP_PACK_SIZE_KW
Definition: rodsKeyWdDef.h:79
rmKeyVal
int rmKeyVal(keyValPair_t *condInput, const char *keyWord)
Definition: rcMisc.cpp:710
chksumLocFile
int chksumLocFile(const char *fileName, char *chksumStr, const char *)
Definition: checksum.cpp:18
BULK_OPR_MISMATCH_FOR_RESTART
@ BULK_OPR_MISMATCH_FOR_RESTART
Definition: rodsErrorTable.h:278
rodsRestart_t::oprType
char oprType[64]
Definition: rodsDef.h:323
RodsPath::size
rodsLong_t size
Definition: rodsPath.h:21
initCondForPut
int initCondForPut(rcComm_t *conn, rodsEnv *myRodsEnv, rodsArguments_t *rodsArgs, dataObjInp_t *dataObjOprInp, bulkOprInp_t *bulkOprInp, rodsRestart_t *rodsRestart)
Definition: putUtil.cpp:452
rcReconnect
int rcReconnect(rcComm_t **conn, char *newHost, rodsEnv *myEnv, int reconnFlag)
Definition: sockComm.cpp:1547
irods.pypyodbc.status
status
Definition: pypyodbc.py:467
rodsArguments_t::retries
int retries
Definition: parseCommandLine.h:121
putUtil.h
rcComm_t::operProgress
operProgress_t operProgress
Definition: rcConnect.h:119
rodsLog.h
GenQueryOut::rowCnt
int rowCnt
Definition: rodsGenQuery.h:68
bulkPutDirUtil
int bulkPutDirUtil(rcComm_t **myConn, char *srcDir, char *targColl, rodsEnv *myRodsEnv, rodsArguments_t *rodsArgs, dataObjInp_t *dataObjOprInp, bulkOprInp_t *bulkOprInp, rodsRestart_t *rodsRestart)
Definition: putUtil.cpp:925
irods::log
void log(const error &)
Definition: irods_log.cpp:13
DEF_RESC_NAME_KW
#define DEF_RESC_NAME_KW
Definition: rodsKeyWdDef.h:21
rodsArguments_t::resourceString
char * resourceString
Definition: parseCommandLine.h:87
FILE_NOT_RESTART
@ FILE_NOT_RESTART
Definition: rcConnect.h:68
bulkOprInp_t::objPath
char objPath[(1024+64)]
Definition: bulkDataObjPut.h:18
KEY_VALUE_PASSTHROUGH_KW
#define KEY_VALUE_PASSTHROUGH_KW
Definition: rodsKeyWdDef.h:284
LOG_NOTICE
#define LOG_NOTICE
Definition: rodsLog.h:33
rcPortalOpr.h
fileRestartInfo_t
Definition: rcConnect.h:72
irods::is_path_valid_for_recursion
bool is_path_valid_for_recursion(boost::filesystem::path const &, recursion_map_t &, bool)
Definition: irods_path_recursion.cpp:80
irods::exception::client_display_what
virtual const char * client_display_what() const
Definition: irods_exception.cpp:52
rodsRestart_t
Definition: rodsDef.h:317
rcComm_t::fileRestart
fileRestart_t fileRestart
Definition: rcConnect.h:135
ERROR
#define ERROR(code_, message_)
Definition: irods_error.hpp:117
rodsArguments_t::wlock
int wlock
Definition: parseCommandLine.h:59
rodsArguments_t::force
int force
Definition: parseCommandLine.h:38
DATA_OBJ_T
@ DATA_OBJ_T
Definition: rodsType.h:38
DataObjInp::createMode
int createMode
Definition: dataObjInpOut.h:67
bulkOprInfo_t::count
int count
Definition: putUtil.h:25
ticketAdminInp_t::arg3
char * arg3
Definition: ticketAdmin.h:9
ticketAdminInp_t::arg4
char * arg4
Definition: ticketAdmin.h:10
fileRestartInfo_t::fileName
char fileName[(1024+64)]
Definition: rcConnect.h:73
rodsArguments_t::physicalPath
int physicalPath
Definition: parseCommandLine.h:70
rodsEnv::rodsDefResource
char rodsDefResource[64]
Definition: getRodsEnv.h:17
getPhyBunDir
int getPhyBunDir(char *phyBunRootDir, char *userName, char *outPhyBunDir)
Definition: putUtil.cpp:986
fileRestart_t::flags
fileRestartFlag_t flags
Definition: rcConnect.h:82
RodsPath::outPath
char outPath[(1024+64)]
Definition: rodsPath.h:24
bulkOprInfo_t::flags
int flags
Definition: putUtil.h:24
rodsArguments_t::kv_pass
int kv_pass
Definition: parseCommandLine.h:140
printTiming
int printTiming(rcComm_t *conn, char *objPath, rodsLong_t fileSize, char *localFile, struct timeval *startTime, struct timeval *endTime)
Definition: miscUtil.cpp:396
rodsRestart_t::fd
int fd
Definition: rodsDef.h:319
rodsArguments_t::recursive
int recursive
Definition: parseCommandLine.h:85
FORCE_FLAG_KW
#define FORCE_FLAG_KW
Definition: rodsKeyWdDef.h:13
MATCHED_RESTART_COLL
#define MATCHED_RESTART_COLL
Definition: rodsDef.h:307
irods::experimental::filesystem::client::remove
auto remove(rcComm_t &_comm, const path &_p, remove_options _opts=remove_options::none) -> bool
Definition: filesystem.cpp:699
WRITE_LOCK_TYPE
#define WRITE_LOCK_TYPE
Definition: dataObjLock.h:10
RodsPath::objMode
uint objMode
Definition: rodsPath.h:22
DataObjInp::oprType
int oprType
Definition: dataObjInpOut.h:72
rcChksumLocFile
int rcChksumLocFile(char *fileName, char *chksumFlag, keyValPair_t *condInput, const char *)
Definition: checksum.cpp:228
CAT_NO_ROWS_FOUND
@ CAT_NO_ROWS_FOUND
Definition: rodsErrorTable.h:423
rodsArguments_t::lfrestart
int lfrestart
Definition: parseCommandLine.h:118
ticketAdminInp_t::arg5
char * arg5
Definition: ticketAdmin.h:11
VERIFY_CHKSUM_KW
#define VERIFY_CHKSUM_KW
Definition: rodsKeyWdDef.h:16
USER_INPUT_OPTION_ERR
@ USER_INPUT_OPTION_ERR
Definition: rodsErrorTable.h:249
openRestartFile
int openRestartFile(char *restartFile, rodsRestart_t *rodsRestart)
Definition: rcMisc.cpp:2552
rodsLogError
void rodsLogError(int level, int errCode, char *formatStr,...)
Definition: rodsLog.cpp:422
MAX_BULK_OPR_FILE_SIZE
#define MAX_BULK_OPR_FILE_SIZE
Definition: rodsDef.h:137
rcDataObjUnlink
int rcDataObjUnlink(rcComm_t *conn, dataObjInp_t *dataObjUnlinkInp)
Definition: rcDataObjUnlink.cpp:63
RodsPathInp::numSrc
int numSrc
Definition: rodsPath.h:34
ticketAdminInp_t
Definition: ticketAdmin.h:6
BULK_OPR_LARGE_FILES
#define BULK_OPR_LARGE_FILES
Definition: putUtil.h:19
rodsArguments_t::lfrestartFileString
char * lfrestartFileString
Definition: parseCommandLine.h:119
USER_PATH_EXCEEDS_MAX
@ USER_PATH_EXCEEDS_MAX
Definition: rodsErrorTable.h:267
irods::exception
Definition: irods_exception.hpp:15
fillAttriArrayOfBulkOprInp
int fillAttriArrayOfBulkOprInp(char *objPath, int dataMode, char *inpChksum, int offset, bulkOprInp_t *bulkOprInp)
Definition: rcMisc.cpp:3763
rodsArguments_t::replNum
int replNum
Definition: parseCommandLine.h:63
bulkOprInfo_t
Definition: putUtil.h:23
sockComm.h
rodsErrorTable.h
False
#define False
Definition: parseCommandLine.h:12
rcDataObjPut
int rcDataObjPut(rcComm_t *conn, dataObjInp_t *dataObjInp, char *locFilePath)
Definition: rcDataObjPut.cpp:100
irods_path_recursion.hpp
irods_random.hpp
RodsPath
Definition: rodsPath.h:18
rodsRestart_t::curCnt
int curCnt
Definition: rodsDef.h:324
miscUtil.h
rodsArguments_t::numberValue
int numberValue
Definition: parseCommandLine.h:69
rstrcpy
char * rstrcpy(char *dest, const char *src, int maxLen)
Definition: stringOpr.cpp:51
PATH_MATCHING
#define PATH_MATCHING
Definition: rodsDef.h:305
LOCAL_FILE_T
@ LOCAL_FILE_T
Definition: rodsType.h:41
DataObjInp::condInput
keyValPair_t condInput
Definition: dataObjInpOut.h:74
FILE_RESTARTED
@ FILE_RESTARTED
Definition: rcConnect.h:69
METADATA_INCLUDED_KW
#define METADATA_INCLUDED_KW
Definition: rodsKeyWdDef.h:34
rodsEnv
Definition: getRodsEnv.h:8
NAME_LEN
#define NAME_LEN
Definition: rodsDef.h:55
irods::recursion_map_t
std::map< std::string, std::string > recursion_map_t
Definition: irods_path_recursion.hpp:20
KeyValPair
Definition: objInfo.h:120
chkStateForResume
int chkStateForResume(rcComm_t *conn, rodsRestart_t *rodsRestart, char *targPath, rodsArguments_t *rodsArgs, objType_t objType, keyValPair_t *condInput, int deleteFlag)
Definition: putUtil.cpp:34
clearKeyVal
int clearKeyVal(keyValPair_t *condInput)
Definition: rcMisc.cpp:1047
bulkOprInp_t::attriArray
genQueryOut_t attriArray
Definition: bulkDataObjPut.h:19
FILE_PATH_KW
#define FILE_PATH_KW
Definition: rodsKeyWdDef.h:28
rodsArguments_t::reconnect
int reconnect
Definition: parseCommandLine.h:103
NO_RECONN
#define NO_RECONN
Definition: rcConnect.h:34
rodsArguments_t::replNumValue
char * replNumValue
Definition: parseCommandLine.h:64
USER_INPUT_PATH_ERR
@ USER_INPUT_PATH_ERR
Definition: rodsErrorTable.h:248
rodsArguments_t::restart
int restart
Definition: parseCommandLine.h:116
rodsArguments_t::dataType
int dataType
Definition: parseCommandLine.h:54
rodsEnv::rodsHost
char rodsHost[64]
Definition: getRodsEnv.h:10
DataObjInp::numThreads
int numThreads
Definition: dataObjInpOut.h:71
irods_log.hpp
rodsLong_t
long long rodsLong_t
Definition: rodsType.h:32
rodsArguments_t::number
int number
Definition: parseCommandLine.h:68
objType_t
objType_t
Definition: rodsType.h:36
lfRestartPutWithInfo
int lfRestartPutWithInfo(rcComm_t *conn, fileRestartInfo_t *info)
Definition: rcPortalOpr.cpp:1543
USER__NULL_INPUT_ERR
@ USER__NULL_INPUT_ERR
Definition: rodsErrorTable.h:247