irods  4.2.8
About: iRODS (the integrated Rule Oriented Data System) is a distributed data-management system for creating data grids, digital libraries, persistent archives, and real-time data systems.
  Fossies Dox: irods-4.2.8.tar.gz  ("unofficial" and yet experimental doxygen-generated source code documentation)  

rsDataObjRepl.cpp
Go to the documentation of this file.
1 
3 /* This is script-generated code (for the most part). */
4 /* See dataObjRepl.h for a description of this API call.*/
5 
6 
7 #include "dataObjRepl.h"
8 #include "dataObjOpr.hpp"
9 #include "dataObjCreate.h"
10 #include "dataObjOpen.h"
11 #include "dataObjClose.h"
12 #include "dataObjPut.h"
13 #include "dataObjGet.h"
14 #include "rodsLog.h"
15 #include "objMetaOpr.hpp"
16 #include "physPath.hpp"
17 #include "specColl.hpp"
18 #include "resource.hpp"
19 #include "icatDefines.h"
20 #include "getRemoteZoneResc.h"
21 #include "l3FileGetSingleBuf.h"
22 #include "l3FilePutSingleBuf.h"
23 #include "fileSyncToArch.h"
24 #include "fileStageToCache.h"
25 #include "unbunAndRegPhyBunfile.h"
26 #include "dataObjTrim.h"
27 #include "dataObjLock.h"
28 #include "miscServerFunct.hpp"
29 #include "rsDataObjRepl.hpp"
30 #include "apiNumber.h"
31 #include "rsDataCopy.hpp"
32 #include "rsDataObjCreate.hpp"
33 #include "rsDataObjOpen.hpp"
34 #include "rsDataObjClose.hpp"
35 #include "rsL3FileGetSingleBuf.hpp"
36 #include "rsDataObjGet.hpp"
37 #include "rsDataObjPut.hpp"
38 #include "rsL3FilePutSingleBuf.hpp"
39 #include "rsFileStageToCache.hpp"
40 #include "rsFileSyncToArch.hpp"
42 
45 #include "irods_log.hpp"
46 #include "irods_stacktrace.hpp"
49 #include "irods_random.hpp"
51 
52 #include <vector>
53 
54 #include <boost/lexical_cast.hpp>
55 
56 /* rsDataObjRepl - The Api handler of the rcDataObjRepl call - Replicate
57  * a data object.
58  * Input -
59  * rsComm_t *rsComm
60  * dataObjInp_t *dataObjInp - The replication input
61  * transferStat_t **transStat - transfer stat output
62  */
63 
64 int
65 rsDataObjRepl( rsComm_t *rsComm, dataObjInp_t *dataObjInp,
66  transferStat_t **transStat ) {
67  auto* src_resc_name = getValByKey(&dataObjInp->condInput, RESC_NAME_KW); // -S
68  auto* replica_number = getValByKey(&dataObjInp->condInput, REPL_NUM_KW); // -n
69 
70  if (src_resc_name && replica_number) {
72  }
73 
74  int status;
75  int remoteFlag;
77  dataObjInfo_t *dataObjInfo = NULL;
78  char* lockType = NULL; // JMC - backport 4609
79  int lockFd = -1; // JMC - backport 4609
80 
81  if ( getValByKey( &dataObjInp->condInput, SU_CLIENT_USER_KW ) != NULL ) {
82  /* To SU, cannot be called by normal user directly */
85  }
86  }
87 
88  status = resolvePathInSpecColl( rsComm, dataObjInp->objPath,
89  READ_COLL_PERM, 0, &dataObjInfo );
90 
91  if ( status == DATA_OBJ_T ) {
92  if ( dataObjInfo != NULL && dataObjInfo->specColl != NULL ) {
93  if ( dataObjInfo->specColl->collClass == LINKED_COLL ) {
94  rstrcpy( dataObjInp->objPath, dataObjInfo->objPath,
95  MAX_NAME_LEN );
96  freeAllDataObjInfo( dataObjInfo );
97  }
98  else {
99  freeAllDataObjInfo( dataObjInfo );
101  }
102  }
103  }
104 
105  remoteFlag = getAndConnRemoteZone( rsComm, dataObjInp, &rodsServerHost,
106  REMOTE_OPEN );
107 
108  if ( remoteFlag < 0 ) {
109  return remoteFlag;
110  }
111  else if ( remoteFlag == REMOTE_HOST ) {
112  status = _rcDataObjRepl( rodsServerHost->conn, dataObjInp,
113  transStat );
114  return status;
115  }
116 
117  // =-=-=-=-=-=-=-
118  // if the dest resc name key is set then we will resolve that key as our hier
119  // if a resc name key word is not also set in which case we inadvertently resolve
120  // the DESTINATION resource, not a valid source. unset it before resolve hier
121  // and replace it afterwards to get intended behavior
122  char* dest_resc_ptr = getValByKey( &dataObjInp->condInput, DEST_RESC_NAME_KW );
123  std::string tmp_dest_resc;
124  if ( dest_resc_ptr ) {
125  tmp_dest_resc = dest_resc_ptr;
126  rmKeyVal( &dataObjInp->condInput, DEST_RESC_NAME_KW );
127  }
128 
129  // =-=-=-=-=-=-=-
130  // make sure tmp_dest_resc exists and is available
131  if ( !tmp_dest_resc.empty() ) {
132  irods::error resc_err = irods::is_hier_live( tmp_dest_resc );
133  if ( !resc_err.ok() ) {
134  irods::log( resc_err );
135  return resc_err.code();
136  }
137  }
138 
139  // =-=-=-=-=-=-=-
140  // call redirect for our operation of choice to request the hier string appropriately
141  std::string hier;
142  char* tmp_hier = getValByKey( &dataObjInp->condInput, RESC_HIER_STR_KW );
143 
144  if ( !tmp_hier ) {
145  // set a repl keyword here so resources can respond accordingly
146  addKeyVal( &dataObjInp->condInput, IN_REPL_KW, "" );
148  rsComm, dataObjInp, hier );
149  if ( !ret.ok() ) {
150  std::stringstream msg;
151  msg << "failed in irods::resolve_resource_hierarchy for [";
152  msg << dataObjInp->objPath << "]";
153  irods::log( PASSMSG( msg.str(), ret ) );
154  return ret.code();
155  }
156 
157  addKeyVal( &dataObjInp->condInput, RESC_HIER_STR_KW, hier.c_str() );
158 
159  }
160  else {
161  hier = tmp_hier;
162  }
163 
164  // =-=-=-=-=-=-=-
165  // reset dest resc name if it was set to begin with
166  if ( !tmp_dest_resc.empty() ) {
167  addKeyVal( &dataObjInp->condInput, DEST_RESC_NAME_KW, tmp_dest_resc.c_str() );
168  }
169 
170  // =-=-=-=-=-=-=-
171  // performing a local replication
172  *transStat = ( transferStat_t* )malloc( sizeof( transferStat_t ) );
173  memset( *transStat, 0, sizeof( transferStat_t ) );
174 
175  // =-=-=-=-=-=-=-
176  // JMC - backport 4609
177  lockType = getValByKey( &dataObjInp->condInput, LOCK_TYPE_KW );
178  if ( lockType != NULL ) {
179  lockFd = irods::server_api_call(
181  rsComm,
182  dataObjInp,
183  NULL,
184  ( void** ) NULL,
185  NULL );
186  if ( lockFd >= 0 ) {
187  /* rm it so it won't be done again causing deadlock */
188  rmKeyVal( &dataObjInp->condInput, LOCK_TYPE_KW );
189  }
190  else {
191  rodsLogError( LOG_ERROR, lockFd,
192  "rsDataObjRepl: lock error for %s. lockType = %s",
193  dataObjInp->objPath, lockType );
194  return lockFd;
195  }
196  }
197  // =-=-=-=-=-=-=-
198 
199  status = _rsDataObjRepl( rsComm, dataObjInp, *transStat, NULL );
200  if ( status < 0 && status != DIRECT_ARCHIVE_ACCESS ) {
201  rodsLog( LOG_NOTICE, "%s - Failed to replicate data object.", __FUNCTION__ );
202  }
203 
204  if ( lockFd > 0 ) {
205  char fd_string[NAME_LEN];
206  snprintf( fd_string, sizeof( fd_string ), "%-d", lockFd );
207  addKeyVal( &dataObjInp->condInput, LOCK_FD_KW, fd_string );
210  rsComm,
211  dataObjInp,
212  NULL,
213  ( void** ) NULL,
214  NULL );
215  }
216 
217  // =-=-=-=-=-=-=-
218  // specifically ignore this error as it should not cause
219  // any issues with replication.
220  if ( status == DIRECT_ARCHIVE_ACCESS ) {
221  return 0;
222  }
223  else {
224  return status;
225  }
226 }
227 
228 int
230  rsComm_t *rsComm,
231  dataObjInp_t *dataObjInp,
232  transferStat_t *transStat,
233  dataObjInfo_t *outDataObjInfo ) {
234  int status;
235  dataObjInfo_t *dataObjInfoHead = NULL;
236  dataObjInfo_t *oldDataObjInfoHead = NULL;
237  dataObjInfo_t *destDataObjInfo = NULL;
238  std::string root_resc_name;
239  ruleExecInfo_t rei;
240  int multiCopyFlag;
241  char *accessPerm;
242  int allFlag;
243  int savedStatus = 0;
244  if ( getValByKey( &dataObjInp->condInput, SU_CLIENT_USER_KW ) != NULL ) {
245  accessPerm = NULL;
246  }
247  else if ( getValByKey( &dataObjInp->condInput, ADMIN_KW ) != NULL ) {
250  }
251  accessPerm = NULL;
252  }
253  else {
254  accessPerm = ACCESS_READ_OBJECT;
255  }
256 
257  initReiWithDataObjInp( &rei, rsComm, dataObjInp );
258  status = applyRule( "acSetMultiReplPerResc", NULL, &rei, NO_SAVE_REI );
260  free(rei.condInputData);
261  if ( strcmp( rei.statusStr, MULTI_COPIES_PER_RESC ) == 0 ) {
262  multiCopyFlag = 1;
263  }
264  else {
265  multiCopyFlag = 0;
266  }
267 
268  /* query rcat for dataObjInfo and sort it */
269  if ( multiCopyFlag ) {
270  status = getDataObjInfo( rsComm, dataObjInp, &dataObjInfoHead,
271  accessPerm, 0 );
272  }
273  else {
274  /* No multiCopy allowed. ignoreCondInput - need to find all copies
275  * to make sure no multiCopy in the same resource */
276  status = getDataObjInfo( rsComm, dataObjInp, &dataObjInfoHead, accessPerm, 1 );
277  }
278 
279  if ( status < 0 ) {
281  "%s: getDataObjInfo for [%s] failed", __FUNCTION__, dataObjInp->objPath );
282  return status;
283  }
284 
285  char* resc_hier = getValByKey( &dataObjInp->condInput, RESC_HIER_STR_KW );
286  char* dest_hier = getValByKey( &dataObjInp->condInput, DEST_RESC_HIER_STR_KW );
287  const bool update_requested = (getValByKey(&dataObjInp->condInput, UPDATE_REPL_KW) != nullptr);
288  bool update_replica = update_requested;
289 
290  if (!dest_hier) {
291  // Get the destination resource that the client specified.
292  dest_hier = getValByKey(&dataObjInp->condInput, DEST_RESC_NAME_KW);
293 
294  // Only use the default resource if the client did not specify a
295  // destination resource to replicate to and they did not set the
296  // update flag [-U].
297  if (!dest_hier && !update_requested) {
298  dest_hier = getValByKey(&dataObjInp->condInput, DEF_RESC_NAME_KW);
299  }
300 
301  if (dest_hier) {
302  std::vector<std::string> hier;
303 
304  for (auto& object : dataObjInfoHead) {
305  hier.clear();
306  irods::string_tokenize(object.rescHier, ";", hier);
307 
308  if (hier[0] == dest_hier) {
309  update_replica = true;
310  dest_hier = object.rescHier;
311  break;
312  }
313 
314  auto e = std::end(hier);
315 
316  if (std::find(std::next(std::begin(hier)), e, dest_hier) != e) {
317  return DIRECT_CHILD_ACCESS;
318  }
319  }
320  }
321  }
322  else {
323  // This block is needed because of compound resources.
324  // In [libcompound.cpp], the function [repl_object] is called when a file
325  // is modified under a compound resource. This triggers a sync. The [repl_object]
326  // function never sets the UPDATE_REPL_KW. Because of that, this block is needed
327  // to correctly determine if an update needs to take place or not.
328  update_replica = irods::is_hier_in_obj_info_list(dest_hier, dataObjInfoHead);
329  }
330 
331  // If [dest_hier] is not null, then the replica matching [dest_hier] will appear
332  // at the head of the list. The list it appears in is determined by whether it is
333  // current or stale.
334  status = sortObjInfoForRepl( &dataObjInfoHead, &oldDataObjInfoHead, 0, resc_hier, dest_hier );
335  if ( status < 0 ) {
336  rodsLog( LOG_NOTICE, "%s - Failed to sort objects for replication.", __FUNCTION__ );
337  return status;
338  }
339 
340  if (update_replica) {
341  // =-=-=-=-=-=-=-
342  // if a resc is specified and it has a stale copy then we should just treat this as an update
343  // also consider the 'update' keyword as that might also have some bearing on updates
344  if ( ( !multiCopyFlag && oldDataObjInfoHead ) || update_requested ) {
345  /* update old repl to new repl */
346  status = _rsDataObjReplUpdate( rsComm, dataObjInp, dataObjInfoHead, oldDataObjInfoHead, transStat );
347 
348  if ( status >= 0 && outDataObjInfo != NULL ) {
349  *outDataObjInfo = *oldDataObjInfoHead; // JMC - possible double free situation
350  outDataObjInfo->next = NULL;
351  }
352  else if ( status < 0 && status != DIRECT_ARCHIVE_ACCESS ) {
353  rodsLog( LOG_NOTICE, "%s - Failed to update replica.", __FUNCTION__ );
354  }
355 
356  freeAllDataObjInfo( dataObjInfoHead );
357  freeAllDataObjInfo( oldDataObjInfoHead );
358 
359  return status;
360  } // repl update
361  }
362 
363  /* if multiCopy allowed, remove old so they won't be overwritten */
364  status = sortObjInfoForRepl( &dataObjInfoHead, &oldDataObjInfoHead, multiCopyFlag, resc_hier, dest_hier );
365  if ( status < 0 ) {
366  rodsLog( LOG_NOTICE, "%s - Failed to sort objects for replication.", __FUNCTION__ );
367  return status;
368  }
369 
370  if ( getValByKey( &dataObjInp->condInput, ALL_KW ) != NULL ) {
371  allFlag = 1;
372  }
373  else {
374  allFlag = 0;
375  }
376 
377  /* query rcat for resource info and sort it */
378  dataObjInp->oprType = REPLICATE_OPR; // JMC - backport 4660
379  status = getRescForCreate( rsComm, dataObjInp, root_resc_name );
380  if ( status < 0 ) {
381  rodsLog( LOG_NOTICE, "%s - Failed to get a resource group for create.", __FUNCTION__ );
382  return status;
383  }
384 
385  if ( multiCopyFlag == 0 ) { // JMC - backport 4594
386  /* if one copy per resource, see if a good copy already exists,
387  * If it does, the copy is returned in destDataObjInfo.
388  * Otherwise, Resources in &myRescGrpInfo are trimmed. Only those
389  ( target resources remained are left in &myRescGrpInfo.
390  * Also, the copies need to be overwritten is returned
391  * in destDataObjInfo. */
392  status = resolveSingleReplCopy( &dataObjInfoHead, &oldDataObjInfoHead,
393  root_resc_name, &destDataObjInfo,
394  &dataObjInp->condInput );
395 
396  if ( status == HAVE_GOOD_COPY ) {
397  // =-=-=-=-=-=-=-
398  // JMC - backport 4450
399  // =-=-=-=-=-=-=-
400  if ( outDataObjInfo != NULL && destDataObjInfo != NULL ) {
401  /* pass back the GOOD_COPY */
402  *outDataObjInfo = *destDataObjInfo;
403  outDataObjInfo->next = NULL;
404  }
405 
406  freeAllDataObjInfo( dataObjInfoHead );
407  freeAllDataObjInfo( oldDataObjInfoHead );
408  freeAllDataObjInfo( destDataObjInfo ); // JMC - backport 4494
409 
410  return 0;
411  }
412 
413  if ( status < 0 ) {
414  freeAllDataObjInfo( dataObjInfoHead );
415  freeAllDataObjInfo( oldDataObjInfoHead );
416  freeAllDataObjInfo( destDataObjInfo ); // JMC - backport 4494
417  rodsLog( LOG_NOTICE, "%s - Failed to resolve a single replication copy.", __FUNCTION__ );
418 
419  return status;
420  }
421  /* NO_GOOD_COPY drop through here */
422  } // if multicopy flag
423 
424  status = applyPreprocRuleForOpen( rsComm, dataObjInp, &dataObjInfoHead );
425  if ( status < 0 ) {
426  freeAllDataObjInfo( dataObjInfoHead );
427  freeAllDataObjInfo( oldDataObjInfoHead );
428  freeAllDataObjInfo( destDataObjInfo ); // JMC - backport 4494
429 
430  return status;
431  }
432 
433  /* If destDataObjInfo is not NULL, we will overwrite it. Otherwise
434  * replicate to myRescGrpInfo */
435  if ( destDataObjInfo != NULL ) {
436 
437  status = _rsDataObjReplUpdate( rsComm, dataObjInp, dataObjInfoHead,
438  destDataObjInfo, transStat );
439  if ( status >= 0 ) {
440  if ( outDataObjInfo != NULL ) {
441  *outDataObjInfo = *destDataObjInfo;
442  outDataObjInfo->next = NULL;
443  }
444  if ( allFlag == 0 ) {
445  freeAllDataObjInfo( dataObjInfoHead );
446  freeAllDataObjInfo( oldDataObjInfoHead );
447  freeAllDataObjInfo( destDataObjInfo ); // JMC - backport 4494
448 
449  return 0;
450  }
451  else { // JMC - backport 4494
452  /* queue destDataObjInfo in &dataObjInfoHead so that stage to cache
453  * can evaluate it */
454  queDataObjInfo( &dataObjInfoHead, destDataObjInfo, 0, 1 );
455  destDataObjInfo = NULL;
456  }
457  }
458  else {
459  savedStatus = status;
460  }
461  }
462 
463  if ( !root_resc_name.empty() ) {
464  /* new replication to the resource group */
465  status = _rsDataObjReplNewCopy( rsComm, dataObjInp, dataObjInfoHead,
466  root_resc_name.c_str(), transStat,
467  outDataObjInfo );
468  if ( status < 0 ) {
469  savedStatus = status;
470  }
471  }
472 
473  freeAllDataObjInfo( dataObjInfoHead );
474  freeAllDataObjInfo( oldDataObjInfoHead );
475 
476  return savedStatus;
477 }
478 
479 /* _rsDataObjRepl - Update existing copies from srcDataObjInfoHead to
480  * destDataObjInfoHead.
481  * Additional input -
482  * dataObjInfo_t *srcDataObjInfoHead _ a link list of the src to be
483 
484  * dataObjInfo_t *destDataObjInfoHead - a link of copies to be updated.
485  * The dataSize in this struct will also be updated.
486  * dataObjInfo_t *oldDataObjInfo - this is for destDataObjInfo is a
487  * COMPOUND_CL resource. If it is, need to find an old copy of
488  * the resource in the same group so that it can be updated first.
489  */
491  rsComm_t* rsComm,
492  dataObjInp_t* dataObjInp,
493  dataObjInfo_t* srcDataObjInfoHead,
494  dataObjInfo_t* destDataObjInfoHead,
495  transferStat_t* transStat ) {
496 
497  // =-=-=-=-=-=-=-
498  //
499  dataObjInfo_t *destDataObjInfo = 0;
500  dataObjInfo_t *srcDataObjInfo = 0;
501  int status = 0;
502  int allFlag = 0;
503  int savedStatus = 0;
504  int replCnt = 0;
505 
506  // =-=-=-=-=-=-=-
507  // set all or single flag
508  if ( getValByKey( &dataObjInp->condInput, ALL_KW ) != NULL ) {
509  allFlag = 1;
510  }
511  else {
512  allFlag = 0;
513  }
514 
515  // =-=-=-=-=-=-=-
516  // cache a copy of the dest resc hier if there is one
517  std::string dst_resc_hier;
518  char* dst_resc_hier_ptr = getValByKey( &dataObjInp->condInput, DEST_RESC_HIER_STR_KW );
519  if ( dst_resc_hier_ptr ) {
520  dst_resc_hier = dst_resc_hier_ptr;
521 
522  }
523 
524  // =-=-=-=-=-=-=-
525  // loop over all the dest data obj info structs
526  transStat->bytesWritten = srcDataObjInfoHead->dataSize;
527  destDataObjInfo = destDataObjInfoHead;
528  while ( destDataObjInfo != NULL ) {
529  // =-=-=-=-=-=-=-
530  // skip invalid data ids
531  if ( destDataObjInfo->dataId == 0 ) {
532  destDataObjInfo = destDataObjInfo->next;
533  continue;
534  }
535 
536  // =-=-=-=-=-=-=-
537  // overwrite a specific destDataObjInfo
538  srcDataObjInfo = srcDataObjInfoHead;
539  while ( srcDataObjInfo != NULL ) {
540  // =-=-=-=-=-=-=-
541  // if the dst hier kw is not set, then set the dest resc hier kw from the dest obj info
542  // as it is already known and we do not want the resc hier making this decision again
543  addKeyVal( &dataObjInp->condInput, DEST_RESC_HIER_STR_KW, destDataObjInfo->rescHier );
544  status = _rsDataObjReplS( rsComm, dataObjInp, srcDataObjInfo, "", destDataObjInfo, 1 );
545 
546  if ( status >= 0 ) {
547  break;
548  }
549  srcDataObjInfo = srcDataObjInfo->next;
550  }
551 
552  if ( status >= 0 ) {
553  transStat->numThreads = dataObjInp->numThreads;
554  if ( allFlag == 0 ) {
555  return 0;
556  }
557  }
558  else {
559  savedStatus = status;
560  replCnt ++;
561  }
562  destDataObjInfo = destDataObjInfo->next;
563  }
564 
565  // =-=-=-=-=-=-=-
566  // repave the key word if it was set before
567  if ( !dst_resc_hier.empty() ) {
568  addKeyVal( &dataObjInp->condInput, DEST_RESC_HIER_STR_KW, dst_resc_hier.c_str() );
569 
570  }
571 
572  return savedStatus;
573 
574 } // _rsDataObjReplUpdate
575 
576 /* _rsDataObjReplNewCopy - Replicate new copies to destRescGrpInfo.
577  * Additional input -
578  * dataObjInfo_t *srcDataObjInfoHead _ a link list of the src to be
579  * replicated. Only one will be picked.
580  * rescGrpInfo_t *destRescGrpInfo - The dest resource info
581  * dataObjInfo_t *oldDataObjInfo - this is for destDataObjInfo is a
582  * COMPOUND_CL resource. If it is, need to find an old copy of
583  * the resource in the same group so that it can be updated first.
584  * dataObjInfo_t *outDataObjInfo - If it is not NULL, output the last
585  * dataObjInfo_t of the new copy.
586  */
587 
588 int
590  rsComm_t *rsComm,
591  dataObjInp_t *dataObjInp,
592  dataObjInfo_t *srcDataObjInfoHead,
593  const char* _root_resc_name,
594  transferStat_t *transStat,
595  dataObjInfo_t *outDataObjInfo ) {
596  // =-=-=-=-=-=-=-
597 
598  dataObjInfo_t *srcDataObjInfo;
599  int status;
600  int allFlag;
601  int savedStatus = 0;
602 
603  if ( getValByKey( &dataObjInp->condInput, ALL_KW ) != NULL ) {
604  allFlag = 1;
605  }
606  else {
607  allFlag = 0;
608  }
609 
610  // =-=-=-=-=-=-=-
611  transStat->bytesWritten = srcDataObjInfoHead->dataSize;
612 
613  srcDataObjInfo = srcDataObjInfoHead;
614  while ( srcDataObjInfo != NULL ) {
615  status = _rsDataObjReplS( rsComm, dataObjInp, srcDataObjInfo, _root_resc_name, outDataObjInfo, 0 );
616  if ( status >= 0 ) {
617  break;
618  }
619  else {
620  savedStatus = status;
621  }
622  srcDataObjInfo = srcDataObjInfo->next;
623  }
624 
625  if ( status >= 0 ) {
626  transStat->numThreads = dataObjInp->numThreads;
627  if ( allFlag == 0 ) {
628  return 0;
629  }
630  }
631  else {
632  savedStatus = status;
633  }
634 
635  return savedStatus;
636 }
637 
638 
639 /* _rsDataObjReplS - replicate a single obj
640  * dataObjInfo_t *srcDataObjInfo - the src to be replicated.
641  * _resc_name - only meaningful if the destDataObj does not exist.
642  * dataObjInfo_t *destDataObjInfo - This can be both input and output.
643  * If destDataObjInfo == NULL, dest is new and no output is required.
644  * If destDataObjInfo != NULL:
645  * If updateFlag == 0, Output only. Output the dataObjInfo
646  * of the replicated copy. This is needed by msiSysReplDataObj and
647  * msiStageDataObj which need a copy of destDataObjInfo.
648  * If updateFlag > 0, the dest repl exists. Need to
649  * update it.
650  */
651 int
653  rsComm_t * rsComm,
654  dataObjInp_t * dataObjInp,
655  dataObjInfo_t * srcDataObjInfo,
656  const char * _root_resc_name,
657  dataObjInfo_t * destDataObjInfo,
658  int updateFlag ) {
659  // =-=-=-=-=-=-=-
660 
661  int status, status1;
662  int l1descInx;
663  openedDataObjInp_t dataObjCloseInp;
664  dataObjInfo_t *myDestDataObjInfo = NULL;
665 
666  l1descInx = dataObjOpenForRepl( rsComm, dataObjInp, srcDataObjInfo,
667  _root_resc_name, destDataObjInfo, updateFlag );
668  if ( l1descInx < 0 ) {
669  return l1descInx;
670  }
671 
672  int single_buff_sz;
673  try {
674  single_buff_sz = irods::get_advanced_setting<const int>(irods::CFG_MAX_SIZE_FOR_SINGLE_BUFFER) * 1024 * 1024;
675  } catch ( const irods::exception& e ) {
676  irods::log(e);
677  return e.code();
678  }
679 
680  if ( L1desc[l1descInx].stageFlag != NO_STAGING ) {
681  status = l3DataStageSync( rsComm, l1descInx );
682  }
683  else if ( L1desc[l1descInx].dataObjInp->numThreads == 0 &&
684  L1desc[L1desc[l1descInx].srcL1descInx].dataObjInfo->dataSize <= single_buff_sz ) {
685  status = l3DataCopySingleBuf( rsComm, l1descInx );
686  }
687  else {
688  status = dataObjCopy( rsComm, l1descInx );
689  }
690 
691  memset( &dataObjCloseInp, 0, sizeof( dataObjCloseInp ) );
692 
693  dataObjCloseInp.l1descInx = l1descInx;
694  /* myDestDataObjInfo = L1desc[l1descInx].dataObjInfo; */
695  L1desc[l1descInx].oprStatus = status;
696  if ( status >= 0 ) {
697  L1desc[l1descInx].bytesWritten = L1desc[l1descInx].dataObjInfo->dataSize;
698  }
699 
700  // Need to propagate the in pdmo flag
701  char* pdmo_kw = getValByKey( &dataObjInp->condInput, IN_PDMO_KW );
702  if ( pdmo_kw != NULL ) {
703  addKeyVal( &dataObjCloseInp.condInput, IN_PDMO_KW, pdmo_kw );
704  }
705 
706  status1 = irsDataObjClose( rsComm, &dataObjCloseInp, &myDestDataObjInfo );
707 
708  if ( destDataObjInfo != NULL ) {
709  if ( destDataObjInfo->dataId <= 0 && myDestDataObjInfo != NULL ) {
710  destDataObjInfo->dataId = myDestDataObjInfo->dataId;
711  destDataObjInfo->replNum = myDestDataObjInfo->replNum;
712  }
713  else if ( myDestDataObjInfo ) {
714  /* the size could change */
715  destDataObjInfo->dataSize = myDestDataObjInfo->dataSize;
716  }
717  }
718 
719  freeDataObjInfo( myDestDataObjInfo );
720  clearKeyVal( &dataObjCloseInp.condInput );
721 
722  if ( status < 0 ) {
723  return status;
724  }
725  else if ( status1 < 0 ) {
726  return status1;
727  }
728  else {
729  return status;
730  }
731 }
732 
733 /* dataObjOpenForRepl - Create/open the dest and open the src
734  */
735 
736 int
738  rsComm_t * rsComm,
739  dataObjInp_t * dataObjInp,
740  dataObjInfo_t * inpSrcDataObjInfo,
741  const char* _root_resc_name,
742  dataObjInfo_t * inpDestDataObjInfo,
743  int updateFlag ) {
744 
745  irods::error resc_err;
746  const char *my_resc_name; // replaces myDestRescInfo
747  if ( _root_resc_name && strlen( _root_resc_name ) ) {
748  my_resc_name = _root_resc_name;
749  }
750  else {
751  my_resc_name = inpDestDataObjInfo->rescName;
752  }
753 
754  resc_err = irods::is_hier_live( my_resc_name );
755  if ( !resc_err.ok() ) {
756  return resc_err.code();
757  }
758 
759  resc_err = irods::is_hier_live( inpSrcDataObjInfo->rescName );
760  if ( !resc_err.ok() ) {
761  return resc_err.code();
762  }
763 
764  dataObjInfo_t * srcDataObjInfo = ( dataObjInfo_t* )calloc( 1, sizeof( dataObjInfo_t ) );
765  if ( NULL == srcDataObjInfo ) { // JMC cppcheck - nullptr
766  rodsLog( LOG_ERROR, "dataObjOpenForRepl - srcDataObjInfo is NULL" );
767  return -1;
768  }
769  *srcDataObjInfo = *inpSrcDataObjInfo;
770 
771  memset( &srcDataObjInfo->condInput, 0, sizeof( srcDataObjInfo->condInput ) );
772  replKeyVal( &inpSrcDataObjInfo->condInput, &srcDataObjInfo->condInput );
773 
774  /* open the dest */
775  dataObjInp_t myDataObjInp = *dataObjInp;
776  replKeyVal( &dataObjInp->condInput, &myDataObjInp.condInput );
777  myDataObjInp.dataSize = inpSrcDataObjInfo->dataSize;
778 
779  int destL1descInx = allocL1desc();
780 
781  if ( destL1descInx < 0 ) {
782  freeDataObjInfo( srcDataObjInfo );
783  return destL1descInx;
784  }
785 
786  // =-=-=-=-=-=-=-=-
787  // use for redirect
788  std::string op_name;
789  dataObjInfo_t * myDestDataObjInfo = ( dataObjInfo_t* )calloc( 1, sizeof( dataObjInfo_t ) );
790 
791  int replStatus;
792  if ( updateFlag > 0 ) {
793  // =-=-=-=-=-=-=-
794  // set a open operation
795  op_name = irods::WRITE_OPERATION;
796 
797  L1desc[destL1descInx].openType = OPEN_FOR_WRITE_TYPE;
798 
799  /* update an existing copy */
800  if ( inpDestDataObjInfo == NULL || inpDestDataObjInfo->dataId <= 0 ) {
801  rodsLog( LOG_ERROR, "dataObjOpenForRepl: dataId of %s copy to be updated not defined",
802  srcDataObjInfo->objPath );
803  freeDataObjInfo( myDestDataObjInfo );
804  freeDataObjInfo( srcDataObjInfo );
806  }
807 
808  /* inherit the replStatus of the src */
809  inpDestDataObjInfo->replStatus = srcDataObjInfo->replStatus;
810  *myDestDataObjInfo = *inpDestDataObjInfo;
811  // =-=-=-=-=-=-=-
812  // JMC :: deep copy of condInput - necessary for preventing a double-free
813  // :: after an irsDataObjClose is called and then a freeDataObjInfo
814  // :: on the copied outgoing dataObjInfo. see _rsDataObjReplS()
815  memset( &myDestDataObjInfo->condInput, 0, sizeof( keyValPair_t ) );
816  replKeyVal( &inpDestDataObjInfo->condInput, &myDestDataObjInfo->condInput );
817 
818  replStatus = srcDataObjInfo->replStatus | OPEN_EXISTING_COPY;
819  addKeyVal( &myDataObjInp.condInput, FORCE_FLAG_KW, "" );
820  myDataObjInp.openFlags |= ( O_TRUNC | O_WRONLY );
821 
822  }
823  else { /* a new copy */
824  // =-=-=-=-=-=-=-
825  // set a creation operation
826  op_name = irods::CREATE_OPERATION;
827 
828  L1desc[destL1descInx].openType = CREATE_TYPE;
829 
830  initDataObjInfoForRepl( myDestDataObjInfo, srcDataObjInfo, _root_resc_name );
831  replStatus = srcDataObjInfo->replStatus;
832  }
833 
834  // =-=-=-=-=-=-=-
835  // call redirect for our operation of choice to request the hier string appropriately
836  std::string hier;
837  char* dst_hier_str = getValByKey( &dataObjInp->condInput, DEST_RESC_HIER_STR_KW );
838  if ( 0 == dst_hier_str ) {
839  // set a repl keyword here so resources can respond accordingly
840  addKeyVal( &dataObjInp->condInput, IN_REPL_KW, "" );
841 
842  dataObjInp_t dest_inp;
843  memset( &dest_inp, 0, sizeof( dest_inp ) );
844  memset( &dest_inp.condInput, 0, sizeof( dest_inp.condInput ) );
845  strncpy( dest_inp.objPath, dataObjInp->objPath, MAX_NAME_LEN );
846  addKeyVal( &( dest_inp.condInput ), RESC_NAME_KW, my_resc_name );
847 
848  irods::error ret = irods::resolve_resource_hierarchy( op_name, rsComm, &dest_inp, hier );
849  clearKeyVal( &dest_inp.condInput );
850 
851  if ( !ret.ok() ) {
852  std::stringstream msg;
853  msg << "failed in irods::resolve_resource_hierarchy for [";
854  msg << dest_inp.objPath << "]";
855  irods::log( PASSMSG( msg.str(), ret ) );
856 
857  freeDataObjInfo( srcDataObjInfo );
858  freeDataObjInfo( myDestDataObjInfo );
859  return ret.code();
860  }
861 
862  addKeyVal( &dataObjInp->condInput, DEST_RESC_HIER_STR_KW, hier.c_str() );
863 
864  }
865  else {
866  hier = dst_hier_str;
867 
868  }
869 
870  // =-=-=-=-=-=-=-
871  // expected by fillL1desc
872  // repave the rescName with the leaf name as it was initialized to the root name
874  parser.set_string( hier );
875  std::string leaf_resc;
876  parser.last_resc(leaf_resc);
877  rstrcpy( myDestDataObjInfo->rescHier, hier.c_str(), MAX_NAME_LEN );
878  rstrcpy( myDestDataObjInfo->rescName, leaf_resc.c_str(), NAME_LEN );
879 
880  rodsLong_t dst_resc_id;
881  resc_mgr.hier_to_leaf_id( hier, dst_resc_id );
882  myDestDataObjInfo->rescId = dst_resc_id;
883 
884  // =-=-=-=-=-=-=-
885  // JMC :: [ ticket 1746 ] this should always be set - this was overwriting the KW
886  // :: in the incoming dataObjInp leaving this here for future consideration if issues arise
887  // addKeyVal( &(myDataObjInp.condInput), RESC_HIER_STR_KW, hier.c_str() ); // <===============
888  fillL1desc( destL1descInx, &myDataObjInp, myDestDataObjInfo, replStatus, srcDataObjInfo->dataSize );
889 
890  dataObjInp_t * l1DataObjInp = L1desc[destL1descInx].dataObjInp;
891  if ( l1DataObjInp->oprType == PHYMV_OPR ) {
892  L1desc[destL1descInx].oprType = PHYMV_DEST;
893  myDestDataObjInfo->replNum = srcDataObjInfo->replNum;
894  myDestDataObjInfo->dataId = srcDataObjInfo->dataId;
895  }
896  else {
897  L1desc[destL1descInx].oprType = REPLICATE_DEST;
898  }
899 
900  // =-=-=-=-=-=-=-
901  // reproduce the stage / sync behavior using keywords rather
902  // than the resource class for use in the compound resource plugin
903  char* stage_kw = getValByKey( &dataObjInp->condInput, STAGE_OBJ_KW );
904  char* sync_kw = getValByKey( &dataObjInp->condInput, SYNC_OBJ_KW );
905  if ( stage_kw ) {
906  L1desc[destL1descInx].stageFlag = STAGE_SRC;
907  }
908  else if ( sync_kw ) {
909  L1desc[destL1descInx].stageFlag = SYNC_DEST;
910  }
911 
912  l1DataObjInp->numThreads = dataObjInp->numThreads =
913  getNumThreads( rsComm, l1DataObjInp->dataSize, l1DataObjInp->numThreads,
914  &dataObjInp->condInput, dst_hier_str, srcDataObjInfo->rescHier, dataObjInp->oprType );
915 
916  int single_buff_sz;
917  try {
918  single_buff_sz = irods::get_advanced_setting<const int>(irods::CFG_MAX_SIZE_FOR_SINGLE_BUFFER) * 1024 * 1024;
919  } catch ( const irods::exception& e ) {
920  irods::log(e);
921  freeDataObjInfo( srcDataObjInfo );
922  return e.code();
923  }
924 
925  if ( ( l1DataObjInp->numThreads > 0 ||
926  l1DataObjInp->dataSize > single_buff_sz ) &&
927  L1desc[destL1descInx].stageFlag == NO_STAGING ) {
928  int status = 0;
929  if ( updateFlag > 0 ) {
930  status = dataOpen( rsComm, destL1descInx );
931  }
932  else {
933  status = getFilePathName( rsComm, myDestDataObjInfo, L1desc[destL1descInx].dataObjInp );
934  if ( status >= 0 ) {
935  status = dataCreate( rsComm, destL1descInx );
936  }
937  }
938 
939  if ( status < 0 ) {
940  freeL1desc( destL1descInx );
941  freeDataObjInfo( srcDataObjInfo );
942  return status;
943  }
944  }
945  else {
946  if ( updateFlag == 0 ) {
947  int status = getFilePathName( rsComm, myDestDataObjInfo, L1desc[destL1descInx].dataObjInp );
948  if ( status < 0 ) {
949  freeL1desc( destL1descInx );
950  freeDataObjInfo( srcDataObjInfo );
951  return status;
952  }
953  }
954  }
955 
956  if ( inpDestDataObjInfo != NULL && updateFlag == 0 ) {
957  /* a new replica */
958  *inpDestDataObjInfo = *myDestDataObjInfo;
959 
960  // =-=-=-=-=-=-=-
961  // JMC :: deep copy of condInput - necessary for preventing a double-free
962  // :: after an irsDataObjClose is called and then a freeDataObjInfo
963  // :: on the copied outgoing dataObjInfo. see _rsDataObjReplS()
964  memset( &inpDestDataObjInfo->condInput, 0, sizeof( keyValPair_t ) );
965  replKeyVal( &myDestDataObjInfo->condInput, &inpDestDataObjInfo->condInput );
966 
967  inpDestDataObjInfo->next = NULL;
968  }
969 
970  // =-=-=-=-=-=-=-
971  // notify the dest resource hierarchy that something is afoot
972  irods::file_object_ptr file_obj(
973  new irods::file_object(
974  rsComm,
975  myDestDataObjInfo ) );
976  irods::error ret = fileNotify(
977  rsComm,
978  file_obj,
980  if ( !ret.ok() ) {
981  std::stringstream msg;
982  msg << "Failed to signal the resource that the data object \"";
983  msg << myDestDataObjInfo->objPath;
984  msg << "\" was modified.";
985  ret = PASSMSG( msg.str(), ret );
986  irods::log( ret );
987  freeDataObjInfo( srcDataObjInfo );
988  return ret.code();
989  }
990 
991  /* open the src */
992  rstrcpy( srcDataObjInfo->rescHier, inpSrcDataObjInfo->rescHier, MAX_NAME_LEN );
993  ret = resc_mgr.hier_to_leaf_id(inpSrcDataObjInfo->rescHier,srcDataObjInfo->rescId);
994  if( !ret.ok() ) {
995  irods::log(PASS(ret));
996  }
997 
998  int srcL1descInx = allocL1desc();
999  if ( srcL1descInx < 0 ) {
1000  freeDataObjInfo( srcDataObjInfo );
1001  return srcL1descInx;
1002  }
1003  fillL1desc( srcL1descInx, &myDataObjInp, srcDataObjInfo, srcDataObjInfo->replStatus, srcDataObjInfo->dataSize );
1004  l1DataObjInp = L1desc[srcL1descInx].dataObjInp;
1005  l1DataObjInp->numThreads = dataObjInp->numThreads;
1006  if ( l1DataObjInp->oprType == PHYMV_OPR ) {
1007  L1desc[srcL1descInx].oprType = PHYMV_SRC;
1008  }
1009  else {
1010  L1desc[srcL1descInx].oprType = REPLICATE_SRC;
1011  }
1012 
1013  if ( getValByKey( &dataObjInp->condInput, PURGE_CACHE_KW ) != NULL ) {
1014  L1desc[srcL1descInx].purgeCacheFlag = 1;
1015  }
1016 
1017  if ( ( l1DataObjInp->numThreads > 0 ||
1018  l1DataObjInp->dataSize > single_buff_sz ) &&
1019  L1desc[destL1descInx].stageFlag == NO_STAGING ) {
1020  openedDataObjInp_t dataObjCloseInp;
1021 
1022  l1DataObjInp->openFlags = O_RDONLY;
1023  int status = dataOpen( rsComm, srcL1descInx );
1024  if ( status < 0 ) {
1025  freeL1desc( srcL1descInx );
1026  memset( &dataObjCloseInp, 0, sizeof( dataObjCloseInp ) );
1027  dataObjCloseInp.l1descInx = destL1descInx;
1028  rsDataObjClose( rsComm, &dataObjCloseInp );
1029  return status;
1030  }
1031  }
1032 
1033  L1desc[destL1descInx].srcL1descInx = srcL1descInx;
1034 
1035  clearKeyVal( &myDataObjInp.condInput );
1036 
1037  return destL1descInx;
1038 }
1039 
1040 int
1041 dataObjCopy( rsComm_t * rsComm, int l1descInx ) {
1042 
1043  dataCopyInp_t dataCopyInp;
1044  bzero( &dataCopyInp, sizeof( dataCopyInp ) );
1045  dataOprInp_t *dataOprInp = &dataCopyInp.dataOprInp;
1046  int srcL1descInx = L1desc[l1descInx].srcL1descInx;
1047  int destL1descInx = l1descInx;
1048 
1049  int srcL3descInx = L1desc[srcL1descInx].l3descInx;
1050  int destL3descInx = L1desc[destL1descInx].l3descInx;
1051 
1052  int srcRemoteFlag;
1053  if ( L1desc[srcL1descInx].remoteZoneHost != NULL ) {
1054  srcRemoteFlag = REMOTE_ZONE_HOST;
1055  }
1056  else {
1057  srcRemoteFlag = FileDesc[srcL3descInx].rodsServerHost->localFlag;
1058  }
1059  int destRemoteFlag;
1060  if ( L1desc[destL1descInx].remoteZoneHost != NULL ) {
1061  destRemoteFlag = REMOTE_ZONE_HOST;
1062  }
1063  else {
1064  destRemoteFlag = FileDesc[destL3descInx].rodsServerHost->localFlag;
1065  }
1066 
1068  if ( srcRemoteFlag == REMOTE_ZONE_HOST &&
1069  destRemoteFlag == REMOTE_ZONE_HOST ) {
1070  /* remote zone to remote zone copy. Have to do L1 level copy */
1071  initDataOprInp( &dataCopyInp.dataOprInp, l1descInx, COPY_TO_REM_OPR );
1072  L1desc[l1descInx].dataObjInp->numThreads = 0;
1073  dataCopyInp.portalOprOut.l1descInx = l1descInx;
1074  int status = singleL1Copy( rsComm, &dataCopyInp );
1075  clearKeyVal( &dataOprInp->condInput );
1076  return status;
1077  }
1078  else if ( srcRemoteFlag != REMOTE_ZONE_HOST &&
1079  destRemoteFlag != REMOTE_ZONE_HOST &&
1080  FileDesc[srcL3descInx].rodsServerHost ==
1081  FileDesc[destL3descInx].rodsServerHost ) {
1082  /* local zone same host copy */
1083  initDataOprInp( &dataCopyInp.dataOprInp, l1descInx, SAME_HOST_COPY_OPR );
1084  /* dataCopyInp.portalOprOut.numThreads is needed by rsDataCopy */
1085  dataCopyInp.portalOprOut.numThreads =
1086  dataCopyInp.dataOprInp.numThreads;
1087  if ( srcRemoteFlag == LOCAL_HOST ) {
1088  addKeyVal( &dataOprInp->condInput, EXEC_LOCALLY_KW, "" );
1089  }
1090 
1091  }
1092  else if ( ( srcRemoteFlag == LOCAL_HOST && destRemoteFlag != LOCAL_HOST ) ||
1093  destRemoteFlag == REMOTE_ZONE_HOST ) {
1094  initDataOprInp( &dataCopyInp.dataOprInp, l1descInx, COPY_TO_REM_OPR );
1095  /* do it locally if numThreads == 0 */
1096  if ( L1desc[l1descInx].dataObjInp->numThreads > 0 ) {
1097  /* copy from local to remote */
1098  /* preProcParaPut to establish portalOprOut without data transfer */
1099  int status = preProcParaPut( rsComm, destL1descInx, &portalOprOut );
1100  if ( status < 0 || NULL == portalOprOut ) { // JMC cppcheck - nullptr
1102  "dataObjCopy: preProcParaPut error for %s",
1103  L1desc[srcL1descInx].dataObjInfo->objPath );
1104  free( portalOprOut );
1105  return status;
1106  }
1107  dataCopyInp.portalOprOut = *portalOprOut;
1108  }
1109  else {
1110  dataCopyInp.portalOprOut.l1descInx = destL1descInx;
1111  }
1112  if ( srcRemoteFlag == LOCAL_HOST ) {
1113  addKeyVal( &dataOprInp->condInput, EXEC_LOCALLY_KW, "" );
1114  }
1115  }
1116  else if ( ( srcRemoteFlag != LOCAL_HOST && destRemoteFlag == LOCAL_HOST ) ||
1117  srcRemoteFlag == REMOTE_ZONE_HOST ) {
1118  /* copy from remote to local */
1119  initDataOprInp( &dataCopyInp.dataOprInp, l1descInx, COPY_TO_LOCAL_OPR );
1120  /* do it locally if numThreads == 0 */
1121  if ( L1desc[l1descInx].dataObjInp->numThreads > 0 ) {
1122  /* preProcParaGet to establish portalOprOut without data transfer */
1123  int status = preProcParaGet( rsComm, srcL1descInx, &portalOprOut );
1124  if ( status < 0 || NULL == portalOprOut ) { // JMC cppcheck - null ptr
1126  "dataObjCopy: preProcParaGet error for %s",
1127  L1desc[srcL1descInx].dataObjInfo->objPath );
1128  free( portalOprOut );
1129  return status;
1130  }
1131  dataCopyInp.portalOprOut = *portalOprOut;
1132  }
1133  else {
1134  dataCopyInp.portalOprOut.l1descInx = srcL1descInx;
1135  }
1136  if ( destRemoteFlag == LOCAL_HOST ) {
1137  addKeyVal( &dataOprInp->condInput, EXEC_LOCALLY_KW, "" );
1138  }
1139  }
1140  else {
1141  /* remote to remote */
1142  initDataOprInp( &dataCopyInp.dataOprInp, l1descInx, COPY_TO_LOCAL_OPR );
1143  /* preProcParaGet only establish &portalOprOut without data transfer */
1144  /* do it locally if numThreads == 0 */
1145  if ( L1desc[l1descInx].dataObjInp->numThreads > 0 ) {
1146  int status = preProcParaGet( rsComm, srcL1descInx, &portalOprOut );
1147  if ( status < 0 || NULL == portalOprOut ) { // JMC cppcheck - null ptr
1149  "dataObjCopy: preProcParaGet error for %s",
1150  L1desc[srcL1descInx].dataObjInfo->objPath );
1151  free( portalOprOut );
1152  return status;
1153  }
1154  dataCopyInp.portalOprOut = *portalOprOut;
1155  }
1156  else {
1157  dataCopyInp.portalOprOut.l1descInx = srcL1descInx;
1158  }
1159  }
1160  /* rsDataCopy - does the physical data transfer */
1161  if ( getValByKey( &L1desc[l1descInx].dataObjInp->condInput,
1162  NO_CHK_COPY_LEN_KW ) != NULL ) {
1163  /* don't check the transfer len */
1164  addKeyVal( &dataOprInp->condInput, NO_CHK_COPY_LEN_KW, "" );
1165  if ( L1desc[l1descInx].dataObjInp->numThreads > 1 ) {
1166  L1desc[l1descInx].dataObjInp->numThreads =
1167  L1desc[srcL1descInx].dataObjInp->numThreads =
1168  dataCopyInp.portalOprOut.numThreads = 1;
1169  }
1170  }
1171  int status = rsDataCopy( rsComm, &dataCopyInp );
1172 
1173  if ( status >= 0 && portalOprOut != NULL &&
1174  L1desc[l1descInx].dataObjInp != NULL ) {
1175  /* update numThreads since it could be changed by remote server */
1177  }
1178  clearKeyVal( &dataOprInp->condInput );
1179  free( portalOprOut );
1180 
1181  return status;
1182 }
1183 
1184 int
1185 l3DataCopySingleBuf( rsComm_t * rsComm, int l1descInx ) {
1186  bytesBuf_t dataBBuf;
1187  int bytesRead, bytesWritten;
1188  int srcL1descInx;
1189 
1190  memset( &dataBBuf, 0, sizeof( bytesBuf_t ) );
1191  srcL1descInx = L1desc[l1descInx].srcL1descInx;
1192 
1193  if ( L1desc[srcL1descInx].dataSize < 0 ) {
1194  rodsLog( LOG_ERROR,
1195  "l3DataCopySingleBuf: dataSize %lld for %s is negative",
1196  L1desc[srcL1descInx].dataSize,
1197  L1desc[srcL1descInx].dataObjInfo->objPath );
1198  return SYS_COPY_LEN_ERR;
1199  }
1200  else if ( L1desc[srcL1descInx].dataSize == 0 ) {
1201  bytesRead = 0;
1202  }
1203  else {
1204  dataBBuf.buf = malloc( L1desc[srcL1descInx].dataSize );
1205  bytesRead = rsL3FileGetSingleBuf( rsComm, &srcL1descInx, &dataBBuf );
1206  }
1207 
1208  if ( bytesRead < 0 ) {
1209  free( dataBBuf.buf ); // JMC cppcheck - leak
1210  return bytesRead;
1211  }
1212  else if ( getValByKey( &L1desc[l1descInx].dataObjInp->condInput,
1213  NO_CHK_COPY_LEN_KW ) != NULL ) {
1214  /* need to update size */
1215  L1desc[l1descInx].dataSize = L1desc[l1descInx].dataObjInp->dataSize
1216  = bytesRead;
1217  }
1218 
1219  bytesWritten = rsL3FilePutSingleBuf( rsComm, &l1descInx, &dataBBuf );
1220 
1221  L1desc[l1descInx].bytesWritten = bytesWritten;
1222 
1223  if ( dataBBuf.buf != NULL ) {
1224  free( dataBBuf.buf );
1225  memset( &dataBBuf, 0, sizeof( bytesBuf_t ) );
1226  }
1227 
1228  if ( bytesWritten != bytesRead ) {
1229  if ( bytesWritten >= 0 ) {
1231  "l3DataCopySingleBuf: l3FilePut error, towrite %d, written %d",
1232  bytesRead, bytesWritten );
1233  return SYS_COPY_LEN_ERR;
1234  }
1235  else {
1236  return bytesWritten;
1237  }
1238  }
1239 
1240 
1241  return 0;
1242 }
1243 
1244 int
1245 l3DataStageSync( rsComm_t * rsComm, int l1descInx ) {
1246  bytesBuf_t dataBBuf;
1247  int srcL1descInx;
1248  int status = 0;
1249 
1250  memset( &dataBBuf, 0, sizeof( bytesBuf_t ) );
1251 
1252  srcL1descInx = L1desc[l1descInx].srcL1descInx;
1253  if ( L1desc[srcL1descInx].dataSize < 0 &&
1254  L1desc[srcL1descInx].dataSize != UNKNOWN_FILE_SZ ) {
1255  rodsLog( LOG_ERROR,
1256  "l3DataStageSync: dataSize %lld for %s is negative",
1257  L1desc[srcL1descInx].dataSize,
1258  L1desc[srcL1descInx].dataObjInfo->objPath );
1259  return SYS_COPY_LEN_ERR;
1260  }
1261  else if ( L1desc[srcL1descInx].dataSize >= 0 ||
1262  L1desc[srcL1descInx].dataSize == UNKNOWN_FILE_SZ ) {
1263  if ( L1desc[l1descInx].stageFlag == SYNC_DEST ) {
1264  /* dest a DO_STAGE type, sync */
1265  status = l3FileSync( rsComm, srcL1descInx, l1descInx );
1266  }
1267  else {
1268  /* src a DO_STAGE type, stage */
1269  status = l3FileStage( rsComm, srcL1descInx, l1descInx );
1270  }
1271  }
1272  else {
1273 
1274  }
1275 
1276  if ( status < 0 ) {
1277  L1desc[l1descInx].bytesWritten = -1;
1278  }
1279  else {
1280  L1desc[l1descInx].bytesWritten = L1desc[l1descInx].dataSize =
1281  L1desc[srcL1descInx].dataSize;
1282  }
1283 
1284  return status;
1285 }
1286 
1287 int
1288 l3FileSync( rsComm_t * rsComm, int srcL1descInx, int destL1descInx ) {
1289  dataObjInfo_t *srcDataObjInfo, *destDataObjInfo;
1290  // int rescTypeInx, cacheRescTypeInx;
1291  fileStageSyncInp_t fileSyncToArchInp;
1292  dataObjInp_t *dataObjInp;
1293  int status;
1294  dataObjInfo_t tmpDataObjInfo;
1295  std::string object_id;
1296 
1297  srcDataObjInfo = L1desc[srcL1descInx].dataObjInfo;
1298  destDataObjInfo = L1desc[destL1descInx].dataObjInfo;
1299 
1300  int dst_create_path = 0;
1301  irods::error err = irods::get_resource_property< int >(
1302  destDataObjInfo->rescId,
1304  dst_create_path );
1305  if ( !err.ok() ) {
1306  irods::log( PASS( err ) );
1307  }
1308 
1309  if ( CREATE_PATH == dst_create_path ) {
1310 
1311  status = chkOrphanFile( rsComm, destDataObjInfo->filePath, destDataObjInfo->rescName, &tmpDataObjInfo );
1312  if ( status == 0 && tmpDataObjInfo.dataId != destDataObjInfo->dataId ) {
1313  /* someone is using it */
1314  char tmp_str[ MAX_NAME_LEN ];
1315  snprintf( tmp_str, MAX_NAME_LEN, "%s.%-u", destDataObjInfo->filePath, irods::getRandom<unsigned int>() );
1316  strncpy( destDataObjInfo->filePath, tmp_str, MAX_NAME_LEN );
1317  }
1318  }
1319 
1320  memset( &fileSyncToArchInp, 0, sizeof( fileSyncToArchInp ) );
1321  dataObjInp = L1desc[destL1descInx].dataObjInp;
1322  fileSyncToArchInp.dataSize = srcDataObjInfo->dataSize;
1323 
1324  // =-=-=-=-=-=-=-
1325  // extract the host location from the resource hierarchy
1326  std::string location;
1327  irods::error ret = irods::get_loc_for_hier_string( srcDataObjInfo->rescHier, location );
1328  if ( !ret.ok() ) {
1329  irods::log( PASSMSG( "l3FileSync - failed in get_loc_for_hier_string", ret ) );
1330  return -1;
1331  }
1332 
1333  rstrcpy( fileSyncToArchInp.addr.hostAddr, location.c_str(), NAME_LEN );
1334 
1335  rstrcpy( fileSyncToArchInp.filename, destDataObjInfo->filePath, MAX_NAME_LEN );
1336  rstrcpy( fileSyncToArchInp.rescHier, destDataObjInfo->rescHier, MAX_NAME_LEN );
1337  rstrcpy( fileSyncToArchInp.objPath, srcDataObjInfo->objPath, MAX_NAME_LEN );
1338  rstrcpy( fileSyncToArchInp.cacheFilename, srcDataObjInfo->filePath, MAX_NAME_LEN );
1339 
1340  // add object id keyword to pass down to resource plugins
1341  object_id = boost::lexical_cast<std::string>(srcDataObjInfo->dataId);
1342  addKeyVal(&fileSyncToArchInp.condInput, DATA_ID_KW, object_id.c_str());
1343 
1344  fileSyncToArchInp.mode = getFileMode( dataObjInp );
1345  fileSyncOut_t* sync_out = 0;
1346  status = rsFileSyncToArch( rsComm, &fileSyncToArchInp, &sync_out );
1347 
1348  if ( status >= 0 &&
1349  CREATE_PATH == dst_create_path &&
1350  NULL != sync_out ) {
1351 
1352  /* path name is created by the resource */
1353  rstrcpy( destDataObjInfo->filePath, sync_out->file_name, MAX_NAME_LEN );
1354  L1desc[destL1descInx].replStatus |= FILE_PATH_HAS_CHG;
1355  }
1356  free( sync_out );
1357  return status;
1358 }
1359 
1360 int
1361 l3FileStage( rsComm_t * rsComm, int srcL1descInx, int destL1descInx ) {
1362  dataObjInfo_t *srcDataObjInfo, *destDataObjInfo;
1363  int mode, status; // JMC - backport 4527
1364 
1365  srcDataObjInfo = L1desc[srcL1descInx].dataObjInfo;
1366  destDataObjInfo = L1desc[destL1descInx].dataObjInfo;
1367 
1368  mode = getFileMode( L1desc[destL1descInx].dataObjInp );
1369 
1370  status = _l3FileStage( rsComm, srcDataObjInfo, destDataObjInfo, mode );
1371 
1372  return status;
1373 }
1374 
1375 int
1376 _l3FileStage( rsComm_t * rsComm, dataObjInfo_t * srcDataObjInfo, // JMC - backport 4527
1377  dataObjInfo_t * destDataObjInfo, int mode ) {
1378  // int rescTypeInx, cacheRescTypeInx;
1379  fileStageSyncInp_t file_stage;
1380  int status;
1381 
1382  std::string resc_loc;
1383  std::string resc_hier( destDataObjInfo->rescHier );
1384 
1385  memset( &file_stage, 0, sizeof( file_stage ) );
1386  file_stage.dataSize = srcDataObjInfo->dataSize;
1387 
1388 
1389  irods::error ret = irods::get_loc_for_hier_string( resc_hier, resc_loc );
1390  if ( !ret.ok() ) {
1391  irods::log( PASSMSG( "irods::get_loc_for_hier_string() failed", ret ) );
1392  return ret.code();
1393  }
1394  rstrcpy( file_stage.addr.hostAddr, resc_loc.c_str(), NAME_LEN );
1395 
1396 
1397  rstrcpy( file_stage.cacheFilename, destDataObjInfo->filePath, MAX_NAME_LEN );
1398  rstrcpy( file_stage.filename, srcDataObjInfo->filePath, MAX_NAME_LEN );
1399  rstrcpy( file_stage.rescHier, destDataObjInfo->rescHier, MAX_NAME_LEN );
1400  rstrcpy( file_stage.objPath, srcDataObjInfo->objPath, MAX_NAME_LEN );
1401 
1402  file_stage.mode = mode;
1403  status = rsFileStageToCache( rsComm, &file_stage );
1404  return status;
1405 }
1406 
1407 /* rsReplAndRequeDataObjInfo - This routine handle the msiSysReplDataObj
1408  * micro-service. It replicates from srcDataObjInfoHead to destRescName
1409  * and support options flags given in flagStr.
1410  * Flags supported are: ALL_KW, RBUDP_TRANSFER_KW, SU_CLIENT_USER_KW
1411  * and UPDATE_REPL_KW. Multiple flags can be input with % as separator.
1412  * The replicated DataObjInfoHead will be put on top of the queue.
1413  */
1414 
1415 int
1417  dataObjInfo_t **srcDataObjInfoHead, char * destRescName, char * flagStr ) {
1418  dataObjInfo_t *dataObjInfoHead, *myDataObjInfo;
1419  transferStat_t transStat;
1420  dataObjInp_t dataObjInp;
1421  char tmpStr[NAME_LEN];
1422  int status;
1423 
1424  dataObjInfoHead = *srcDataObjInfoHead;
1425  myDataObjInfo = ( dataObjInfo_t* )malloc( sizeof( dataObjInfo_t ) );
1426  memset( myDataObjInfo, 0, sizeof( dataObjInfo_t ) );
1427  memset( &dataObjInp, 0, sizeof( dataObjInp_t ) );
1428  memset( &transStat, 0, sizeof( transStat ) );
1429 
1430  if ( flagStr != NULL ) {
1431  if ( strstr( flagStr, ALL_KW ) != NULL ) {
1432  addKeyVal( &dataObjInp.condInput, ALL_KW, "" );
1433  }
1434  if ( strstr( flagStr, RBUDP_TRANSFER_KW ) != NULL ) {
1435  addKeyVal( &dataObjInp.condInput, RBUDP_TRANSFER_KW, "" );
1436  }
1437  if ( strstr( flagStr, SU_CLIENT_USER_KW ) != NULL ) {
1438  addKeyVal( &dataObjInp.condInput, SU_CLIENT_USER_KW, "" );
1439  }
1440  if ( strstr( flagStr, UPDATE_REPL_KW ) != NULL ) {
1441  addKeyVal( &dataObjInp.condInput, UPDATE_REPL_KW, "" );
1442  }
1443  }
1444 
1445  rstrcpy( dataObjInp.objPath, dataObjInfoHead->objPath, MAX_NAME_LEN );
1446  snprintf( tmpStr, NAME_LEN, "%d", dataObjInfoHead->replNum );
1447  addKeyVal( &dataObjInp.condInput, REPL_NUM_KW, tmpStr );
1448  addKeyVal( &dataObjInp.condInput, DEST_RESC_NAME_KW, destRescName );
1449 
1450  status = _rsDataObjRepl( rsComm, &dataObjInp, &transStat,
1451  myDataObjInfo );
1452 
1453  /* fix mem leak */
1454  clearKeyVal( &dataObjInp.condInput );
1455  if ( status >= 0 ) {
1456  status = 1;
1457  /* que the cache copy at the top */
1458  queDataObjInfo( srcDataObjInfoHead, myDataObjInfo, 0, 1 );
1459  }
1460  else {
1461  freeAllDataObjInfo( myDataObjInfo );
1462  }
1463 
1464  return status;
1465 }
1466 
1467 int
1468 stageBundledData( rsComm_t * rsComm, dataObjInfo_t **subfileObjInfoHead ) {
1469  int status;
1470  dataObjInfo_t *dataObjInfoHead = *subfileObjInfoHead;
1471  char *cacheRescName;
1472  dataObjInp_t dataObjInp;
1473  dataObjInfo_t *cacheObjInfo;
1474 
1475  status = unbunAndStageBunfileObj( rsComm, dataObjInfoHead->filePath,
1476  &cacheRescName );
1477 
1478  if ( status < 0 ) {
1479  return status;
1480  }
1481 
1482  /* query the bundle dataObj */
1483  bzero( &dataObjInp, sizeof( dataObjInp ) );
1484  rstrcpy( dataObjInp.objPath, dataObjInfoHead->objPath, MAX_NAME_LEN );
1485  addKeyVal( &dataObjInp.condInput, RESC_NAME_KW, cacheRescName );
1486  status = getDataObjInfo( rsComm, &dataObjInp, &cacheObjInfo, NULL, 0 );
1487  clearKeyVal( &dataObjInp.condInput );
1488  if ( status < 0 ) {
1489  rodsLog( LOG_ERROR,
1490  "unbunAndStageBunfileObj: getDataObjInfo of subfile %s failed.stat=%d",
1491  dataObjInp.objPath, status );
1492  return status;
1493  }
1494  /* que the cache copy at the top */
1495  queDataObjInfo( subfileObjInfoHead, cacheObjInfo, 0, 1 );
1496 
1497 
1498  return status;
1499 }
1500 
1501 int
1502 unbunAndStageBunfileObj( rsComm_t * rsComm, char * bunfileObjPath, char **outCacheRescName ) {
1503  dataObjInfo_t *bunfileObjInfoHead;
1504  dataObjInp_t dataObjInp;
1505  int status;
1506 
1507  /* query the bundle dataObj */
1508  bzero( &dataObjInp, sizeof( dataObjInp ) );
1509  rstrcpy( dataObjInp.objPath, bunfileObjPath, MAX_NAME_LEN );
1510 
1511  status = getDataObjInfo( rsComm, &dataObjInp, &bunfileObjInfoHead, NULL, 1 );
1512  if ( status < 0 ) {
1513  rodsLog( LOG_ERROR,
1514  "unbunAndStageBunfileObj: getDataObjInfo of bunfile %s failed.stat=%d",
1515  dataObjInp.objPath, status );
1516  return status;
1517  }
1518  status = _unbunAndStageBunfileObj( rsComm, &bunfileObjInfoHead, &dataObjInp.condInput,
1519  outCacheRescName, 0 );
1520 
1521  freeAllDataObjInfo( bunfileObjInfoHead );
1522 
1523  return status;
1524 }
1525 
1526 int
1527 _unbunAndStageBunfileObj( rsComm_t * rsComm, dataObjInfo_t **bunfileObjInfoHead, keyValPair_t * condInput,
1528  char **outCacheRescName, int rmBunCopyFlag ) {
1529  int status;
1530  dataObjInp_t dataObjInp;
1531 
1532  bzero( &dataObjInp, sizeof( dataObjInp ) );
1533  bzero( &dataObjInp.condInput, sizeof( dataObjInp.condInput ) );
1534  rstrcpy( dataObjInp.objPath, ( *bunfileObjInfoHead )->objPath, MAX_NAME_LEN );
1535  status = sortObjInfoForOpen( bunfileObjInfoHead, condInput, 0 );
1536 
1537  addKeyVal( &dataObjInp.condInput, RESC_HIER_STR_KW, ( *bunfileObjInfoHead )->rescHier );
1538  if ( status < 0 ) {
1539  return status;
1540  }
1541 
1542  if ( outCacheRescName != NULL ) {
1543  *outCacheRescName = ( *bunfileObjInfoHead )->rescName;
1544  }
1545 
1546  addKeyVal( &dataObjInp.condInput, BUN_FILE_PATH_KW, // JMC - backport 4768
1547  ( *bunfileObjInfoHead )->filePath );
1548  if ( rmBunCopyFlag > 0 ) {
1549  addKeyVal( &dataObjInp.condInput, RM_BUN_COPY_KW, "" );
1550  }
1551  if ( strlen( ( *bunfileObjInfoHead )->dataType ) > 0 ) { // JMC - backport 4664
1552  addKeyVal( &dataObjInp.condInput, DATA_TYPE_KW,
1553  ( *bunfileObjInfoHead )->dataType );
1554  }
1555  status = _rsUnbunAndRegPhyBunfile( rsComm, &dataObjInp,
1556  ( *bunfileObjInfoHead )->rescName );
1557 
1558  return status;
1559 }
allocL1desc
int allocL1desc()
Definition: objDesc.cpp:39
rodsLog
void rodsLog(int level, const char *formatStr,...)
Definition: rodsLog.cpp:86
getValByKey
char * getValByKey(const keyValPair_t *condInput, const char *keyWord)
Definition: rcMisc.cpp:675
DataObjInp::objPath
char objPath[(1024+64)]
Definition: dataObjInpOut.h:66
_l3FileStage
int _l3FileStage(rsComm_t *rsComm, dataObjInfo_t *srcDataObjInfo, dataObjInfo_t *destDataObjInfo, int mode)
Definition: rsDataObjRepl.cpp:1376
fileStageSyncInp_t::mode
int mode
Definition: fileStageToCache.h:10
OpenedDataObjInp
Definition: dataObjInpOut.h:104
L1desc
l1desc_t L1desc[1026]
Definition: irods_server_globals.cpp:29
l3DataStageSync
int l3DataStageSync(rsComm_t *rsComm, int l1descInx)
Definition: rsDataObjRepl.cpp:1245
RESC_NAME_KW
#define RESC_NAME_KW
Definition: rodsKeyWdDef.h:19
LOCK_TYPE_KW
#define LOCK_TYPE_KW
Definition: rodsKeyWdDef.h:115
NULL
#define NULL
Definition: rodsDef.h:70
ACCESS_READ_OBJECT
#define ACCESS_READ_OBJECT
Definition: icatDefines.h:52
rsComm_t
Definition: rcConnect.h:145
rsFileSyncToArch
int rsFileSyncToArch(rsComm_t *rsComm, fileStageSyncInp_t *fileSyncToArchInp, fileSyncOut_t **)
Definition: rsFileSyncToArch.cpp:22
fileStageSyncInp_t::condInput
keyValPair_t condInput
Definition: fileStageToCache.h:18
transferStat_t
Definition: objInfo.h:291
addKeyVal
int addKeyVal(keyValPair_t *condInput, const char *keyWord, const char *value)
Definition: rcMisc.cpp:789
rsL3FilePutSingleBuf
int rsL3FilePutSingleBuf(rsComm_t *rsComm, int *l1descInx, bytesBuf_t *dataObjInBBuf)
Definition: rsL3FilePutSingleBuf.cpp:21
rodsServerHost::localFlag
int localFlag
Definition: rodsConnect.h:68
DataOprInp::condInput
keyValPair_t condInput
Definition: dataObjInpOut.h:134
getRemoteZoneResc.h
STAGE_OBJ_KW
#define STAGE_OBJ_KW
Definition: rodsKeyWdDef.h:228
UNKNOWN_FILE_SZ
#define UNKNOWN_FILE_SZ
Definition: rodsDef.h:92
DataObjInp::openFlags
int openFlags
Definition: dataObjInpOut.h:68
getFilePathName
int getFilePathName(rsComm_t *rsComm, dataObjInfo_t *dataObjInfo, dataObjInp_t *dataObjInp)
Definition: physPath.cpp:87
irods_server_properties.hpp
specColl.hpp
portalOprOut
Definition: dataObjInpOut.h:114
_rsDataObjReplUpdate
int _rsDataObjReplUpdate(rsComm_t *rsComm, dataObjInp_t *dataObjInp, dataObjInfo_t *srcDataObjInfoHead, dataObjInfo_t *destDataObjInfoHead, transferStat_t *transStat)
Definition: rsDataObjRepl.cpp:490
BytesBuf::buf
void * buf
Definition: rodsDef.h:199
rsDataObjOpen.hpp
rodsServerHost::conn
rcComm_t * conn
Definition: rodsConnect.h:64
rsDataObjCreate.hpp
irods.six.next
next
Definition: six.py:518
fileStageSyncInp_t::addr
rodsHostAddr_t addr
Definition: fileStageToCache.h:13
unbunAndRegPhyBunfile.h
DIRECT_CHILD_ACCESS
@ DIRECT_CHILD_ACCESS
Definition: rodsErrorTable.h:765
FILE_PATH_HAS_CHG
#define FILE_PATH_HAS_CHG
Definition: objInfo.h:117
RBUDP_TRANSFER_KW
#define RBUDP_TRANSFER_KW
Definition: rodsKeyWdDef.h:76
irods_stacktrace.hpp
irods::server_api_call
int server_api_call(int _api_index, rsComm_t *_comm, INP_T *_input, bytesBuf_t *_input_buffer, OUT_T **_output, bytesBuf_t *_output_buffer)
Definition: irods_server_api_call.hpp:11
RM_BUN_COPY_KW
#define RM_BUN_COPY_KW
Definition: rodsKeyWdDef.h:87
rsDataCopy.hpp
PASS
#define PASS(prev_error_)
Definition: irods_error.hpp:118
resc_mgr
irods::resource_manager resc_mgr
Definition: irods_resource_manager.cpp:31
l1desc::dataSize
rodsLong_t dataSize
Definition: objDesc.hpp:46
DataObjInp::dataSize
rodsLong_t dataSize
Definition: dataObjInpOut.h:70
irods::resolve_resource_hierarchy
error resolve_resource_hierarchy(const std::string &, rsComm_t *, dataObjInp_t *, std::string &, dataObjInfo_t **_data_obj_info=0)
Definition: irods_resource_redirect.cpp:505
fileStageSyncInp_t::rescHier
char rescHier[(1024+64)]
Definition: fileStageToCache.h:17
OpenedDataObjInp::l1descInx
int l1descInx
Definition: dataObjInpOut.h:105
REPLICATE_SRC
#define REPLICATE_SRC
Definition: dataObjInpOut.h:174
NO_SAVE_REI
#define NO_SAVE_REI
Definition: rodsDef.h:103
preProcParaPut
int preProcParaPut(rsComm_t *rsComm, int l1descInx, portalOprOut_t **portalOprOut)
Definition: rsDataObjPut.cpp:258
REMOTE_HOST
#define REMOTE_HOST
Definition: rodsConnect.h:45
REMOTE_OPEN
#define REMOTE_OPEN
Definition: getRemoteZoneResc.h:5
HAVE_GOOD_COPY
#define HAVE_GOOD_COPY
Definition: dataObjOpr.hpp:24
irods::hierarchy_parser
Definition: irods_hierarchy_parser.hpp:14
l1desc::oprStatus
int oprStatus
Definition: objDesc.hpp:39
irods::is_hier_live
error is_hier_live(const std::string &)
Definition: irods_resource_backport.cpp:234
freeDataObjInfo
int freeDataObjInfo(dataObjInfo_t *dataObjInfo)
Definition: rcMisc.cpp:544
DataObjInfo::rescId
rodsLong_t rescId
Definition: objInfo.h:164
DataOprInp::numThreads
int numThreads
Definition: dataObjInpOut.h:124
fileSyncOut_t::file_name
char file_name[(1024+64)]
Definition: fileStageToCache.h:23
fileNotify
irods::error fileNotify(rsComm_t *_comm, irods::first_class_object_ptr _object, const std::string &)
Definition: fileDriver.cpp:737
l1desc::dataObjInfo
dataObjInfo_t * dataObjInfo
Definition: objDesc.hpp:42
DataObjInfo::rescHier
char rescHier[(1024+64)]
Definition: objInfo.h:132
DataObjInfo::replStatus
int replStatus
Definition: objInfo.h:141
LOCAL_HOST
#define LOCAL_HOST
Definition: rodsConnect.h:44
irods::experimental::filesystem::replica_number
replica_number
Definition: filesystem.hpp:51
DataCopyInp
Definition: dataCopy.h:7
DataObjInfo::condInput
keyValPair_t condInput
Definition: objInfo.h:161
rsFileStageToCache.hpp
irods_resource_backport.hpp
l1desc::oprType
int oprType
Definition: objDesc.hpp:37
fillL1desc
int fillL1desc(int l1descInx, dataObjInp_t *dataObjInp, dataObjInfo_t *dataObjInfo, int replStatus, rodsLong_t dataSize)
Definition: objDesc.cpp:154
rsUnbunAndRegPhyBunfile.hpp
REPL_NUM_KW
#define REPL_NUM_KW
Definition: rodsKeyWdDef.h:30
LOCK_FD_KW
#define LOCK_FD_KW
Definition: rodsKeyWdDef.h:117
l1desc::l3descInx
int l3descInx
Definition: objDesc.hpp:35
PASSMSG
#define PASSMSG(message_, prev_error_)
Definition: irods_error.hpp:119
ALL_KW
#define ALL_KW
Definition: rodsKeyWdDef.h:10
singleL1Copy
int singleL1Copy(rsComm_t *rsComm, dataCopyInp_t *dataCopyInp)
Definition: miscServerFunct.cpp:2641
irods::CFG_MAX_SIZE_FOR_SINGLE_BUFFER
const std::string CFG_MAX_SIZE_FOR_SINGLE_BUFFER("maximum_size_for_single_buffer_in_megabytes")
deploy_schemas_locally.parser
parser
Definition: deploy_schemas_locally.py:59
DataObjInfo::filePath
char filePath[(1024+64)]
Definition: objInfo.h:137
SAME_HOST_COPY_OPR
#define SAME_HOST_COPY_OPR
Definition: dataObjInpOut.h:169
LOG_ERROR
#define LOG_ERROR
Definition: rodsLog.h:43
dataObjCreate.h
irods_resource_redirect.hpp
DataObjInfo::dataId
rodsLong_t dataId
Definition: objInfo.h:143
irods::string_tokenize
void string_tokenize(const std::string &, const std::string &, std::vector< std::string > &)
Definition: irods_string_tokenize.cpp:9
DataObjInfo::next
struct DataObjInfo * next
Definition: objInfo.h:163
DataObjInfo::specColl
specColl_t * specColl
Definition: objInfo.h:158
l1desc::openType
int openType
Definition: objDesc.hpp:38
PURGE_CACHE_KW
#define PURGE_CACHE_KW
Definition: rodsKeyWdDef.h:110
DATA_TYPE_KW
#define DATA_TYPE_KW
Definition: rodsKeyWdDef.h:23
transferStat_t::bytesWritten
rodsLong_t bytesWritten
Definition: objInfo.h:294
l3FileSync
int l3FileSync(rsComm_t *rsComm, int srcL1descInx, int destL1descInx)
Definition: rsDataObjRepl.cpp:1288
rsL3FileGetSingleBuf
int rsL3FileGetSingleBuf(rsComm_t *rsComm, int *l1descInx, bytesBuf_t *dataObjOutBBuf)
Definition: rsL3FileGetSingleBuf.cpp:21
CREATE_TYPE
#define CREATE_TYPE
Definition: dataObjInpOut.h:193
initReiWithDataObjInp
int initReiWithDataObjInp(ruleExecInfo_t *rei, rsComm_t *rsComm, dataObjInp_t *dataObjIn)
Definition: irods_re_structs.cpp:164
irods::experimental::filesystem::client::end
auto end(const collection_iterator &) noexcept -> const collection_iterator
Definition: collection_iterator.hpp:88
PHYMV_OPR
#define PHYMV_OPR
Definition: dataObjInpOut.h:181
fileSyncToArch.h
irsDataObjClose
int irsDataObjClose(rsComm_t *rsComm, openedDataObjInp_t *dataObjCloseInp, dataObjInfo_t **outDataObjInfo)
Definition: rsDataObjClose.cpp:63
DEST_RESC_NAME_KW
#define DEST_RESC_NAME_KW
Definition: rodsKeyWdDef.h:20
RuleExecInfo::condInputData
keyValPair_t * condInputData
Definition: irods_re_structs.hpp:34
irods_string_tokenize.hpp
dataObjCopy
int dataObjCopy(rsComm_t *rsComm, int l1descInx)
Definition: rsDataObjRepl.cpp:1041
irods::experimental::filesystem::client::begin
auto begin(collection_iterator _iter) noexcept -> collection_iterator
Definition: collection_iterator.hpp:83
EXEC_LOCALLY_KW
#define EXEC_LOCALLY_KW
Definition: rodsKeyWdDef.h:12
irods::OPEN_OPERATION
const std::string OPEN_OPERATION("OPEN")
irods::error::code
long long code() const
Definition: irods_error.cpp:194
DATA_ID_KW
#define DATA_ID_KW
Definition: rodsKeyWdDef.h:48
dataObjOpenForRepl
int dataObjOpenForRepl(rsComm_t *rsComm, dataObjInp_t *dataObjInp, dataObjInfo_t *inpSrcDataObjInfo, const char *_root_resc_name, dataObjInfo_t *inpDestDataObjInfo, int updateFlag)
Definition: rsDataObjRepl.cpp:737
LINKED_COLL
@ LINKED_COLL
Definition: objInfo.h:49
transferStat_t::numThreads
int numThreads
Definition: objInfo.h:292
RESC_HIER_STR_KW
#define RESC_HIER_STR_KW
Definition: rodsKeyWdDef.h:225
getNumThreads
int getNumThreads(rsComm_t *rsComm, rodsLong_t dataSize, int inpNumThr, keyValPair_t *condInput, char *destRescName, char *srcRescName, int oprType)
Definition: objDesc.cpp:279
l1desc::stageFlag
int stageFlag
Definition: objDesc.hpp:53
SYNC_OBJ_KW
#define SYNC_OBJ_KW
Definition: rodsKeyWdDef.h:229
SYS_COPY_LEN_ERR
@ SYS_COPY_LEN_ERR
Definition: rodsErrorTable.h:95
PHYMV_DEST
#define PHYMV_DEST
Definition: dataObjInpOut.h:183
rsComm_t::proxyUser
userInfo_t proxyUser
Definition: rcConnect.h:152
freeAllDataObjInfo
int freeAllDataObjInfo(dataObjInfo_t *dataObjInfoHead)
Definition: rcMisc.cpp:561
CAT_INSUFFICIENT_PRIVILEGE_LEVEL
@ CAT_INSUFFICIENT_PRIVILEGE_LEVEL
Definition: rodsErrorTable.h:445
rsReplAndRequeDataObjInfo
int rsReplAndRequeDataObjInfo(rsComm_t *rsComm, dataObjInfo_t **srcDataObjInfoHead, char *destRescName, char *flagStr)
Definition: rsDataObjRepl.cpp:1416
unbunAndStageBunfileObj
int unbunAndStageBunfileObj(rsComm_t *rsComm, char *bunfileObjPath, char **outCacheRescName)
Definition: rsDataObjRepl.cpp:1502
RuleExecInfo::statusStr
char statusStr[(1024+64)]
Definition: irods_re_structs.hpp:20
rsDataObjClose
int rsDataObjClose(rsComm_t *rsComm, openedDataObjInp_t *dataObjCloseInp)
Definition: rsDataObjClose.cpp:56
rsFileStageToCache
int rsFileStageToCache(rsComm_t *rsComm, fileStageSyncInp_t *fileStageToCacheInp)
Definition: rsFileStageToCache.cpp:23
NO_STAGING
#define NO_STAGING
Definition: objInfo.h:256
rsL3FileGetSingleBuf.hpp
DataObjInp
Definition: dataObjInpOut.h:65
rsDataObjClose.hpp
irods::RESOURCE_CREATE_PATH
const std::string RESOURCE_CREATE_PATH("resource_property_create_path")
REMOTE_PRIV_USER_AUTH
#define REMOTE_PRIV_USER_AUTH
Definition: rodsUser.h:35
fileDesc_t::rodsServerHost
rodsServerHost_t * rodsServerHost
Definition: fileOpr.hpp:26
l1desc::dataObjInp
dataObjInp_t * dataObjInp
Definition: objDesc.hpp:41
IN_REPL_KW
#define IN_REPL_KW
Definition: rodsKeyWdDef.h:230
MAX_NAME_LEN
#define MAX_NAME_LEN
Definition: rodsDef.h:61
terminate_irods_processes.e
e
Definition: terminate_irods_processes.py:19
COPY_TO_REM_OPR
#define COPY_TO_REM_OPR
Definition: dataObjInpOut.h:171
fileStageSyncInp_t::filename
char filename[(1024+64)]
Definition: fileStageToCache.h:14
OPEN_EXISTING_COPY
#define OPEN_EXISTING_COPY
Definition: objInfo.h:116
l3DataCopySingleBuf
int l3DataCopySingleBuf(rsComm_t *rsComm, int l1descInx)
Definition: rsDataObjRepl.cpp:1185
rsDataCopy
int rsDataCopy(rsComm_t *rsComm, dataCopyInp_t *dataCopyInp)
Definition: rsDataCopy.cpp:36
rmKeyVal
int rmKeyVal(keyValPair_t *condInput, const char *keyWord)
Definition: rcMisc.cpp:710
COPY_TO_LOCAL_OPR
#define COPY_TO_LOCAL_OPR
Definition: dataObjInpOut.h:170
SU_CLIENT_USER_KW
#define SU_CLIENT_USER_KW
Definition: rodsKeyWdDef.h:86
REMOTE_ZONE_HOST
#define REMOTE_ZONE_HOST
Definition: rodsConnect.h:47
ADMIN_KW
#define ADMIN_KW
Definition: rodsKeyWdDef.h:62
sortObjInfoForOpen
int sortObjInfoForOpen(dataObjInfo_t **dataObjInfoHead, keyValPair_t *condInput, int writeFlag)
Definition: dataObjOpr.cpp:548
fileStageSyncInp_t::dataSize
rodsLong_t dataSize
Definition: fileStageToCache.h:12
DataObjInfo::replNum
int replNum
Definition: objInfo.h:140
LOCAL_PRIV_USER_AUTH
#define LOCAL_PRIV_USER_AUTH
Definition: rodsUser.h:36
UPDATE_REPL_KW
#define UPDATE_REPL_KW
Definition: rodsKeyWdDef.h:75
irods.pypyodbc.status
status
Definition: pypyodbc.py:467
l1desc::srcL1descInx
int srcL1descInx
Definition: objDesc.hpp:50
READ_COLL_PERM
@ READ_COLL_PERM
Definition: objInfo.h:98
rodsLog.h
OPEN_FOR_WRITE_TYPE
#define OPEN_FOR_WRITE_TYPE
Definition: dataObjInpOut.h:195
physPath.hpp
BUN_FILE_PATH_KW
#define BUN_FILE_PATH_KW
Definition: rodsKeyWdDef.h:29
irods::log
void log(const error &)
Definition: irods_log.cpp:13
DEF_RESC_NAME_KW
#define DEF_RESC_NAME_KW
Definition: rodsKeyWdDef.h:21
DEST_RESC_HIER_STR_KW
#define DEST_RESC_HIER_STR_KW
Definition: rodsKeyWdDef.h:226
initDataOprInp
int initDataOprInp(dataOprInp_t *dataOprInp, int l1descInx, int oprType)
Definition: objDesc.cpp:453
LOG_NOTICE
#define LOG_NOTICE
Definition: rodsLog.h:33
rsFileSyncToArch.hpp
icatDefines.h
rsComm_t::clientUser
userInfo_t clientUser
Definition: rcConnect.h:153
dataObjClose.h
getDataObjInfo
int getDataObjInfo(rsComm_t *rsComm, dataObjInp_t *dataObjInp, dataObjInfo_t **dataObjInfoHead, char *accessPerm, int ignoreCondInput)
Definition: dataObjOpr.cpp:87
BytesBuf
Definition: rodsDef.h:197
resolvePathInSpecColl
int resolvePathInSpecColl(rsComm_t *rsComm, char *objPath, specCollPerm_t specCollPerm, int inCachOnly, dataObjInfo_t **dataObjInfo)
Definition: specColl.cpp:644
irods::error
Definition: irods_error.hpp:23
_rsUnbunAndRegPhyBunfile
int _rsUnbunAndRegPhyBunfile(rsComm_t *rsComm, dataObjInp_t *dataObjInp, const char *_resc_name)
Definition: rsUnbunAndRegPhyBunfile.cpp:54
dataOpen
int dataOpen(rsComm_t *rsComm, int l1descInx)
Definition: rsDataObjOpen.cpp:403
miscServerFunct.hpp
DATA_OBJ_T
@ DATA_OBJ_T
Definition: rodsType.h:38
dataObjPut.h
portalOprOut::l1descInx
int l1descInx
Definition: dataObjInpOut.h:116
dataObjRepl.h
irods::is_hier_in_obj_info_list
bool is_hier_in_obj_info_list(const std::string &, dataObjInfo_t *)
Definition: irods_resource_redirect.cpp:425
apiNumber.h
rsDataObjGet.hpp
SYS_UPDATE_REPL_INFO_ERR
@ SYS_UPDATE_REPL_INFO_ERR
Definition: rodsErrorTable.h:171
NO_CHK_COPY_LEN_KW
#define NO_CHK_COPY_LEN_KW
Definition: rodsKeyWdDef.h:108
fileStageSyncInp_t::objPath
char objPath[(1024+64)]
Definition: fileStageToCache.h:16
FileDesc
fileDesc_t FileDesc[1026]
Definition: irods_server_globals.cpp:28
DataCopyInp::portalOprOut
portalOprOut_t portalOprOut
Definition: dataCopy.h:9
OpenedDataObjInp::condInput
keyValPair_t condInput
Definition: dataObjInpOut.h:111
IN_PDMO_KW
#define IN_PDMO_KW
Definition: rodsKeyWdDef.h:227
REPLICATE_OPR
#define REPLICATE_OPR
Definition: dataObjInpOut.h:172
fileStageToCache.h
getRescForCreate
int getRescForCreate(rsComm_t *rsComm, dataObjInp_t *dataObjInp, std::string &_resc_name)
Definition: rsDataObjCreate.cpp:604
FORCE_FLAG_KW
#define FORCE_FLAG_KW
Definition: rodsKeyWdDef.h:13
SYS_REG_OBJ_IN_SPEC_COLL
@ SYS_REG_OBJ_IN_SPEC_COLL
Definition: rodsErrorTable.h:124
rsDataObjRepl
int rsDataObjRepl(rsComm_t *rsComm, dataObjInp_t *dataObjInp, transferStat_t **transStat)
Definition: rsDataObjRepl.cpp:65
irods::resource_manager::hier_to_leaf_id
error hier_to_leaf_id(const std::string &, rodsLong_t &)
Definition: irods_resource_manager.cpp:1082
MULTI_COPIES_PER_RESC
#define MULTI_COPIES_PER_RESC
Definition: dataObjOpr.hpp:32
_rsDataObjReplNewCopy
int _rsDataObjReplNewCopy(rsComm_t *rsComm, dataObjInp_t *dataObjInp, dataObjInfo_t *srcDataObjInfoHead, const char *_root_resc_name, transferStat_t *transStat, dataObjInfo_t *outDataObjInfo)
Definition: rsDataObjRepl.cpp:589
irods::file_object
Definition: irods_file_object.hpp:19
rodsServerHost
Definition: rodsConnect.h:62
DataObjInfo::rescName
char rescName[64]
Definition: objInfo.h:131
stageBundledData
int stageBundledData(rsComm_t *rsComm, dataObjInfo_t **subfileObjInfoHead)
Definition: rsDataObjRepl.cpp:1468
DataObjInp::oprType
int oprType
Definition: dataObjInpOut.h:72
rodsHostAddr_t::hostAddr
char hostAddr[256]
Definition: rodsDef.h:297
DATA_OBJ_UNLOCK_AN
#define DATA_OBJ_UNLOCK_AN
Definition: apiNumber.h:109
l1desc::bytesWritten
rodsLong_t bytesWritten
Definition: objDesc.hpp:45
_unbunAndStageBunfileObj
int _unbunAndStageBunfileObj(rsComm_t *rsComm, dataObjInfo_t **bunfileObjInfoHead, keyValPair_t *condInput, char **outCacheRescName, int rmBunCopyFlag)
Definition: rsDataObjRepl.cpp:1527
preProcParaGet
int preProcParaGet(rsComm_t *rsComm, int l1descInx, portalOprOut_t **portalOprOut)
Definition: rsDataObjGet.cpp:251
irods::file_object_ptr
boost::shared_ptr< file_object > file_object_ptr
Definition: irods_file_object.hpp:145
RuleExecInfo
Definition: irods_re_structs.hpp:18
portalOprOut::numThreads
int numThreads
Definition: dataObjInpOut.h:117
DataOprInp
Definition: dataObjInpOut.h:122
rodsLogError
void rodsLogError(int level, int errCode, char *formatStr,...)
Definition: rodsLog.cpp:422
_rcDataObjRepl
int _rcDataObjRepl(rcComm_t *conn, dataObjInp_t *dataObjInp, transferStat_t **transferStat)
Definition: rcDataObjRepl.cpp:104
irods::WRITE_OPERATION
const std::string WRITE_OPERATION("WRITE")
PHYMV_SRC
#define PHYMV_SRC
Definition: dataObjInpOut.h:182
getAndConnRemoteZone
int getAndConnRemoteZone(rsComm_t *rsComm, dataObjInp_t *dataObjInp, rodsServerHost_t **rodsServerHost, char *remotZoneOpr)
Definition: rodsConnect.cpp:784
_rsDataObjRepl
int _rsDataObjRepl(rsComm_t *rsComm, dataObjInp_t *dataObjInp, transferStat_t *transStat, dataObjInfo_t *outDataObjInfo)
Definition: rsDataObjRepl.cpp:229
irods_server_api_call.hpp
rsL3FilePutSingleBuf.hpp
applyRule
int applyRule(char *inAction, msParamArray_t *inMsParamArray, ruleExecInfo_t *rei, int reiSaveFlag)
Definition: irods_re_structs.cpp:65
resolveSingleReplCopy
int resolveSingleReplCopy(dataObjInfo_t **dataObjInfoHead, dataObjInfo_t **oldDataObjInfoHead, const std::string &_resc_name, dataObjInfo_t **destDataObjInfo, keyValPair_t *condInput)
Definition: dataObjOpr.cpp:1301
resource.hpp
irods::exception
Definition: irods_exception.hpp:15
sortObjInfoForRepl
int sortObjInfoForRepl(dataObjInfo_t **dataObjInfoHead, dataObjInfo_t **oldDataObjInfoHead, int deleteOldFlag, const char *resc_hier, const char *dst_resc_hier)
Definition: dataObjOpr.cpp:945
USER_INCOMPATIBLE_PARAMS
@ USER_INCOMPATIBLE_PARAMS
Definition: rodsErrorTable.h:288
getFileMode
int getFileMode(dataObjInp_t *dataObjInp)
Definition: physPath.cpp:48
l3FileStage
int l3FileStage(rsComm_t *rsComm, int srcL1descInx, int destL1descInx)
Definition: rsDataObjRepl.cpp:1361
DataCopyInp::dataOprInp
dataOprInp_t dataOprInp
Definition: dataCopy.h:8
l1desc::replStatus
int replStatus
Definition: objDesc.hpp:48
CREATE_PATH
@ CREATE_PATH
Definition: objInfo.h:267
STAGE_SRC
#define STAGE_SRC
Definition: objInfo.h:257
fileStageSyncInp_t
Definition: fileStageToCache.h:9
dataObjOpen.h
DataObjInfo::objPath
char objPath[(1024+64)]
Definition: objInfo.h:130
irods_random.hpp
userInfo_t::authInfo
authInfo_t authInfo
Definition: rodsUser.h:70
DATA_OBJ_LOCK_AN
#define DATA_OBJ_LOCK_AN
Definition: apiNumber.h:126
initDataObjInfoForRepl
int initDataObjInfoForRepl(dataObjInfo_t *destDataObjInfo, dataObjInfo_t *srcDataObjInfo, const char *_resc_name)
Definition: objDesc.cpp:544
mode
int mode
Definition: filesystem.cpp:104
rstrcpy
char * rstrcpy(char *dest, const char *src, int maxLen)
Definition: stringOpr.cpp:51
dataObjOpr.hpp
dataObjLock.h
DataObjInp::condInput
keyValPair_t condInput
Definition: dataObjInpOut.h:74
irods::get_loc_for_hier_string
error get_loc_for_hier_string(const std::string &_hier, std::string &_loc)
Definition: irods_resource_backport.cpp:633
REPLICATE_DEST
#define REPLICATE_DEST
Definition: dataObjInpOut.h:173
NAME_LEN
#define NAME_LEN
Definition: rodsDef.h:55
l3FilePutSingleBuf.h
freeL1desc
int freeL1desc(int l1descInx)
Definition: objDesc.cpp:124
KeyValPair
Definition: objInfo.h:120
fileSyncOut_t
Definition: fileStageToCache.h:22
authInfo_t::authFlag
int authFlag
Definition: rodsUser.h:42
clearKeyVal
int clearKeyVal(keyValPair_t *condInput)
Definition: rcMisc.cpp:1047
irods::error::ok
bool ok()
Definition: irods_error.cpp:258
DataObjInfo
Definition: objInfo.h:129
fileStageSyncInp_t::cacheFilename
char cacheFilename[(1024+64)]
Definition: fileStageToCache.h:15
dataObjTrim.h
DataObjInfo::dataSize
rodsLong_t dataSize
Definition: objInfo.h:134
irods::CREATE_OPERATION
const std::string CREATE_OPERATION("CREATE")
DataObjInp::numThreads
int numThreads
Definition: dataObjInpOut.h:71
queDataObjInfo
int queDataObjInfo(dataObjInfo_t **dataObjInfoHead, dataObjInfo_t *dataObjInfo, int singleInfoFlag, int topFlag)
Definition: rcMisc.cpp:581
l3FileGetSingleBuf.h
dataObjGet.h
DIRECT_ARCHIVE_ACCESS
@ DIRECT_ARCHIVE_ACCESS
Definition: rodsErrorTable.h:763
rsDataObjPut.hpp
replKeyVal
int replKeyVal(const keyValPair_t *srcCondInput, keyValPair_t *destCondInput)
Definition: rcMisc.cpp:739
rsDataObjRepl.hpp
irods_log.hpp
_rsDataObjReplS
int _rsDataObjReplS(rsComm_t *rsComm, dataObjInp_t *dataObjInp, dataObjInfo_t *srcDataObjInfo, const char *_root_resc_name, dataObjInfo_t *destDataObjInfo, int updateFlag)
Definition: rsDataObjRepl.cpp:652
rodsLong_t
long long rodsLong_t
Definition: rodsType.h:32
applyPreprocRuleForOpen
int applyPreprocRuleForOpen(rsComm_t *rsComm, dataObjInp_t *dataObjInp, dataObjInfo_t **dataObjInfoHead)
Definition: rsDataObjOpen.cpp:546
SYNC_DEST
#define SYNC_DEST
Definition: objInfo.h:258
objMetaOpr.hpp
SpecColl::collClass
specCollClass_t collClass
Definition: objInfo.h:77
dataCreate
int dataCreate(rsComm_t *rsComm, int l1descInx)
Definition: rsDataObjCreate.cpp:476
chkOrphanFile
int chkOrphanFile(rsComm_t *rsComm, char *filePath, const char *rescName, dataObjInfo_t *dataObjInfo)
Definition: dataObjOpr.cpp:1099
l1desc::purgeCacheFlag
int purgeCacheFlag
Definition: objDesc.hpp:54