irods  4.2.8
About: iRODS (the integrated Rule Oriented Data System) is a distributed data-management system for creating data grids, digital libraries, persistent archives, and real-time data systems.
  Fossies Dox: irods-4.2.8.tar.gz  ("unofficial" and yet experimental doxygen-generated source code documentation)  

rcPortalOpr.cpp
Go to the documentation of this file.
1 
3 #include "rcMisc.h"
4 #include "rcPortalOpr.h"
5 #include "dataObjClose.h"
6 #include "dataObjOpen.h"
7 #include "dataObjWrite.h"
8 #include "dataObjRead.h"
9 #include "dataObjLseek.h"
10 #include "fileLseek.h"
11 #include "dataObjOpr.hpp"
12 #include "rodsLog.h"
13 #include "rcGlobalExtern.h"
14 #include "sockComm.h"
15 
16 // =-=-=-=-=-=-=-
17 #include "irods_stacktrace.hpp"
20 
21 #include <openssl/md5.h>
22 
23 #include <boost/thread/thread.hpp>
24 #include <boost/thread/scoped_thread.hpp>
25 #include <boost/thread/mutex.hpp>
26 #include <boost/thread/condition.hpp>
27 #include <iomanip>
28 #include <fstream>
29 #include <boost/filesystem/operations.hpp>
30 #include <boost/filesystem/convenience.hpp>
31 using namespace boost::filesystem;
32 
33 
34 
35 int
36 sendTranHeader( int sock, int oprType, int flags, rodsLong_t offset,
37  rodsLong_t length ) {
38  transferHeader_t myHeader;
39  int retVal;
40 
41  myHeader.oprType = htonl( oprType );
42  myHeader.flags = htonl( flags );
43  myHtonll( offset, ( rodsLong_t * ) &myHeader.offset );
44  myHtonll( length, ( rodsLong_t * ) &myHeader.length );
45 
46  retVal = myWrite( sock, ( void * ) &myHeader, sizeof( myHeader ), NULL );
47 
48  if ( retVal != sizeof( myHeader ) ) {
50  "sendTranHeader: toWrite = %d, written = %d",
51  sizeof( myHeader ), retVal );
52  if ( retVal < 0 ) {
53  return retVal;
54  }
55  else {
56  return SYS_COPY_LEN_ERR;
57  }
58  }
59  else {
60  return 0;
61  }
62 }
63 
64 int
65 rcvTranHeader( int sock, transferHeader_t *myHeader ) {
66  int retVal;
67  transferHeader_t tmpHeader;
68 
69  retVal = myRead( sock, ( void * ) &tmpHeader, sizeof( tmpHeader ),
70  NULL, NULL );
71 
72  if ( retVal != sizeof( tmpHeader ) ) {
74  "rcvTranHeader: toread = %d, read = %d",
75  sizeof( tmpHeader ), retVal );
76  if ( retVal < 0 ) {
77  return retVal;
78  }
79  else {
80  return SYS_COPY_LEN_ERR;
81  }
82  }
83 
84  myHeader->oprType = htonl( tmpHeader.oprType );
85  myHeader->flags = htonl( tmpHeader.flags );
86  myNtohll( tmpHeader.offset, &myHeader->offset );
87  myNtohll( tmpHeader.length, &myHeader->length );
88 
89  return 0;
90 }
91 
92 int
93 fillBBufWithFile( rcComm_t *conn, bytesBuf_t *myBBuf, char *locFilePath,
94  rodsLong_t dataSize ) {
95  int in_fd, status;
96 
97  rodsEnv rods_env;
98  if ( int status = getRodsEnv( &rods_env ) ) {
99  rodsLog( LOG_ERROR, "getRodsEnv failed in %s with status %s", __FUNCTION__, status );
100  return status;
101  }
102  int single_buff_sz = rods_env.irodsMaxSizeForSingleBuffer * 1024 * 1024;
103 
104  if ( dataSize > single_buff_sz ) {
106  "fillBBufWithFile: dataSize %lld too large", dataSize );
107  return USER_FILE_TOO_LARGE;
108  }
109 
110 #ifdef windows_platform
111  in_fd = iRODSNt_bopen( locFilePath, O_RDONLY, 0 );
112 #else
113  in_fd = open( locFilePath, O_RDONLY, 0 );
114 #endif
115  if ( in_fd < 0 ) { /* error */
116  status = UNIX_FILE_OPEN_ERR - errno;
118  "cannot open file %s", locFilePath, status );
119  return status;
120  }
121 
122 
123  myBBuf->buf = malloc( dataSize );
124  myBBuf->len = dataSize;
125  conn->transStat.bytesWritten = dataSize;
126 
127  status = myRead( in_fd, myBBuf->buf, ( int ) dataSize,
128  NULL, NULL );
129 
130  close( in_fd );
131 
132  return status;
133 }
134 
135 int
137  char *locFilePath, char *objPath, rodsLong_t dataSize ) {
139  int retVal = 0;
140 
141  if ( portalOprOut == NULL || portalOprOut->numThreads <= 0 ) {
143  "putFileToPortal: invalid portalOprOut" );
144  return SYS_INVALID_PORTAL_OPR;
145  }
146 
147  const int numThreads = portalOprOut->numThreads;
148 
149  portList_t *myPortList = &portalOprOut->portList;
150 
152  for ( int i = 0; i < portalOprOut->numThreads; i++ ) {
153  const int sock = connectToRhostPortal( myPortList->hostAddr,
154  myPortList->portNum, myPortList->cookie, myPortList->windowSize );
155  if ( sock >= 0 ) {
156  close( sock );
157  }
158  }
160  "putFileToPortal: numThreads %d too large",
162  return SYS_INVALID_PORTAL_OPR;
163  }
164 
165  initFileRestart( conn, locFilePath, objPath, dataSize,
167  memset( myInput, 0, sizeof( myInput ) );
168 
169  if ( numThreads == 1 ) {
170  const int sock = connectToRhostPortal( myPortList->hostAddr,
171  myPortList->portNum, myPortList->cookie, myPortList->windowSize );
172  if ( sock < 0 ) {
173  return sock;
174  }
175 #ifdef windows_platform
176  const int in_fd = iRODSNt_bopen( locFilePath, O_RDONLY, 0 );
177 #else
178  const int in_fd = open( locFilePath, O_RDONLY, 0 );
179 #endif
180  if ( in_fd < 0 ) { /* error */
181  retVal = UNIX_FILE_OPEN_ERR - errno;
182  rodsLogError( LOG_ERROR, retVal,
183  "cannot open file %s", locFilePath, retVal );
184  close( sock );
185  return retVal;
186  }
187 
188  fillRcPortalTransferInp( conn, &myInput[0], sock, in_fd, 0 );
189 
190  rcPartialDataPut( &myInput[0] );
191  if ( myInput[0].status < 0 ) {
192  return myInput[0].status;
193  }
194  else {
195  if ( dataSize <= 0 || myInput[0].bytesWritten == dataSize ) {
196  return 0;
197  }
198  else {
200  "putFileToPortal: bytesWritten %lld dataSize %lld mismatch",
201  myInput[0].bytesWritten, dataSize );
202  return SYS_COPY_LEN_ERR;
203  }
204  }
205  }
206  else {
207  rodsLong_t totalWritten = 0;
208  std::unique_ptr<boost::scoped_thread<>> tid[MAX_NUM_CONFIG_TRAN_THR];
209  memset( tid, 0, sizeof( tid ) );
210 
211  for ( int i = 0; i < numThreads; i++ ) {
212  const int sock = connectToRhostPortal( myPortList->hostAddr,
213  myPortList->portNum, myPortList->cookie, myPortList->windowSize );
214  if ( sock < 0 ) {
215  return sock;
216  }
217  const int in_fd = open( locFilePath, O_RDONLY, 0 );
218  if ( in_fd < 0 ) { /* error */
219  retVal = UNIX_FILE_OPEN_ERR - errno;
220  rodsLogError( LOG_ERROR, retVal,
221  "cannot open file %s", locFilePath, retVal );
222  close( sock );
223  continue;
224  }
225  fillRcPortalTransferInp( conn, &myInput[i], sock, in_fd, i );
226  try {
227  tid[i] = std::make_unique<boost::scoped_thread<>>( boost::thread( rcPartialDataPut, &myInput[i] ) );
228  }
229  catch ( const boost::thread_resource_error& ) {
230  rodsLog( LOG_ERROR, "boost encountered thread_resource_error on constructing thread." );
232  }
233  }
234  if ( retVal < 0 ) {
235  return retVal;
236  }
237 
238  for ( int i = 0; i < numThreads; i++ ) {
239  if ( tid[i] != 0 ) {
240  try {
241  tid[i]->join();
242  }
243  catch ( const boost::thread_resource_error& ) {
244  rodsLog( LOG_ERROR, "boost encountered thread_resource_error on joining thread." );
246  }
247  }
248  totalWritten += myInput[i].bytesWritten;
249  if ( myInput[i].status < 0 ) {
250  retVal = myInput[i].status;
251  }
252  }
253  if ( retVal < 0 ) {
254  return retVal;
255  }
256  else {
257  if ( dataSize <= 0 || totalWritten == dataSize ) {
258  if ( gGuiProgressCB != NULL ) {
259  gGuiProgressCB( &conn->operProgress );
260  }
261  return 0;
262  }
263  else {
265  "putFileToPortal: totalWritten %lld dataSize %lld mismatch",
266  totalWritten, dataSize );
267  return SYS_COPY_LEN_ERR;
268  }
269  }
270  }
271 }
272 
273 int
275  int destFd, int srcFd, int threadNum ) {
276  if ( myInput == NULL ) {
278  }
279 
280  myInput->conn = conn;
281  myInput->destFd = destFd;
282  myInput->srcFd = srcFd;
283  myInput->threadNum = threadNum;
284  memcpy( myInput->shared_secret, conn->shared_secret, NAME_LEN );
285 
286  return 0;
287 }
288 
289 
290 void
292  int destFd = 0;
293  int srcFd = 0;
294  transferStat_t *myTransStat = 0;
295  rodsLong_t curOffset = 0;
296  rcComm_t *conn = 0;
297  fileRestartInfo_t* info = 0;
298  int threadNum = 0;
299 
300  if ( myInput == NULL ) {
302  "rcPartialDataPut: NULL input" );
303  return;
304  }
305 
306  conn = myInput->conn;
307  info = &conn->fileRestart.info;
308  threadNum = myInput->threadNum;
309 
310  myTransStat = &myInput->conn->transStat;
311 
312  destFd = myInput->destFd;
313  srcFd = myInput->srcFd;
314 
315  myInput->bytesWritten = 0;
316 
317  if ( gGuiProgressCB != NULL ) {
318  conn->operProgress.flag = 1;
319  }
320 
321  // =-=-=-=-=-=-=-
322  // flag to determine if we need to use encryption
323  bool use_encryption_flg =
324  ( myInput->conn->negotiation_results ==
326 
327  // =-=-=-=-=-=-=-
328  // get the client side Env to determine
329  // encryption parameters
330  rodsEnv rods_env;
331  int status = getRodsEnv( &rods_env );
332  if ( status < 0 ) {
333  printf( "Failed to get irodsEnv" );
334  return;
335  }
336 
337  // =-=-=-=-=-=-=-
338  // create an encryption context
339  int iv_size = 0;
343  irods::buffer_crypt::array_t shared_secret;
344  irods::buffer_crypt crypt(
345  rods_env.rodsEncryptionKeySize,
346  rods_env.rodsEncryptionSaltSize,
348  rods_env.rodsEncryptionAlgorithm );
349 
350  // =-=-=-=-=-=-=-
351  // set iv size
352  if ( use_encryption_flg ) {
353  iv_size = crypt.key_size();
354  shared_secret.assign(
355  &myInput->shared_secret[0],
356  &myInput->shared_secret[iv_size] );
357  }
358 
359  // =-=-=-=-=-=-=-
360  // allocate a buffer for writing
361  rodsLong_t trans_buff_sz = ( rodsLong_t )rods_env.irodsTransBufferSizeForParaTrans * 1024 * 1024;
362  rodsLong_t buf_size = 2 * trans_buff_sz * sizeof( unsigned char );
363  unsigned char* buf = ( unsigned char* )malloc( buf_size );
364  transferHeader_t myHeader;
365 
366  while ( myInput->status >= 0 ) {
367  rodsLong_t toPut;
368 
369  myInput->status = rcvTranHeader( destFd, &myHeader );
370 
371  if ( myInput->status < 0 ) {
372  break;
373  }
374 
375  if ( myHeader.oprType == DONE_OPR ) {
376  break;
377  }
378  if ( myHeader.offset != curOffset ) {
379  curOffset = myHeader.offset;
380  if ( lseek( srcFd, curOffset, SEEK_SET ) < 0 ) {
381  myInput->status = UNIX_FILE_LSEEK_ERR - errno;
382  rodsLogError( LOG_ERROR, myInput->status,
383  "rcPartialDataPut: lseek to %lld error, status = %d",
384  curOffset, myInput->status );
385  break;
386  }
387  if ( info->numSeg > 0 ) { /* file restart */
388  info->dataSeg[threadNum].offset = curOffset;
389  }
390  }
391 
392  toPut = myHeader.length;
393  while ( toPut > 0 ) {
394  rodsLong_t toRead;
395  int bytesRead, bytesWritten;
396 
397  if ( toPut > trans_buff_sz ) {
398  toRead = trans_buff_sz;
399  }
400  else {
401  toRead = toPut;
402  }
403 
404  bytesRead = myRead(
405  srcFd,
406  buf,
407  toRead,
408  &bytesRead,
409  NULL );
410  if ( bytesRead != toRead ) {
411  myInput->status = SYS_COPY_LEN_ERR - errno;
412  rodsLogError( LOG_ERROR, myInput->status,
413  "rcPartialDataPut: toPut %lld, bytesRead %d",
414  toPut, bytesRead );
415  break;
416  }
417 
418  // =-=-=-=-=-=-=-
419  // compute an iv for this particular transmission and use
420  // it to encrypt this buffer
421  int new_size = bytesRead;
422  if ( use_encryption_flg ) {
423  irods::error ret = crypt.initialization_vector( iv );
424  if ( !ret.ok() ) {
425  ret = PASS( ret );
426  printf( "%s", ret.result().c_str() );
427  break;
428  }
429 
430  // =-=-=-=-=-=-=-
431  // encrypt
432  in_buf.assign(
433  &buf[0],
434  &buf[ bytesRead ] );
435  ret = crypt.encrypt(
436  shared_secret,
437  iv,
438  in_buf,
439  cipher );
440  if ( !ret.ok() ) {
441  ret = PASS( ret );
442  printf( "%s", ret.result().c_str() );
443  break;
444  }
445 
446  // =-=-=-=-=-=-=-
447  // capture the iv with the cipher text
448  memset( buf, 0, buf_size );
449  std::copy(
450  iv.begin(),
451  iv.end(),
452  &buf[0] );
453  std::copy(
454  cipher.begin(),
455  cipher.end(),
456  &buf[iv_size] );
457 
458  new_size = iv_size + cipher.size();
459 
460  // =-=-=-=-=-=-=-
461  // need to send the incoming size as encryption might change
462  // the size of the data from the writen values
463  bytesWritten = myWrite(
464  destFd,
465  &new_size,
466  sizeof( int ),
467  &bytesWritten );
468 
469  }
470 
471  // =-=-=-=-=-=-=-
472  // then write the actual buffer
473  bytesWritten = myWrite(
474  destFd,
475  buf,
476  new_size,
477  &bytesWritten );
478 
479  if ( bytesWritten != new_size ) {
480  myInput->status = SYS_COPY_LEN_ERR - errno;
481  rodsLogError( LOG_ERROR, myInput->status,
482  "rcPartialDataPut: toWrite %d, bytesWritten %d, errno = %d",
483  bytesRead, bytesWritten, errno );
484  break;
485  }
486 
487  toPut -= bytesRead;
488  if ( info->numSeg > 0 ) { /* file restart */
489  info->dataSeg[threadNum].len += bytesRead;
490  conn->fileRestart.writtenSinceUpdated += bytesRead;
491  if ( threadNum == 0 && conn->fileRestart.writtenSinceUpdated >=
493  int status;
494  /* time to write to the restart file */
496  &conn->fileRestart.info );
497  if ( status < 0 ) {
499  "rcPartialDataPut: writeLfRestartFile for %s, status = %d",
500  conn->fileRestart.info.fileName, status );
501  }
503  }
504  }
505 
506  } // while
507 
508  curOffset += myHeader.length;
509  myInput->bytesWritten += myHeader.length;
510  /* should lock this. But window browser is the only one using it */
511  myTransStat->bytesWritten += myHeader.length;
512  /* should lock this. but it is info only */
513  if ( gGuiProgressCB != NULL ) {
514  conn->operProgress.curFileSizeDone += myHeader.length;
515  if ( myInput->threadNum == 0 ) {
516  gGuiProgressCB( &conn->operProgress );
517  }
518  }
519  }
520 
521 
522  free( buf );
523  close( srcFd );
524  mySockClose( destFd );
525 }
526 
527 int
528 putFile( rcComm_t *conn, int l1descInx, char *locFilePath, char *objPath,
529  rodsLong_t dataSize ) {
530  int in_fd, status;
531  bytesBuf_t dataObjWriteInpBBuf;
532  openedDataObjInp_t dataObjWriteInp;
533  int bytesWritten;
534  rodsLong_t totalWritten = 0;
535  int bytesRead;
536  int progressCnt = 0;
537  fileRestartInfo_t *info = &conn->fileRestart.info;
538  rodsLong_t lastUpdateSize = 0;
539 
540 
541 #ifdef windows_platform
542  in_fd = iRODSNt_bopen( locFilePath, O_RDONLY, 0 );
543 #else
544  in_fd = open( locFilePath, O_RDONLY, 0 );
545 #endif
546  if ( in_fd < 0 ) { /* error */
547  status = UNIX_FILE_OPEN_ERR - errno;
549  "cannot open file %s", locFilePath, status );
550  return status;
551  }
552 
553  rodsEnv rods_env;
554  if ( int status = getRodsEnv( &rods_env ) ) {
555  rodsLog( LOG_ERROR, "getRodsEnv failed in %s with status %s", __FUNCTION__, status );
556  return status;
557  }
558  size_t trans_buff_sz = rods_env.irodsTransBufferSizeForParaTrans * 1024 * 1024;
559 
560  bzero( &dataObjWriteInp, sizeof( dataObjWriteInp ) );
561  dataObjWriteInpBBuf.buf = malloc( trans_buff_sz + 1 );
562  dataObjWriteInpBBuf.len = 0;
563  dataObjWriteInp.l1descInx = l1descInx;
564  initFileRestart( conn, locFilePath, objPath, dataSize, 1 );
565 
566  if ( gGuiProgressCB != NULL ) {
567  conn->operProgress.flag = 1;
568  }
569 
570  while ( ( dataObjWriteInpBBuf.len =
571  myRead( in_fd, dataObjWriteInpBBuf.buf, trans_buff_sz,
572  &bytesRead, NULL ) ) > 0 ) {
573  /* Write to the data object */
574 
575  dataObjWriteInp.len = dataObjWriteInpBBuf.len;
576  bytesWritten = rcDataObjWrite( conn, &dataObjWriteInp,
577  &dataObjWriteInpBBuf );
578  if ( bytesWritten < dataObjWriteInp.len ) {
580  "putFile: Read %d bytes, Wrote %d bytes.\n ",
581  dataObjWriteInp.len, bytesWritten );
582  free( dataObjWriteInpBBuf.buf );
583  close( in_fd );
584  return SYS_COPY_LEN_ERR;
585  }
586  else {
587  totalWritten += bytesWritten;
588  conn->transStat.bytesWritten = totalWritten;
589  if ( info->numSeg > 0 ) { /* file restart */
590  info->dataSeg[0].len += bytesWritten;
591  if ( totalWritten - lastUpdateSize >= RESTART_FILE_UPDATE_SIZE ) {
592  /* time to write to the restart file */
594  &conn->fileRestart.info );
595  if ( status < 0 ) {
597  "putFile: writeLfRestartFile for %s, status = %d",
598  locFilePath, status );
599  free( dataObjWriteInpBBuf.buf );
600  close( in_fd );
601  return status;
602  }
603  lastUpdateSize = totalWritten;
604  }
605  }
606  if ( gGuiProgressCB != NULL ) {
607  if ( progressCnt >= ( MAX_PROGRESS_CNT - 1 ) ) {
609  ( ( MAX_PROGRESS_CNT - 1 ) * trans_buff_sz + bytesWritten );
610  gGuiProgressCB( &conn->operProgress );
611  progressCnt = 0;
612  }
613  else {
614  progressCnt ++;
615  }
616  }
617  }
618  }
619 
620  free( dataObjWriteInpBBuf.buf );
621  close( in_fd );
622 
623  if ( dataSize <= 0 || totalWritten == dataSize ) {
624  if ( gGuiProgressCB != NULL ) {
626  gGuiProgressCB( &conn->operProgress );
627  }
628  return 0;
629  }
630  else {
632  "putFile: totalWritten %lld dataSize %lld mismatch",
633  totalWritten, dataSize );
634  return SYS_COPY_LEN_ERR;
635  }
636 }
637 
638 int
639 getIncludeFile( rcComm_t *conn, bytesBuf_t *dataObjOutBBuf, char *locFilePath ) {
640  int status, out_fd, bytesWritten;
641 
642  if ( strcmp( locFilePath, STDOUT_FILE_NAME ) == 0 ) {
643  if ( dataObjOutBBuf->len <= 0 ) {
644  return 0;
645  }
646  bytesWritten = fwrite( dataObjOutBBuf->buf, dataObjOutBBuf->len,
647  1, stdout );
648  if ( bytesWritten == 1 ) {
649  bytesWritten = dataObjOutBBuf->len;
650  }
651  }
652  else {
653 #ifdef windows_platform
654  out_fd = iRODSNt_bopen( locFilePath, O_WRONLY | O_CREAT | O_TRUNC, 0640 );
655 #else
656  out_fd = open( locFilePath, O_WRONLY | O_CREAT | O_TRUNC, 0640 );
657 #endif
658  if ( out_fd < 0 ) { /* error */
659  status = UNIX_FILE_OPEN_ERR - errno;
661  "cannot open file %s", locFilePath, status );
662  return status;
663  }
664 
665  if ( dataObjOutBBuf->len <= 0 ) {
666  close( out_fd );
667  return 0;
668  }
669 
670  bytesWritten = myWrite( out_fd, dataObjOutBBuf->buf,
671  dataObjOutBBuf->len, NULL );
672 
673  close( out_fd );
674  }
675  if ( bytesWritten != dataObjOutBBuf->len ) {
677  "getIncludeFile: Read %d bytes, Wrote %d bytes. errno = %d\n ",
678  dataObjOutBBuf->len, bytesWritten, errno );
679  return SYS_COPY_LEN_ERR;
680  }
681  else {
682  conn->transStat.bytesWritten = bytesWritten;
683  return 0;
684  }
685 }
686 
687 int
688 getFile( rcComm_t *conn, int l1descInx, char *locFilePath, char *objPath,
689  rodsLong_t dataSize ) {
690  int out_fd, status;
691  bytesBuf_t dataObjReadInpBBuf;
692  openedDataObjInp_t dataObjReadInp;
693  int bytesWritten, bytesRead;
694  rodsLong_t totalWritten = 0;
695  int progressCnt = 0;
696  fileRestartInfo_t *info = &conn->fileRestart.info;
697  rodsLong_t lastUpdateSize = 0;
698 
699  if ( strcmp( locFilePath, STDOUT_FILE_NAME ) == 0 ) {
700  /* streaming to stdout */
701  out_fd = 1;
702  }
703  else {
704 #ifdef windows_platform
705  out_fd = iRODSNt_bopen( locFilePath, O_WRONLY | O_CREAT | O_TRUNC, 0640 );
706 #else
707  out_fd = open( locFilePath, O_WRONLY | O_CREAT | O_TRUNC, 0640 );
708 #endif
709  }
710 
711  if ( out_fd < 0 ) { /* error */
712  status = UNIX_FILE_OPEN_ERR - errno;
714  "cannot open file %s", locFilePath, status );
715  return status;
716  }
717 
718  rodsEnv rods_env;
719  if ( int status = getRodsEnv( &rods_env ) ) {
720  rodsLog( LOG_ERROR, "getRodsEnv failed in %s with status %s", __FUNCTION__, status );
721  return status;
722  }
723  size_t trans_buff_sz = rods_env.irodsTransBufferSizeForParaTrans * 1024 * 1024;
724 
725  bzero( &dataObjReadInp, sizeof( dataObjReadInp ) );
726  dataObjReadInpBBuf.buf = malloc( trans_buff_sz + 1 );
727  dataObjReadInpBBuf.len = dataObjReadInp.len = trans_buff_sz;
728  dataObjReadInp.l1descInx = l1descInx;
729  initFileRestart( conn, locFilePath, objPath, dataSize, 1 );
730 
731  if ( gGuiProgressCB != NULL ) {
732  conn->operProgress.flag = 1;
733  }
734 
735  while ( ( bytesRead = rcDataObjRead( conn, &dataObjReadInp,
736  &dataObjReadInpBBuf ) ) > 0 ) {
737 
738  if ( out_fd == 1 ) {
739  bytesWritten = fwrite( dataObjReadInpBBuf.buf, bytesRead,
740  1, stdout );
741  if ( bytesWritten == 1 ) {
742  bytesWritten = bytesRead;
743  }
744  }
745  else {
746  bytesWritten = myWrite( out_fd, dataObjReadInpBBuf.buf,
747  bytesRead, NULL );
748  }
749 
750  if ( bytesWritten != bytesRead ) {
752  "getFile: Read %d bytes, Wrote %d bytes.\n ",
753  bytesRead, bytesWritten );
754  free( dataObjReadInpBBuf.buf );
755  if ( out_fd != 1 ) {
756  close( out_fd );
757  }
758  return SYS_COPY_LEN_ERR;
759  }
760  else {
761  totalWritten += bytesWritten;
762  conn->transStat.bytesWritten = totalWritten;
763  if ( info->numSeg > 0 ) { /* file restart */
764  info->dataSeg[0].len += bytesWritten;
765  if ( totalWritten - lastUpdateSize >= RESTART_FILE_UPDATE_SIZE ) {
766  /* time to write to the restart file */
768  &conn->fileRestart.info );
769  if ( status < 0 ) {
771  "getFile: writeLfRestartFile for %s, status = %d",
772  locFilePath, status );
773  free( dataObjReadInpBBuf.buf );
774  if ( out_fd != 1 ) {
775  close( out_fd );
776  }
777  return status;
778  }
779  lastUpdateSize = totalWritten;
780  }
781  }
782  if ( gGuiProgressCB != NULL ) {
783  if ( progressCnt >= ( MAX_PROGRESS_CNT - 1 ) ) {
785  ( ( MAX_PROGRESS_CNT - 1 ) * trans_buff_sz + bytesWritten );
786  gGuiProgressCB( &conn->operProgress );
787  progressCnt = 0;
788  }
789  else {
790  progressCnt ++;
791  }
792  }
793  }
794  }
795 
796  free( dataObjReadInpBBuf.buf );
797  if ( out_fd != 1 ) {
798  close( out_fd );
799  }
800 
801  if ( bytesRead >= 0 ) {
802  if ( gGuiProgressCB != NULL ) {
804  gGuiProgressCB( &conn->operProgress );
805  }
806 
807  // rcDataObjRead may return 0 in an error case, we need
808  // to ensure the total written is matching the total size
809  // otherwise we are also in an error case
810 
811  // in the instance of streaming a single thread, and a specific
812  // resource is specified, the dataSize will be 0 in the case where
813  // we possibly got a bad copy intentionally
814  if ( 0 == dataSize || totalWritten == dataSize ) {
815  return 0;
816 
817  }
818  else {
819  return SYS_COPY_LEN_ERR;
820 
821  }
822  }
823  else {
825  "getFile: totalWritten %lld dataSize %lld mismatch",
826  totalWritten, dataSize );
827  return bytesRead;
828 
829  }
830 }
831 
832 int
834  char *locFilePath, char *objPath, rodsLong_t dataSize ) {
836  int retVal = 0;
837 
838  if ( portalOprOut == NULL || portalOprOut->numThreads <= 0 ) {
840  "getFileFromPortal: invalid portalOprOut" );
841  return SYS_INVALID_PORTAL_OPR;
842  }
843 
844  const int numThreads = portalOprOut->numThreads;
845  portList_t *myPortList = &portalOprOut->portList;
846 
848  /* drain the connection or it will be stuck */
849  for ( int i = 0; i < numThreads; i++ ) {
850  const int sock = connectToRhostPortal( myPortList->hostAddr,
851  myPortList->portNum, myPortList->cookie, myPortList->windowSize );
852  if ( sock >= 0 ) {
853  close( sock );
854  }
855  }
857  "getFileFromPortal: numThreads %d too large", numThreads );
858  return SYS_INVALID_PORTAL_OPR;
859  }
860 
861  memset( myInput, 0, sizeof( myInput ) );
862 
863  initFileRestart( conn, locFilePath, objPath, dataSize,
865 
866  if ( numThreads == 1 ) {
867  const int sock = connectToRhostPortal( myPortList->hostAddr,
868  myPortList->portNum, myPortList->cookie, myPortList->windowSize );
869  if ( sock < 0 ) {
870  return sock;
871  }
872 #ifdef windows_platform
873  const int out_fd = iRODSNt_bopen( locFilePath, O_WRONLY | O_CREAT | O_TRUNC, 0640 );
874 #else
875  const int out_fd = open( locFilePath, O_WRONLY | O_CREAT | O_TRUNC, 0640 );
876 #endif
877  if ( out_fd < 0 ) { /* error */
878  retVal = UNIX_FILE_OPEN_ERR - errno;
879  rodsLogError( LOG_ERROR, retVal,
880  "cannot open file %s", locFilePath, retVal );
881  close( sock );
882  return retVal;
883  }
884  fillRcPortalTransferInp( conn, &myInput[0], out_fd, sock, 0640 );
885  rcPartialDataGet( &myInput[0] );
886  if ( myInput[0].status < 0 ) {
887  return myInput[0].status;
888  }
889  else {
890  if ( dataSize <= 0 || myInput[0].bytesWritten == dataSize ) {
891  return 0;
892  }
893  else {
895  "getFileFromPortal:bytesWritten %lld dataSize %lld mismatch",
896  myInput[0].bytesWritten, dataSize );
897  return SYS_COPY_LEN_ERR;
898  }
899  }
900  }
901  else {
902  rodsLong_t totalWritten = 0;
903  std::unique_ptr<boost::scoped_thread<>> tid[MAX_NUM_CONFIG_TRAN_THR];
904  memset( tid, 0, sizeof( tid ) );
905 
906  for ( int i = 0; i < numThreads; i++ ) {
907  const int sock = connectToRhostPortal( myPortList->hostAddr,
908  myPortList->portNum, myPortList->cookie, myPortList->windowSize );
909  int out_fd = -1;
910  if ( sock < 0 ) {
911  return sock;
912  }
913  if ( i == 0 ) {
914  out_fd = open( locFilePath, O_WRONLY | O_CREAT | O_TRUNC, 0640 );
915  }
916  else {
917  out_fd = open( locFilePath, O_WRONLY, 0640 );
918  }
919  if ( out_fd < 0 ) { /* error */
920  retVal = UNIX_FILE_OPEN_ERR - errno;
921  rodsLogError( LOG_ERROR, retVal,
922  "cannot open file %s", locFilePath, retVal );
923  CLOSE_SOCK( sock );
924  continue;
925  }
926  fillRcPortalTransferInp( conn, &myInput[i], out_fd, sock, i );
927  try {
928  tid[i] = std::make_unique<boost::scoped_thread<>>( boost::thread( rcPartialDataGet, &myInput[i] ) );
929  }
930  catch ( const boost::thread_resource_error& ) {
931  rodsLog( LOG_ERROR, "boost encountered thread_resource_error on constructing thread." );
933  }
934  }
935 
936  if ( retVal < 0 ) {
937  return retVal;
938  }
939 
940  for ( int i = 0; i < numThreads; i++ ) {
941  if ( tid[i] != 0 ) {
942  try {
943  tid[i]->join();
944  }
945  catch ( const boost::thread_resource_error& ) {
946  rodsLog( LOG_ERROR, "boost encountered thread_resource_error on joining thread." );
948  }
949  }
950  totalWritten += myInput[i].bytesWritten;
951  if ( myInput[i].status < 0 ) {
952  retVal = myInput[i].status;
953  }
954  }
955  if ( retVal < 0 ) {
956  return retVal;
957  }
958  else {
959  if ( dataSize <= 0 || totalWritten == dataSize ) {
960  if ( gGuiProgressCB != NULL ) {
961  gGuiProgressCB( &conn->operProgress );
962  }
963  return 0;
964  }
965  else {
967  "getFileFromPortal: totalWritten %lld dataSize %lld mismatch",
968  totalWritten, dataSize );
969  return SYS_COPY_LEN_ERR;
970  }
971  }
972  }
973 }
974 
975 void
977  transferHeader_t myHeader;
978  int destFd;
979  int srcFd;
980  unsigned char *buf;
981  transferStat_t *myTransStat;
982  rodsLong_t curOffset = 0;
983  rcComm_t *conn;
984  fileRestartInfo_t *info;
985  int threadNum;
986  if ( myInput == NULL ) {
988  "rcPartialDataGet: NULL input" );
989  return;
990  }
991 
992  conn = myInput->conn;
993  info = &conn->fileRestart.info;
994  threadNum = myInput->threadNum;
995 
996  myTransStat = &myInput->conn->transStat;
997 
998  destFd = myInput->destFd;
999  srcFd = myInput->srcFd;
1000 
1001 
1002  myInput->bytesWritten = 0;
1003 
1004  if ( gGuiProgressCB != NULL ) {
1005  conn = myInput->conn;
1006  conn->operProgress.flag = 1;
1007  }
1008 
1009  // =-=-=-=-=-=-=-
1010  // flag to determine if we need to use encryption
1011  bool use_encryption_flg =
1012  ( myInput->conn->negotiation_results ==
1014 
1015  // =-=-=-=-=-=-=-
1016  // get the client side Env to determine
1017  // encryption parameters
1018  rodsEnv rods_env;
1019  int status = getRodsEnv( &rods_env );
1020  if ( status < 0 ) {
1021  printf( "Failed to get irodsEnv" );
1022  return;
1023  }
1024 
1025  // =-=-=-=-=-=-=-
1026  // create an encryption context
1027  int iv_size = 0;
1032  irods::buffer_crypt::array_t shared_secret;
1033  irods::buffer_crypt crypt(
1034  rods_env.rodsEncryptionKeySize,
1035  rods_env.rodsEncryptionSaltSize,
1036  rods_env.rodsEncryptionNumHashRounds,
1037  rods_env.rodsEncryptionAlgorithm );
1038 
1039  // =-=-=-=-=-=-=-
1040  // set iv size
1041  if ( use_encryption_flg ) {
1042  iv_size = crypt.key_size();
1043  shared_secret.assign(
1044  &myInput->shared_secret[0],
1045  &myInput->shared_secret[iv_size] );
1046  }
1047 
1048  rodsLong_t trans_buff_sz = ( rodsLong_t )rods_env.irodsTransBufferSizeForParaTrans * 1024 * 1024;
1049  rodsLong_t buf_size = ( 2 * trans_buff_sz ) * sizeof( unsigned char );
1050  buf = ( unsigned char* )malloc( buf_size );
1051 
1052  while ( myInput->status >= 0 ) {
1053 
1054  myInput->status = rcvTranHeader( srcFd, &myHeader );
1055 
1056  if ( myInput->status < 0 ) {
1057  break;
1058  }
1059 
1060  if ( myHeader.oprType == DONE_OPR ) {
1061  break;
1062  }
1063  if ( myHeader.offset != curOffset ) {
1064  curOffset = myHeader.offset;
1065  if ( lseek( destFd, curOffset, SEEK_SET ) < 0 ) {
1066  myInput->status = UNIX_FILE_LSEEK_ERR - errno;
1067  rodsLogError( LOG_ERROR, myInput->status,
1068  "rcPartialDataGet: lseek to %lld error, status = %d",
1069  curOffset, myInput->status );
1070  break;
1071  }
1072  if ( info->numSeg > 0 ) { /* file restart */
1073  info->dataSeg[threadNum].offset = curOffset;
1074  }
1075  }
1076 
1077  rodsLong_t toGet = myHeader.length;
1078  while ( toGet > 0 ) {
1079  int toRead, bytesRead, bytesWritten;
1080 
1081  if ( toGet > trans_buff_sz ) {
1082  toRead = trans_buff_sz;
1083  }
1084  else {
1085  toRead = toGet;
1086  }
1087 
1088  // =-=-=-=-=-=-=-
1089  // read the incoming size as it might differ due to encryption
1090  int new_size = toRead;
1091  if ( use_encryption_flg ) {
1092  bytesRead = myRead(
1093  srcFd,
1094  &new_size,
1095  sizeof( int ),
1096  NULL, NULL );
1097  if ( bytesRead != sizeof( int ) ) {
1098  rodsLog(
1099  LOG_ERROR,
1100  "_partialDataGet:Bytes Read != %d",
1101  sizeof( int ) );
1102  break;
1103  }
1104  }
1105 
1106  // =-=-=-=-=-=-=-
1107  // now read the provided number of bytes as suggested by
1108  // the incoming size
1109  bytesRead = myRead(
1110  srcFd,
1111  buf,
1112  new_size,
1113  &bytesRead,
1114  NULL );
1115  if ( bytesRead != new_size ) {
1116  myInput->status = SYS_COPY_LEN_ERR - errno;
1117  rodsLogError( LOG_ERROR, myInput->status,
1118  "rcPartialDataGet: toGet %lld, bytesRead %d",
1119  toGet, bytesRead );
1120  break;
1121  }
1122 
1123  // =-=-=-=-=-=-=-
1124  // if using encryption, strip off the iv
1125  // and decrypt before writing
1126  int plain_size = bytesRead;
1127  if ( use_encryption_flg ) {
1128  this_iv.assign(
1129  &buf[ 0 ],
1130  &buf[ iv_size ] );
1131  cipher.assign(
1132  &buf[ iv_size ],
1133  &buf[ new_size ] );
1134  irods::error ret = crypt.decrypt(
1135  shared_secret,
1136  this_iv,
1137  cipher,
1138  plain );
1139  if ( !ret.ok() ) {
1140  irods::log( PASS( ret ) );
1141  myInput->status = SYS_COPY_LEN_ERR;
1142  break;
1143  }
1144 
1145  memset( buf, 0, buf_size );
1146  std::copy(
1147  plain.begin(),
1148  plain.end(),
1149  &buf[0] );
1150  plain_size = plain.size();
1151 
1152  }
1153 
1154  bytesWritten = myWrite(
1155  destFd,
1156  buf,
1157  plain_size,
1158  &bytesWritten );
1159  if ( bytesWritten != plain_size ) {
1160  myInput->status = SYS_COPY_LEN_ERR - errno;
1161  rodsLogError( LOG_ERROR, myInput->status,
1162  "rcPartialDataGet: toWrite %d, bytesWritten %d",
1163  plain_size, bytesWritten );
1164  break;
1165  }
1166 
1167  toGet -= bytesWritten;
1168 
1169  if ( info->numSeg > 0 ) { /* file restart */
1170  info->dataSeg[threadNum].len += bytesWritten;
1171  conn->fileRestart.writtenSinceUpdated += bytesWritten;
1172  if ( threadNum == 0 && conn->fileRestart.writtenSinceUpdated >=
1174  int status;
1175  /* time to write to the restart file */
1177  &conn->fileRestart.info );
1178  if ( status < 0 ) {
1179  rodsLog( LOG_ERROR,
1180  "rcPartialDataGet: writeLfRestartFile for %s, status = %d",
1181  conn->fileRestart.info.fileName, status );
1182  }
1183  conn->fileRestart.writtenSinceUpdated = 0;
1184  }
1185  }
1186  }
1187  curOffset += myHeader.length;
1188  myInput->bytesWritten += myHeader.length;
1189  /* should lock this. But window browser is the only one using it */
1190  myTransStat->bytesWritten += myHeader.length;
1191  /* should lock this. but it is info only */
1192  if ( gGuiProgressCB != NULL ) {
1193  conn->operProgress.curFileSizeDone += myHeader.length;
1194  if ( myInput->threadNum == 0 ) {
1195  gGuiProgressCB( &conn->operProgress );
1196  }
1197  }
1198  }
1199 
1200  free( buf );
1201  close( destFd );
1202  CLOSE_SOCK( srcFd );
1203 }
1204 
1205 /* putFileToPortalRbudp - The client side of putting a file using
1206  * Rbudp. If locFilePath is NULL, the local file has already been opned
1207  * and locFd should be used. If sendRate and packetSize are 0, it will
1208  * try to set it based on env and default.
1209  */
1212  char* locFilePath,
1213  int locFd,
1214  int veryVerbose,
1215  int sendRate,
1216  int packetSize ) {
1217 
1218  portList_t *myPortList;
1219  int status;
1220  rbudpSender_t rbudpSender;
1221  int mysendRate, mypacketSize;
1222  char *tmpStr;
1223 
1224  if ( portalOprOut == NULL || portalOprOut->numThreads != 1 ) {
1225  rodsLog( LOG_ERROR,
1226  "putFileToPortal: invalid portalOprOut" );
1227  return SYS_INVALID_PORTAL_OPR;
1228  }
1229 
1230  myPortList = &portalOprOut->portList;
1231 
1232  bzero( &rbudpSender, sizeof( rbudpSender ) );
1233  status = initRbudpClient( &rbudpSender.rbudpBase, myPortList );
1234  if ( status < 0 ) {
1235  rodsLog( LOG_ERROR,
1236  "putFileToPortalRbudp: initRbudpClient error for %s",
1237  myPortList->hostAddr );
1238  return status;
1239  }
1240  rbudpSender.rbudpBase.verbose = veryVerbose;
1241  if ( ( mysendRate = sendRate ) < 1 &&
1242  ( ( tmpStr = getenv( RBUDP_SEND_RATE_KW ) ) == NULL ||
1243  ( mysendRate = atoi( tmpStr ) ) < 1 ) ) {
1244  mysendRate = DEF_UDP_SEND_RATE;
1245  }
1246  if ( ( mypacketSize = packetSize ) < 1 &&
1247  ( ( tmpStr = getenv( RBUDP_PACK_SIZE_KW ) ) == NULL ||
1248  ( mypacketSize = atoi( tmpStr ) ) < 1 ) ) {
1249  mypacketSize = DEF_UDP_PACKET_SIZE;
1250  }
1251 
1252  if ( locFilePath == NULL ) {
1253  status = sendfileByFd(
1254  &rbudpSender,
1255  mysendRate,
1256  mypacketSize,
1257  locFd );
1258  }
1259  else {
1260  status = rbSendfile(
1261  &rbudpSender,
1262  mysendRate,
1263  mypacketSize,
1264  locFilePath );
1265  }
1266 
1267  sendClose( &rbudpSender );
1268  if ( status < 0 ) {
1269  rodsLog( LOG_ERROR, "putFileToPortalRbudp: sendfile error for %s:%d", // JMC - backport 4590
1270  myPortList->hostAddr, myPortList->portNum & 0xffff0000 );
1271  return status;
1272  }
1273  return status;
1274 }
1275 
1276 /* getFileToPortalRbudp - The client side of getting a file using
1277  * Rbudp. If locFilePath is NULL, the local file has already been opned
1278  * and locFd should be used. If sendRate and packetSize are 0, it will
1279  * try to set it based on env and default.
1280  */
1283  char* locFilePath,
1284  int locFd,
1285  int veryVerbose,
1286  int packetSize ) {
1287 
1288 
1289  portList_t *myPortList;
1290  int status;
1291  rbudpReceiver_t rbudpReceiver;
1292  int mypacketSize;
1293  char *tmpStr;
1294 
1295  if ( portalOprOut == NULL || portalOprOut->numThreads != 1 ) {
1296  rodsLog( LOG_ERROR,
1297  "getFileToPortalRbudp: invalid portalOprOut" );
1298  return SYS_INVALID_PORTAL_OPR;
1299  }
1300 
1301  myPortList = &portalOprOut->portList;
1302 
1303  bzero( &rbudpReceiver, sizeof( rbudpReceiver ) );
1304  status = initRbudpClient( &rbudpReceiver.rbudpBase, myPortList );
1305  if ( status < 0 ) {
1306  rodsLog( LOG_ERROR,
1307  "getFileToPortalRbudp: initRbudpClient error for %s",
1308  myPortList->hostAddr );
1309  return status;
1310  }
1311 
1312  rbudpReceiver.rbudpBase.verbose = veryVerbose;
1313 
1314  if ( ( mypacketSize = packetSize ) < 1 &&
1315  ( ( tmpStr = getenv( RBUDP_PACK_SIZE_KW ) ) == NULL ||
1316  ( mypacketSize = atoi( tmpStr ) ) < 1 ) ) {
1317  mypacketSize = DEF_UDP_PACKET_SIZE;
1318  }
1319 
1320  rodsEnv rods_env;
1321  status = getRodsEnv( &rods_env );
1322  if ( status < 0 ) {
1323  return status;
1324  }
1325 
1326  if ( locFilePath == NULL ) {
1327  status = getfileByFd(
1328  &rbudpReceiver,
1329  locFd,
1330  mypacketSize );
1331 
1332  }
1333  else {
1334  status = getfile(
1335  &rbudpReceiver,
1336  NULL,
1337  locFilePath,
1338  mypacketSize );
1339  }
1340 
1341  recvClose( &rbudpReceiver );
1342  if ( status < 0 ) {
1343  rodsLog( LOG_ERROR,
1344  "getFileToPortalRbudp: getfile error for %s",
1345  myPortList->hostAddr );
1346  return status;
1347  }
1348  return status;
1349 }
1350 
1351 int
1352 initRbudpClient( rbudpBase_t *rbudpBase, portList_t *myPortList ) {
1353  int tcpSock;
1354  int tcpPort, udpPort;
1355  int status;
1356  struct sockaddr_in localUdpAddr;
1357  int udpLocalPort;
1358 
1359  if ( ( udpPort = getUdpPortFromPortList( myPortList ) ) == 0 ) {
1360  rodsLog( LOG_ERROR,
1361  "putFileToPortalRbudp: udpPort == 0" );
1362  return SYS_INVALID_PORTAL_OPR;
1363  }
1364 
1365  tcpPort = getTcpPortFromPortList( myPortList );
1366 
1367  tcpSock = connectToRhostPortal( myPortList->hostAddr,
1368  tcpPort, myPortList->cookie, myPortList->windowSize );
1369  if ( tcpSock < 0 ) {
1370  return tcpSock;
1371  }
1372 
1373  rbudpBase->udpSockBufSize = UDPSOCKBUF;
1374  rbudpBase->tcpPort = tcpPort;
1375  rbudpBase->tcpSockfd = tcpSock;
1376  rbudpBase->hasTcpSock = 0; /* so it will close properly */
1377  rbudpBase->udpRemotePort = udpPort;
1378 
1379  /* connect to the server's UDP port */
1380  status = passiveUDP( rbudpBase, myPortList->hostAddr );
1381 
1382  if ( status < 0 ) {
1383  rodsLog( LOG_ERROR,
1384  "initRbudpClient: passiveUDP connect to %s error. status = %d",
1385  myPortList->hostAddr, status );
1386  return SYS_UDP_CONNECT_ERR + status;
1387  }
1388 
1389  /* inform the server of the UDP port */
1390  rbudpBase->udpLocalPort =
1391  setLocalAddr( rbudpBase->udpSockfd, &localUdpAddr );
1392  if ( rbudpBase->udpLocalPort < 0 ) {
1393  return rbudpBase->udpLocalPort;
1394  }
1395  udpLocalPort = htonl( rbudpBase->udpLocalPort );
1396  status = writen( rbudpBase->tcpSockfd, ( char * ) &udpLocalPort,
1397  sizeof( udpLocalPort ) );
1398  if ( status != sizeof( udpLocalPort ) ) {
1399  rodsLog( LOG_ERROR,
1400  "initRbudpClient: writen error. towrite %d, bytes written %d ",
1401  sizeof( udpLocalPort ), status );
1402  return SYS_UDP_CONNECT_ERR;
1403  }
1404 
1405  return 0;
1406 }
1407 
1408 int
1409 initFileRestart( rcComm_t *conn, char *fileName, char *objPath,
1410  rodsLong_t fileSize, int numThr ) {
1411  fileRestart_t *fileRestart = &conn->fileRestart;
1412  fileRestartInfo_t *info = &fileRestart->info;
1413 
1414  if ( fileRestart->flags != FILE_RESTART_ON ||
1415  fileSize < MIN_RESTART_SIZE || numThr <= 0 ) {
1416  info->numSeg = 0; /* indicate no restart */
1417  return 0;
1418  }
1419  if ( numThr > MAX_NUM_CONFIG_TRAN_THR ) {
1421  "initFileRestart: input numThr %d larger than max %d ",
1422  numThr, MAX_NUM_CONFIG_TRAN_THR );
1423  info->numSeg = 0; /* indicate no restart */
1424  return 0;
1425  }
1426  info->numSeg = numThr;
1427  info->fileSize = fileSize;
1428  rstrcpy( info->fileName, fileName, MAX_NAME_LEN );
1429  rstrcpy( info->objPath, objPath, MAX_NAME_LEN );
1430  bzero( info->dataSeg, sizeof( dataSeg_t ) * MAX_NUM_CONFIG_TRAN_THR );
1431  return 0;
1432 }
1433 
1434 int
1435 writeLfRestartFile( char *infoFile, fileRestartInfo_t *info ) {
1436  bytesBuf_t *packedBBuf = NULL;
1437  int status, fd;
1438 
1439  status = packStruct( ( void * ) info, &packedBBuf,
1440  "FileRestartInfo_PI", RodsPackTable, 0, XML_PROT );
1441  if ( status < 0 || packedBBuf == NULL ) {
1442  rodsLog( LOG_ERROR,
1443  "writeLfRestartFile: packStruct error for %s, status = %d",
1444  info->fileName, status );
1445  freeBBuf( packedBBuf );
1446  return status;
1447  }
1448 
1449  /* write it to a file */
1450  fd = open( infoFile, O_CREAT | O_TRUNC | O_WRONLY, 0640 );
1451  if ( fd < 0 ) {
1452  status = UNIX_FILE_OPEN_ERR - errno;
1453  rodsLog( LOG_ERROR,
1454  "writeLfRestartFile: open failed for %s, status = %d",
1455  infoFile, status );
1456  freeBBuf( packedBBuf );
1457  return status;
1458  }
1459 
1460  status = write( fd, packedBBuf->buf, packedBBuf->len );
1461  close( fd );
1462 
1463  freeBBuf( packedBBuf );
1464  if ( status < 0 ) {
1465  status = UNIX_FILE_WRITE_ERR - errno;
1466  rodsLog( LOG_ERROR,
1467  "writeLfRestartFile: write failed for %s, status = %d",
1468  infoFile, status );
1469  return status;
1470  }
1471  return status;
1472 }
1473 
1474 int
1475 readLfRestartFile( char *infoFile, fileRestartInfo_t **info ) {
1476  int status, fd;
1477  rodsLong_t mySize;
1478  char *buf;
1479 
1480  *info = NULL;
1481  path p( infoFile );
1482  if ( !exists( p ) || !is_regular_file( p ) ) {
1483  status = UNIX_FILE_STAT_ERR - errno;
1484  return status;
1485  }
1486  else if ( ( mySize = file_size( p ) ) <= 0 ) {
1487  status = UNIX_FILE_STAT_ERR - errno;
1488  rodsLog( LOG_ERROR,
1489  "readLfRestartFile restart infoFile size is 0 for %s",
1490  infoFile );
1491  return status;
1492  }
1493 
1494  /* read the restart infoFile */
1495  fd = open( infoFile, O_RDONLY, 0640 );
1496  if ( fd < 0 ) {
1497  status = UNIX_FILE_OPEN_ERR - errno;
1498  rodsLog( LOG_ERROR,
1499  "readLfRestartFile open failed for %s, status = %d",
1500  infoFile, status );
1501  return status;
1502  }
1503 
1504  buf = ( char * ) calloc( 1, 2 * mySize );
1505  if ( buf == NULL ) {
1506  close( fd );
1507  return SYS_MALLOC_ERR;
1508  }
1509  status = read( fd, buf, mySize );
1510  if ( status != mySize ) {
1511  rodsLog( LOG_ERROR,
1512  "readLfRestartFile error failed for %s, toread %d, read %d",
1513  infoFile, mySize, status );
1514  status = UNIX_FILE_READ_ERR - errno;
1515  close( fd );
1516  free( buf );
1517  return status;
1518  }
1519 
1520  close( fd );
1521 
1522  status = unpackStruct( buf, ( void ** ) info, "FileRestartInfo_PI", NULL, XML_PROT );
1523 
1524  if ( status < 0 ) {
1525  rodsLog( LOG_ERROR,
1526  "readLfRestartFile: unpackStruct error for %s, status = %d",
1527  infoFile, status );
1528  }
1529  free( buf );
1530  return status;
1531 }
1532 
1533 
1534 int
1536  unlink( fileRestart->infoFile );
1537  bzero( &fileRestart->info, sizeof( fileRestartInfo_t ) );
1538 
1539  return 0;
1540 }
1541 
1542 int
1544  rodsLong_t curOffset = 0;
1545  bytesBuf_t dataObjWriteInpBBuf;
1546  int status = 0, i = 0;
1547  int localFd = 0, irodsFd = 0;
1548  dataObjInp_t dataObjOpenInp;
1549  openedDataObjInp_t dataObjWriteInp;
1550  openedDataObjInp_t dataObjLseekInp;
1551  openedDataObjInp_t dataObjCloseInp;
1552  fileLseekOut_t *dataObjLseekOut = NULL;
1553  int writtenSinceUpdated = 0;
1554  rodsLong_t gap;
1555 
1556 #ifdef windows_platform
1557  localFd = iRODSNt_bopen( info->fileName, O_RDONLY, 0 );
1558 #else
1559  localFd = open( info->fileName, O_RDONLY, 0 );
1560 #endif
1561  if ( localFd < 0 ) { /* error */
1562  status = UNIX_FILE_OPEN_ERR - errno;
1564  "cannot open file %s", info->fileName, status );
1565  return status;
1566  }
1567 
1568  bzero( &dataObjOpenInp, sizeof( dataObjOpenInp ) );
1569  rstrcpy( dataObjOpenInp.objPath, info->objPath, MAX_NAME_LEN );
1570  dataObjOpenInp.openFlags = O_WRONLY;
1571  addKeyVal( &dataObjOpenInp.condInput, FORCE_FLAG_KW, "" );
1572 
1573  irodsFd = rcDataObjOpen( conn, &dataObjOpenInp );
1574  if ( irodsFd < 0 ) { /* error */
1575  rodsLogError( LOG_ERROR, irodsFd,
1576  "cannot open target file %s, status = %d", info->objPath, irodsFd );
1577  close( localFd );
1578  return irodsFd;
1579  }
1580 
1581  rodsEnv rods_env;
1582  if ( int status = getRodsEnv( &rods_env ) ) {
1583  rodsLog( LOG_ERROR, "getRodsEnv failed in %s with status %s", __FUNCTION__, status );
1584  return status;
1585  }
1586  size_t trans_buff_sz = rods_env.irodsTransBufferSizeForParaTrans * 1024 * 1024;
1587 
1588  bzero( &dataObjWriteInp, sizeof( dataObjWriteInp ) );
1589  dataObjWriteInpBBuf.buf = malloc( trans_buff_sz + 1 );
1590  dataObjWriteInpBBuf.len = 0;
1591  dataObjWriteInp.l1descInx = irodsFd;
1592 
1593  memset( &dataObjLseekInp, 0, sizeof( dataObjLseekInp ) );
1594  dataObjLseekInp.whence = SEEK_SET;
1595  for ( i = 0; i < info->numSeg; i++ ) {
1596  gap = info->dataSeg[i].offset - curOffset;
1597  if ( gap < 0 ) {
1598  /* should not be here */
1599  }
1600  else if ( gap > 0 ) {
1601  rodsLong_t tmpLen, *lenToUpdate;
1602  if ( i == 0 ) {
1603  /* should not be here */
1604  tmpLen = 0;
1605  lenToUpdate = &tmpLen;
1606  }
1607  else {
1608  lenToUpdate = &info->dataSeg[i - 1].len;
1609  }
1610 
1611  status = putSeg( conn, gap, localFd, &dataObjWriteInp,
1612  &dataObjWriteInpBBuf, trans_buff_sz, &writtenSinceUpdated,
1613  info, lenToUpdate );
1614  if ( status < 0 ) {
1615  break;
1616  }
1617  curOffset += gap;
1618  }
1619  if ( info->dataSeg[i].len > 0 ) {
1620  curOffset += info->dataSeg[i].len;
1621  if ( lseek( localFd, curOffset, SEEK_SET ) < 0 ) {
1622  status = UNIX_FILE_LSEEK_ERR - errno;
1624  "lfRestartWithInfo: lseek to %lld error for %s",
1625  curOffset, info->fileName );
1626  break;
1627  }
1628  dataObjLseekInp.l1descInx = irodsFd;
1629  dataObjLseekInp.offset = curOffset;
1630  status = rcDataObjLseek( conn, &dataObjLseekInp, &dataObjLseekOut );
1631  if ( status < 0 ) {
1633  "lfRestartWithInfo: rcDataObjLseek to %lld error for %s",
1634  curOffset, info->objPath );
1635  break;
1636  }
1637  else {
1638  if ( dataObjLseekOut != NULL ) {
1639  free( dataObjLseekOut );
1640  }
1641  }
1642  }
1643  }
1644  if ( status >= 0 ) {
1645  gap = info->fileSize - curOffset;
1646  if ( gap > 0 ) {
1647  status = putSeg( conn, gap, localFd, &dataObjWriteInp,
1648  &dataObjWriteInpBBuf, trans_buff_sz, &writtenSinceUpdated,
1649  info, &info->dataSeg[i - 1].len );
1650  }
1651  }
1652  free( dataObjWriteInpBBuf.buf );
1653  close( localFd );
1654  memset( &dataObjCloseInp, 0, sizeof( dataObjCloseInp ) );
1655  dataObjCloseInp.l1descInx = irodsFd;
1656  rcDataObjClose( conn, &dataObjCloseInp );
1657  return status;
1658 }
1659 
1660 int
1661 putSeg( rcComm_t *conn, rodsLong_t segSize, int localFd,
1662  openedDataObjInp_t *dataObjWriteInp, bytesBuf_t *dataObjWriteInpBBuf,
1663  int bufLen, int *writtenSinceUpdated, fileRestartInfo_t *info,
1664  rodsLong_t *dataSegLen ) {
1665  rodsLong_t gap = segSize;
1666  int bytesWritten;
1667  int status;
1668 
1669  while ( gap > 0 ) {
1670  int toRead;
1671  if ( gap > bufLen ) {
1672  toRead = bufLen;
1673  }
1674  else {
1675  toRead = ( int ) gap;
1676  }
1677 
1678  dataObjWriteInpBBuf->len = myRead( localFd,
1679  dataObjWriteInpBBuf->buf, toRead, NULL, NULL );
1680  /* Write to the data object */
1681  dataObjWriteInp->len = dataObjWriteInpBBuf->len;
1682  bytesWritten = rcDataObjWrite( conn, dataObjWriteInp,
1683  dataObjWriteInpBBuf );
1684  if ( bytesWritten < dataObjWriteInp->len ) {
1685  rodsLog( LOG_ERROR,
1686  "putFile: Read %d bytes, Wrote %d bytes.\n ",
1687  dataObjWriteInp->len, bytesWritten );
1688  return SYS_COPY_LEN_ERR;
1689  }
1690  else {
1691  gap -= toRead;
1692  *writtenSinceUpdated += toRead;
1693  *dataSegLen += toRead;
1694  if ( *writtenSinceUpdated >= RESTART_FILE_UPDATE_SIZE ) {
1695  status = writeLfRestartFile( conn->fileRestart.infoFile, info );
1696  if ( status < 0 ) {
1697  rodsLog( LOG_ERROR,
1698  "putSeg: writeLfRestartFile for %s, status = %d",
1699  info->fileName, status );
1700  return status;
1701  }
1702  *writtenSinceUpdated = 0;
1703  }
1704  }
1705  }
1706  return 0;
1707 }
1708 
1709 int
1711  rodsLong_t curOffset = 0;
1712  bytesBuf_t dataObjReadInpBBuf;
1713  int status = 0, i = 0;
1714  int localFd = 0, irodsFd = 0;
1715  dataObjInp_t dataObjOpenInp;
1716  openedDataObjInp_t dataObjReadInp;
1717  openedDataObjInp_t dataObjLseekInp;
1718  openedDataObjInp_t dataObjCloseInp;
1719  fileLseekOut_t *dataObjLseekOut = NULL;
1720  int writtenSinceUpdated = 0;
1721  rodsLong_t gap;
1722 
1723 #ifdef windows_platform
1724  localFd = iRODSNt_bopen( info->fileName, O_RDONLY, 0 );
1725 #else
1726  localFd = open( info->fileName, O_RDWR, 0 );
1727 #endif
1728  if ( localFd < 0 ) { /* error */
1729  status = UNIX_FILE_OPEN_ERR - errno;
1731  "cannot open local file %s, status = %d", info->fileName, status );
1732  return status;
1733  }
1734 
1735  bzero( &dataObjOpenInp, sizeof( dataObjOpenInp ) );
1736  rstrcpy( dataObjOpenInp.objPath, info->objPath, MAX_NAME_LEN );
1737  dataObjOpenInp.openFlags = O_RDONLY;
1738  irodsFd = rcDataObjOpen( conn, &dataObjOpenInp );
1739  if ( irodsFd < 0 ) { /* error */
1740  rodsLogError( LOG_ERROR, irodsFd,
1741  "cannot open iRODS src file %s, status = %d", info->objPath, irodsFd );
1742  close( localFd );
1743  return irodsFd;
1744  }
1745 
1746  rodsEnv rods_env;
1747  if ( int status = getRodsEnv( &rods_env ) ) {
1748  rodsLog( LOG_ERROR, "getRodsEnv failed in %s with status %s", __FUNCTION__, status );
1749  return status;
1750  }
1751  size_t trans_buff_sz = rods_env.irodsTransBufferSizeForParaTrans * 1024 * 1024;
1752 
1753  bzero( &dataObjReadInp, sizeof( dataObjReadInp ) );
1754  dataObjReadInpBBuf.buf = malloc( trans_buff_sz );
1755  dataObjReadInpBBuf.len = 0;
1756  dataObjReadInp.l1descInx = irodsFd;
1757 
1758  memset( &dataObjLseekInp, 0, sizeof( dataObjLseekInp ) );
1759  dataObjLseekInp.whence = SEEK_SET;
1760  dataObjLseekInp.l1descInx = irodsFd;
1761 
1762  for ( i = 0; i < info->numSeg; i++ ) {
1763  gap = info->dataSeg[i].offset - curOffset;
1764  if ( gap < 0 ) {
1765  /* should not be here */
1766  }
1767  else if ( gap > 0 ) {
1768  rodsLong_t tmpLen, *lenToUpdate;
1769  if ( i == 0 ) {
1770  /* should not be here */
1771  tmpLen = 0;
1772  lenToUpdate = &tmpLen;
1773  }
1774  else {
1775  lenToUpdate = &info->dataSeg[i - 1].len;
1776  }
1777  status = getSeg( conn, gap, localFd, &dataObjReadInp,
1778  &dataObjReadInpBBuf, trans_buff_sz, &writtenSinceUpdated,
1779  info, lenToUpdate );
1780  if ( status < 0 ) {
1781  break;
1782  }
1783  curOffset += gap;
1784  }
1785  if ( info->dataSeg[i].len > 0 ) {
1786  curOffset += info->dataSeg[i].len;
1787  if ( lseek( localFd, curOffset, SEEK_SET ) < 0 ) {
1788  status = UNIX_FILE_LSEEK_ERR - errno;
1790  "lfRestartWithInfo: lseek to %lld error for %s",
1791  curOffset, info->fileName );
1792  break;
1793  }
1794  dataObjLseekInp.offset = curOffset;
1795  status = rcDataObjLseek( conn, &dataObjLseekInp, &dataObjLseekOut );
1796  if ( status < 0 ) {
1798  "lfRestartWithInfo: rcDataObjLseek to %lld error for %s",
1799  curOffset, info->objPath );
1800  break;
1801  }
1802  else {
1803  if ( dataObjLseekOut != NULL ) {
1804  free( dataObjLseekOut );
1805  }
1806  }
1807  }
1808  }
1809  if ( status >= 0 ) {
1810  gap = info->fileSize - curOffset;
1811  if ( gap > 0 ) {
1812  status = getSeg( conn, gap, localFd, &dataObjReadInp,
1813  &dataObjReadInpBBuf, trans_buff_sz, &writtenSinceUpdated,
1814  info, &info->dataSeg[i - 1].len );
1815  }
1816  }
1817  free( dataObjReadInpBBuf.buf );
1818  close( localFd );
1819  memset( &dataObjCloseInp, 0, sizeof( dataObjCloseInp ) );
1820  dataObjCloseInp.l1descInx = irodsFd;
1821  rcDataObjClose( conn, &dataObjCloseInp );
1822  return status;
1823 }
1824 
1825 int
1826 getSeg( rcComm_t *conn, rodsLong_t segSize, int localFd,
1827  openedDataObjInp_t *dataObjReadInp, bytesBuf_t *dataObjReadInpBBuf,
1828  int bufLen, int *writtenSinceUpdated, fileRestartInfo_t *info,
1829  rodsLong_t *dataSegLen ) {
1830  rodsLong_t gap = segSize;
1831  int bytesWritten, bytesRead;
1832  int status;
1833 
1834  while ( gap > 0 ) {
1835  int toRead;
1836  if ( gap > bufLen ) {
1837  toRead = bufLen;
1838  }
1839  else {
1840  toRead = ( int ) gap;
1841  }
1842  dataObjReadInp->len = dataObjReadInpBBuf->len = toRead;
1843  bytesRead = rcDataObjRead( conn, dataObjReadInp,
1844  dataObjReadInpBBuf );
1845 
1846  if ( bytesRead < 0 ) {
1847  rodsLog( LOG_ERROR,
1848  "getSeg: rcDataObjRead error. status = %d", bytesRead );
1849  return bytesRead;
1850  }
1851  else if ( bytesRead == 0 ) {
1852  /* EOF */
1853  rodsLog( LOG_ERROR,
1854  "getSeg: rcDataObjRead error. EOF reached. toRead = %d", toRead );
1855  return SYS_COPY_LEN_ERR;
1856  }
1857  bytesWritten = myWrite( localFd, dataObjReadInpBBuf->buf,
1858  bytesRead, NULL );
1859 
1860  if ( bytesWritten != bytesRead ) {
1861  rodsLog( LOG_ERROR,
1862  "getSeg: Read %d bytes, Wrote %d bytes.\n ",
1863  bytesRead, bytesWritten );
1864  return SYS_COPY_LEN_ERR;
1865  }
1866  else {
1867  gap -= bytesWritten;
1868  *writtenSinceUpdated += bytesWritten;
1869  *dataSegLen += bytesWritten;
1870  if ( *writtenSinceUpdated >= RESTART_FILE_UPDATE_SIZE ) {
1871  status = writeLfRestartFile( conn->fileRestart.infoFile, info );
1872  if ( status < 0 ) {
1873  rodsLog( LOG_ERROR,
1874  "getSeg: writeLfRestartFile for %s, status = %d",
1875  info->fileName, status );
1876  return status;
1877  }
1878  *writtenSinceUpdated = 0;
1879  }
1880  }
1881  }
1882  return 0;
1883 }
1884 
1885 int
1886 catDataObj( rcComm_t *conn, char *objPath ) {
1887  dataObjInp_t dataObjOpenInp;
1888  openedDataObjInp_t dataObjCloseInp;
1889  openedDataObjInp_t dataObjReadInp;
1890  bytesBuf_t dataObjReadOutBBuf;
1891  int l1descInx;
1892  int bytesRead;
1893 
1894 
1895  memset( &dataObjOpenInp, 0, sizeof( dataObjOpenInp ) );
1896  rstrcpy( dataObjOpenInp.objPath, objPath, MAX_NAME_LEN );
1897  dataObjOpenInp.openFlags = O_RDONLY;
1898 
1899  l1descInx = rcDataObjOpen( conn, &dataObjOpenInp );
1900  if ( l1descInx < 0 ) {
1901  rodsLogError( LOG_ERROR, l1descInx,
1902  "catDataObj: rcDataObjOpen error for %s", objPath );
1903  return l1descInx;
1904  }
1905 
1906  rodsEnv rods_env;
1907  if ( int status = getRodsEnv( &rods_env ) ) {
1908  rodsLog( LOG_ERROR, "getRodsEnv failed in %s with status %s", __FUNCTION__, status );
1909  return status;
1910  }
1911  size_t trans_buff_sz = rods_env.irodsTransBufferSizeForParaTrans * 1024 * 1024;
1912  bzero( &dataObjReadInp, sizeof( dataObjReadInp ) );
1913  dataObjReadOutBBuf.buf = malloc( trans_buff_sz + 1 );
1914  dataObjReadOutBBuf.len = trans_buff_sz + 1;
1915  dataObjReadInp.l1descInx = l1descInx;
1916  dataObjReadInp.len = trans_buff_sz;
1917 
1918  while ( ( bytesRead = rcDataObjRead( conn, &dataObjReadInp,
1919  &dataObjReadOutBBuf ) ) > 0 ) {
1920  char *buf = ( char * ) dataObjReadOutBBuf.buf;
1921  buf[bytesRead] = '\0';
1922  printf( "%s", buf );
1923  }
1924  free( dataObjReadOutBBuf.buf );
1925  printf( "\n" );
1926  memset( &dataObjCloseInp, 0, sizeof( dataObjCloseInp ) );
1927  dataObjCloseInp.l1descInx = l1descInx;
1928 
1929  rcDataObjClose( conn, &dataObjCloseInp );
1930 
1931  return 0;
1932 }
RBUDP_SEND_RATE_KW
#define RBUDP_SEND_RATE_KW
Definition: rodsKeyWdDef.h:78
rodsLog
void rodsLog(int level, const char *formatStr,...)
Definition: rodsLog.cpp:86
DataObjInp::objPath
char objPath[(1024+64)]
Definition: dataObjInpOut.h:66
OpenedDataObjInp
Definition: dataObjInpOut.h:104
dataObjRead.h
MAX_PROGRESS_CNT
#define MAX_PROGRESS_CNT
Definition: rcPortalOpr.h:19
UNIX_FILE_WRITE_ERR
@ UNIX_FILE_WRITE_ERR
Definition: rodsErrorTable.h:303
UNIX_FILE_LSEEK_ERR
@ UNIX_FILE_LSEEK_ERR
Definition: rodsErrorTable.h:308
rcComm_t
Definition: rcConnect.h:95
NULL
#define NULL
Definition: rodsDef.h:70
irods::buffer_crypt::decrypt
irods::error decrypt(const array_t &, const array_t &, const array_t &, array_t &)
Definition: irods_buffer_encryption.cpp:260
fileRestart_t::info
fileRestartInfo_t info
Definition: rcConnect.h:85
transferStat_t
Definition: objInfo.h:291
addKeyVal
int addKeyVal(keyValPair_t *condInput, const char *keyWord, const char *value)
Definition: rcMisc.cpp:789
rodsEnv::rodsEncryptionKeySize
int rodsEncryptionKeySize
Definition: getRodsEnv.h:27
myRead
int myRead(int sock, void *buf, int len, int *bytesRead, struct timeval *tv)
Definition: rcMisc.cpp:4651
SYS_INTERNAL_NULL_INPUT_ERR
@ SYS_INTERNAL_NULL_INPUT_ERR
Definition: rodsErrorTable.h:92
sendTranHeader
int sendTranHeader(int sock, int oprType, int flags, rodsLong_t offset, rodsLong_t length)
Definition: rcPortalOpr.cpp:36
DataObjInp::openFlags
int openFlags
Definition: dataObjInpOut.h:68
getRodsEnv
int getRodsEnv(rodsEnv *myRodsEnv)
Definition: getRodsEnv.cpp:112
irods::buffer_crypt::array_t
std::vector< unsigned char > array_t
Definition: irods_buffer_encryption.hpp:30
OperProgress::flag
int flag
Definition: guiProgressCallback.h:9
portalOprOut
Definition: dataObjInpOut.h:114
fileRestart_t::infoFile
char infoFile[(1024+64)]
Definition: rcConnect.h:84
rodsEnv::irodsTransBufferSizeForParaTrans
int irodsTransBufferSizeForParaTrans
Definition: getRodsEnv.h:57
BytesBuf::buf
void * buf
Definition: rodsDef.h:199
rcDataObjRead
int rcDataObjRead(rcComm_t *conn, openedDataObjInp_t *dataObjReadInp, bytesBuf_t *dataObjReadOutBBuf)
Definition: rcDataObjRead.cpp:69
initFileRestart
int initFileRestart(rcComm_t *conn, char *fileName, char *objPath, rodsLong_t fileSize, int numThr)
Definition: rcPortalOpr.cpp:1409
SYS_MALLOC_ERR
@ SYS_MALLOC_ERR
Definition: rodsErrorTable.h:84
getFile
int getFile(rcComm_t *conn, int l1descInx, char *locFilePath, char *objPath, rodsLong_t dataSize)
Definition: rcPortalOpr.cpp:688
getFileToPortalRbudp
int getFileToPortalRbudp(portalOprOut_t *portalOprOut, char *locFilePath, int locFd, int veryVerbose, int packetSize)
Definition: rcPortalOpr.cpp:1281
irods_stacktrace.hpp
PASS
#define PASS(prev_error_)
Definition: irods_error.hpp:118
OpenedDataObjInp::l1descInx
int l1descInx
Definition: dataObjInpOut.h:105
OperProgress::curFileSize
rodsLong_t curFileSize
Definition: guiProgressCallback.h:15
fileRestartInfo_t::numSeg
int numSeg
Definition: rcConnect.h:75
RcPortalTransferInp::bytesWritten
rodsLong_t bytesWritten
Definition: rcPortalOpr.h:27
XML_PROT
@ XML_PROT
Definition: rodsDef.h:149
OperProgress::curFileSizeDone
rodsLong_t curFileSizeDone
Definition: guiProgressCallback.h:16
rcPartialDataPut
void rcPartialDataPut(rcPortalTransferInp_t *myInput)
Definition: rcPortalOpr.cpp:291
irods::experimental::administration::client::v1::exists
auto exists(rcComm_t &conn, const user &user) -> bool
Definition: user_administration.cpp:359
rcMisc.h
pid_age.p
p
Definition: pid_age.py:13
fileRestartInfo_t::objPath
char objPath[(1024+64)]
Definition: rcConnect.h:74
getUdpPortFromPortList
int getUdpPortFromPortList(portList_t *thisPortList)
Definition: sockComm.cpp:1475
setLocalAddr
int setLocalAddr(int sock, struct sockaddr_in *localAddr)
Definition: sockComm.cpp:1057
dataObjLseek.h
unpackStruct
int unpackStruct(const void *inPackStr, void **outStruct, const char *packInstName, const packInstruct_t *myPackTable, irodsProt_t irodsProt)
Definition: packStruct.cpp:63
RcPortalTransferInp::shared_secret
unsigned char shared_secret[64]
Definition: rcPortalOpr.h:28
OpenedDataObjInp::whence
int whence
Definition: dataObjInpOut.h:107
gGuiProgressCB
guiProgressCallback gGuiProgressCB
Definition: rcGlobal.cpp:207
fileRestart_t
Definition: rcConnect.h:81
MAX_NUM_CONFIG_TRAN_THR
#define MAX_NUM_CONFIG_TRAN_THR
Definition: rodsDef.h:96
rcGlobalExtern.h
RESTART_FILE_UPDATE_SIZE
#define RESTART_FILE_UPDATE_SIZE
Definition: rodsDef.h:94
putSeg
int putSeg(rcComm_t *conn, rodsLong_t segSize, int localFd, openedDataObjInp_t *dataObjWriteInp, bytesBuf_t *dataObjWriteInpBBuf, int bufLen, int *writtenSinceUpdated, fileRestartInfo_t *info, rodsLong_t *dataSegLen)
Definition: rcPortalOpr.cpp:1661
RcPortalTransferInp::conn
rcComm_t * conn
Definition: rcPortalOpr.h:22
LOG_ERROR
#define LOG_ERROR
Definition: rodsLog.h:43
TransferHeader::flags
int flags
Definition: dataObjInpOut.h:210
connectToRhostPortal
int connectToRhostPortal(char *rodsHost, int rodsPort, int cookie, int windowSize)
Definition: sockComm.cpp:655
fileLseek.h
dataSeg::len
rodsLong_t len
Definition: rcConnect.h:58
putFile
int putFile(rcComm_t *conn, int l1descInx, char *locFilePath, char *objPath, rodsLong_t dataSize)
Definition: rcPortalOpr.cpp:528
transferStat_t::bytesWritten
rodsLong_t bytesWritten
Definition: objInfo.h:294
FILE_RESTART_ON
@ FILE_RESTART_ON
Definition: rcConnect.h:64
initRbudpClient
int initRbudpClient(rbudpBase_t *rbudpBase, portList_t *myPortList)
Definition: rcPortalOpr.cpp:1352
BytesBuf::len
int len
Definition: rodsDef.h:198
portList_t::portNum
int portNum
Definition: dataObjInpOut.h:30
portList_t::windowSize
int windowSize
Definition: dataObjInpOut.h:33
mySockClose
int mySockClose(int sock)
Definition: sockComm.cpp:1573
getSeg
int getSeg(rcComm_t *conn, rodsLong_t segSize, int localFd, openedDataObjInp_t *dataObjReadInp, bytesBuf_t *dataObjReadInpBBuf, int bufLen, int *writtenSinceUpdated, fileRestartInfo_t *info, rodsLong_t *dataSegLen)
Definition: rcPortalOpr.cpp:1826
RcPortalTransferInp::status
int status
Definition: rcPortalOpr.h:26
UNIX_FILE_OPEN_ERR
@ UNIX_FILE_OPEN_ERR
Definition: rodsErrorTable.h:300
irods_buffer_encryption.hpp
TransferHeader
Definition: dataObjInpOut.h:208
SYS_COPY_LEN_ERR
@ SYS_COPY_LEN_ERR
Definition: rodsErrorTable.h:95
myWrite
int myWrite(int sock, void *buf, int len, int *bytesWritten)
Definition: rcMisc.cpp:4610
OpenedDataObjInp::len
int len
Definition: dataObjInpOut.h:106
irods::experimental::filesystem::client::copy
auto copy(rcComm_t &_comm, const path &_from, const path &_to, copy_options _options=copy_options::none) -> void
Definition: filesystem.cpp:370
rcvTranHeader
int rcvTranHeader(int sock, transferHeader_t *myHeader)
Definition: rcPortalOpr.cpp:65
fillBBufWithFile
int fillBBufWithFile(rcComm_t *conn, bytesBuf_t *myBBuf, char *locFilePath, rodsLong_t dataSize)
Definition: rcPortalOpr.cpp:93
fileRestartInfo_t::fileSize
rodsLong_t fileSize
Definition: rcConnect.h:77
readLfRestartFile
int readLfRestartFile(char *infoFile, fileRestartInfo_t **info)
Definition: rcPortalOpr.cpp:1475
irods::CS_NEG_USE_SSL
const std::string CS_NEG_USE_SSL("CS_NEG_USE_SSL")
DataObjInp
Definition: dataObjInpOut.h:65
myNtohll
int myNtohll(rodsLong_t inlonglong, rodsLong_t *outlonglong)
Definition: rcMisc.cpp:256
MAX_NAME_LEN
#define MAX_NAME_LEN
Definition: rodsDef.h:61
USER_FILE_TOO_LARGE
@ USER_FILE_TOO_LARGE
Definition: rodsErrorTable.h:242
RBUDP_PACK_SIZE_KW
#define RBUDP_PACK_SIZE_KW
Definition: rodsKeyWdDef.h:79
irods::buffer_crypt
Definition: irods_buffer_encryption.hpp:25
dataSeg
Definition: rcConnect.h:57
fileRestartInfo_t::dataSeg
dataSeg_t dataSeg[64]
Definition: rcConnect.h:78
putFileToPortalRbudp
int putFileToPortalRbudp(portalOprOut_t *portalOprOut, char *locFilePath, int locFd, int veryVerbose, int sendRate, int packetSize)
Definition: rcPortalOpr.cpp:1210
DONE_OPR
#define DONE_OPR
Definition: dataObjInpOut.h:166
portalOprOut::portList
portList_t portList
Definition: dataObjInpOut.h:119
portList_t
Definition: dataObjInpOut.h:29
irods::buffer_crypt::initialization_vector
irods::error initialization_vector(array_t &)
Definition: irods_buffer_encryption.cpp:134
irods.pypyodbc.status
status
Definition: pypyodbc.py:467
dataObjWrite.h
rcComm_t::operProgress
operProgress_t operProgress
Definition: rcConnect.h:119
rodsLog.h
irods::buffer_crypt::encrypt
irods::error encrypt(const array_t &, const array_t &, const array_t &, array_t &)
Definition: irods_buffer_encryption.cpp:168
irods::log
void log(const error &)
Definition: irods_log.cpp:13
lfRestartGetWithInfo
int lfRestartGetWithInfo(rcComm_t *conn, fileRestartInfo_t *info)
Definition: rcPortalOpr.cpp:1710
LOG_NOTICE
#define LOG_NOTICE
Definition: rodsLog.h:33
rcPortalOpr.h
dataObjClose.h
fileRestartInfo_t
Definition: rcConnect.h:72
rcComm_t::fileRestart
fileRestart_t fileRestart
Definition: rcConnect.h:135
RodsPackTable
const packInstruct_t RodsPackTable[]
Definition: rodsPackTable.h:61
BytesBuf
Definition: rodsDef.h:197
irods::error
Definition: irods_error.hpp:23
rcPartialDataGet
void rcPartialDataGet(rcPortalTransferInp_t *myInput)
Definition: rcPortalOpr.cpp:976
FileLseekOut
Definition: fileLseek.h:53
int
typedef int((*funcPtr)())
fileRestartInfo_t::fileName
char fileName[(1024+64)]
Definition: rcConnect.h:73
TransferHeader::offset
rodsLong_t offset
Definition: dataObjInpOut.h:211
CLOSE_SOCK
#define CLOSE_SOCK
Definition: sockComm.h:36
myHtonll
int myHtonll(rodsLong_t inlonglong, rodsLong_t *outlonglong)
Definition: rcMisc.cpp:232
fileRestart_t::flags
fileRestartFlag_t flags
Definition: rcConnect.h:82
putFileToPortal
int putFileToPortal(rcComm_t *conn, portalOprOut_t *portalOprOut, char *locFilePath, char *objPath, rodsLong_t dataSize)
Definition: rcPortalOpr.cpp:136
lfRestartPutWithInfo
int lfRestartPutWithInfo(rcComm_t *conn, fileRestartInfo_t *info)
Definition: rcPortalOpr.cpp:1543
UNIX_FILE_READ_ERR
@ UNIX_FILE_READ_ERR
Definition: rodsErrorTable.h:302
FORCE_FLAG_KW
#define FORCE_FLAG_KW
Definition: rodsKeyWdDef.h:13
TransferHeader::length
rodsLong_t length
Definition: dataObjInpOut.h:212
writeLfRestartFile
int writeLfRestartFile(char *infoFile, fileRestartInfo_t *info)
Definition: rcPortalOpr.cpp:1435
rcDataObjOpen
int rcDataObjOpen(rcComm_t *conn, dataObjInp_t *dataObjInp)
Definition: rcDataObjOpen.cpp:67
irods::buffer_crypt::key_size
int key_size()
Definition: irods_buffer_encryption.hpp:77
dataSeg::offset
rodsLong_t offset
Definition: rcConnect.h:59
SYS_INVALID_PORTAL_OPR
@ SYS_INVALID_PORTAL_OPR
Definition: rodsErrorTable.h:101
portalOprOut::numThreads
int numThreads
Definition: dataObjInpOut.h:117
rodsLogError
void rodsLogError(int level, int errCode, char *formatStr,...)
Definition: rodsLog.cpp:422
packStruct
int packStruct(const void *inStruct, bytesBuf_t **packedResult, const char *packInstName, const packInstruct_t *myPackTable, int packFlag, irodsProt_t irodsProt)
Definition: packStruct.cpp:21
RcPortalTransferInp
Definition: rcPortalOpr.h:21
RcPortalTransferInp::srcFd
int srcFd
Definition: rcPortalOpr.h:24
rodsEnv::rodsEncryptionNumHashRounds
int rodsEncryptionNumHashRounds
Definition: getRodsEnv.h:29
rcComm_t::transStat
transferStat_t transStat
Definition: rcConnect.h:108
MIN_RESTART_SIZE
#define MIN_RESTART_SIZE
Definition: rodsDef.h:93
catDataObj
int catDataObj(rcComm_t *conn, char *objPath)
Definition: rcPortalOpr.cpp:1886
sockComm.h
OpenedDataObjInp::offset
rodsLong_t offset
Definition: dataObjInpOut.h:109
rodsEnv::irodsMaxSizeForSingleBuffer
int irodsMaxSizeForSingleBuffer
Definition: getRodsEnv.h:55
dataObjOpen.h
clearLfRestartFile
int clearLfRestartFile(fileRestart_t *fileRestart)
Definition: rcPortalOpr.cpp:1535
freeBBuf
int freeBBuf(bytesBuf_t *myBBuf)
Definition: rcMisc.cpp:88
UNIX_FILE_STAT_ERR
@ UNIX_FILE_STAT_ERR
Definition: rodsErrorTable.h:306
rcComm_t::shared_secret
unsigned char shared_secret[64]
Definition: rcConnect.h:126
rstrcpy
char * rstrcpy(char *dest, const char *src, int maxLen)
Definition: stringOpr.cpp:51
dataObjOpr.hpp
getTcpPortFromPortList
int getTcpPortFromPortList(portList_t *thisPortList)
Definition: sockComm.cpp:1482
getIncludeFile
int getIncludeFile(rcComm_t *conn, bytesBuf_t *dataObjOutBBuf, char *locFilePath)
Definition: rcPortalOpr.cpp:639
DataObjInp::condInput
keyValPair_t condInput
Definition: dataObjInpOut.h:74
portList_t::cookie
int cookie
Definition: dataObjInpOut.h:31
rodsEnv
Definition: getRodsEnv.h:8
NAME_LEN
#define NAME_LEN
Definition: rodsDef.h:55
rodsEnv::rodsEncryptionSaltSize
int rodsEncryptionSaltSize
Definition: getRodsEnv.h:28
rcDataObjLseek
int rcDataObjLseek(rcComm_t *conn, openedDataObjInp_t *dataObjLseekInp, fileLseekOut_t **dataObjLseekOut)
Definition: rcDataObjLseek.cpp:77
irods::error::ok
bool ok()
Definition: irods_error.cpp:258
buf
static char buf[64+50+1]
Definition: rsAuthRequest.cpp:21
rodsEnv::rodsEncryptionAlgorithm
char rodsEncryptionAlgorithm[128]
Definition: getRodsEnv.h:30
SYS_THREAD_RESOURCE_ERR
@ SYS_THREAD_RESOURCE_ERR
Definition: rodsErrorTable.h:214
fillRcPortalTransferInp
int fillRcPortalTransferInp(rcComm_t *conn, rcPortalTransferInp_t *myInput, int destFd, int srcFd, int threadNum)
Definition: rcPortalOpr.cpp:274
rcDataObjWrite
int rcDataObjWrite(rcComm_t *conn, openedDataObjInp_t *dataObjWriteInp, bytesBuf_t *dataObjWriteInpBBuf)
Definition: rcDataObjWrite.cpp:79
TransferHeader::oprType
int oprType
Definition: dataObjInpOut.h:209
SYS_UDP_CONNECT_ERR
@ SYS_UDP_CONNECT_ERR
Definition: rodsErrorTable.h:153
RcPortalTransferInp::destFd
int destFd
Definition: rcPortalOpr.h:23
irods::error::result
std::string result() const
Definition: irods_error.cpp:201
RcPortalTransferInp::threadNum
int threadNum
Definition: rcPortalOpr.h:25
portList_t::hostAddr
char hostAddr[256]
Definition: dataObjInpOut.h:34
STDOUT_FILE_NAME
#define STDOUT_FILE_NAME
Definition: rodsPath.h:16
getFileFromPortal
int getFileFromPortal(rcComm_t *conn, portalOprOut_t *portalOprOut, char *locFilePath, char *objPath, rodsLong_t dataSize)
Definition: rcPortalOpr.cpp:833
irods_client_server_negotiation.hpp
rodsLong_t
long long rodsLong_t
Definition: rodsType.h:32
rcComm_t::negotiation_results
char negotiation_results[(1024+64)]
Definition: rcConnect.h:125
rcDataObjClose
int rcDataObjClose(rcComm_t *conn, openedDataObjInp_t *dataObjCloseInp)
Definition: rcDataObjClose.cpp:63
fileRestart_t::writtenSinceUpdated
rodsLong_t writtenSinceUpdated
Definition: rcConnect.h:83