BackupPC-4.4.0.tar.gz: .../BackupPC_backupDuplicate

"Fossies" - the Fresh Open Source Software Archive

Member "BackupPC-4.4.0/bin/BackupPC_backupDuplicate" (20 Jun 2020, 18596 Bytes) of package /linux/privat/BackupPC-4.4.0.tar.gz:

As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Perl source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. See also the latest Fossies "Diffs" side-by-side code changes report for "BackupPC_backupDuplicate":

4.3.2_vs_4.4.0.

1 #!/usr/bin/perl 2 #============================================================= -*-perl-*- 3 # 4 # BackupPC_backupDuplicate: Make a copy of the most recent backup 5 # 6 # DESCRIPTION 7 # 8 # BackupPC_backupDuplicate: make a copy of the last V4 backup 9 # 10 # Usage: 11 # BackupPC_backupDuplicate [-m] [-p] -h host 12 # 13 # AUTHOR 14 # Craig Barratt <cbarratt@users.sourceforge.net> 15 # 16 # COPYRIGHT 17 # Copyright (C) 2001-2020 Craig Barratt 18 # 19 # This program is free software: you can redistribute it and/or modify 20 # it under the terms of the GNU General Public License as published by 21 # the Free Software Foundation, either version 3 of the License, or 22 # (at your option) any later version. 23 # 24 # This program is distributed in the hope that it will be useful, 25 # but WITHOUT ANY WARRANTY; without even the implied warranty of 26 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 27 # GNU General Public License for more details. 28 # 29 # You should have received a copy of the GNU General Public License 30 # along with this program. If not, see <http://www.gnu.org/licenses/>. 31 # 32 #======================================================================== 33 # 34 # Version 4.4.0, released 20 Jun 2020. 35 # 36 # See http://backuppc.sourceforge.net. 37 # 38 #======================================================================== 39 40 use strict; 41 no utf8; 42 43 use lib "__INSTALLDIR__/lib"; 44 use Getopt::Std; 45 use File::Copy; 46 use File::Path; 47 use Data::Dumper; 48 use Digest::MD5; 49 50 use BackupPC::Lib; 51 use BackupPC::XS qw( :all ); 52 use BackupPC::DirOps; 53 use BackupPC::View; 54 55 my $ErrorCnt = 0; 56 my $FileErrorCnt = 0; 57 my $FileCnt = 0; 58 my $FileCntNext = 100; 59 60 die("BackupPC::Lib->new failed\n") if ( !(my $bpc = BackupPC::Lib->new) ); 61 62 my $TopDir = $bpc->TopDir(); 63 my $BinDir = $bpc->BinDir(); 64 my %Conf = $bpc->Conf(); 65 my $Hosts = $bpc->HostInfoRead(); 66 67 my %opts; 68 69 if ( !getopts("mph:", \%opts) || @ARGV >= 1 || !defined($opts{h}) ) { 70 print STDERR <<EOF; 71 usage: BackupPC_backupDuplicate [-m] [-p] -h host 72 Options: 73 -h host host name 74 -m force running even if a backup on this host is running 75 (specifically, don't take the server host mutex) 76 -p don't print progress information 77 EOF 78 exit(1); 79 } 80 81 if ( $opts{h} !~ /^([\w\.\s-]+)$/ 82 || $opts{h} =~ m{(^|/)\.\.(/|$)} 83 || !defined($Hosts->{$opts{h}}) ) { 84 print(STDERR "BackupPC_backupDuplicate: bad host name '$opts{h}'\n"); 85 exit(1); 86 } 87 my $Host = $opts{h}; 88 if ( defined(my $error = $bpc->ConfigRead($Host)) ) { 89 print(STDERR "BackupPC_backupDuplicate: Can't read $Host's config file: $error\n"); 90 exit(1); 91 } 92 %Conf = $bpc->Conf(); 93 94 if ( !$opts{m} 95 && !defined($bpc->ServerConnect($Conf{ServerHost}, $Conf{ServerPort})) 96 && (my $status = $bpc->ServerMesg("hostMutex $Host -1 BackupPC_backupDuplicate")) =~ /fail/ ) { 97 print(STDERR "$0: $status (use -m option to force running)\n"); 98 exit(1); 99 } 100 101 BackupPC::XS::Lib::logLevelSet($Conf{XferLogLevel}); 102 my $LogLevel = $Conf{XferLogLevel}; 103 $bpc->ChildInit(); 104 105 # 106 # Write new-style attrib files (<= 4.0.0beta3 uses old-style), which are 0-length 107 # files with the digest encoded in the file name (eg: attrib_md5HexDigest). We 108 # can still read the old-style files, and we upgrade them as we go. 109 # 110 BackupPC::XS::Attrib::backwardCompat(0, 0); 111 112 my @Backups = $bpc->BackupInfoRead($Host); 113 114 my($lastIdx, $lastNum, $newIdx, $newNum); 115 116 for ( my $i = 0 ; $i < @Backups ; $i++ ) { 117 if ( !defined($lastNum) || $lastNum < $Backups[$i]{num} ) { 118 $lastIdx = $i; 119 $lastNum = $Backups[$i]{num}; 120 } 121 } 122 if ( @Backups == 0 || !defined($lastIdx) ) { 123 print(STDERR "BackupPC_backupDuplicate: no backups on host $Host\n"); 124 exit(1); 125 } 126 my $Compress = $Backups[$lastIdx]{compress}; 127 my $SrcDir = "$TopDir/pc/$Host/$lastNum"; 128 my $Inode = 1; 129 my $DestDir; 130 $newIdx = @Backups; 131 $newNum = $lastNum; 132 do { 133 $newNum++; 134 $DestDir = "$TopDir/pc/$Host/$newNum"; 135 } while ( -d $DestDir ); 136 137 for ( my $i = 0 ; $i < @Backups ; $i++ ) { 138 $Inode = $Backups[$i]{inodeLast} + 1 if ( $Inode <= $Backups[$i]{inodeLast} ); 139 } 140 141 %{$Backups[$newIdx]} = %{$Backups[$lastIdx]}; 142 $Backups[$newIdx]{num} = $newNum; 143 $Backups[$newIdx]{noFill} = 0; 144 $Backups[$newIdx]{compress} = $Compress; 145 $Backups[$newIdx]{version} = $bpc->Version(); 146 $Backups[$newIdx]{keep} = 0; 147 148 print("__bpc_pidStart__ $$\n") if ( !$opts{p} ); 149 150 createFsckFile($DestDir); 151 $bpc->flushXSLibMesgs(); 152 153 if ( $Backups[$lastIdx]{version} >= 4 ) { 154 # 155 # Duplicate a V4 backup by copying #lastNum to #newNum. 156 # We write directly to $DestDir. 157 # 158 if ( !$opts{p} ) { 159 print("__bpc_progress_state__ copy #$lastNum -> #$newNum\n"); 160 } 161 if ( !chdir($SrcDir) ) { 162 print(STDERR "BackupPC_backupDuplicate: cannot chdir to $SrcDir\n"); 163 print("__bpc_pidEnd__ $$\n") if ( !$opts{p} ); 164 exit(1); 165 } 166 # 167 # initially set the prior backup to unfilled, which we will update 168 # once we finish successfully. 169 # 170 $Backups[$lastIdx]{noFill} = 1; 171 print("Copying backup #$lastNum to #$newNum\n") if ( $LogLevel >= 1 ); 172 copy_v4_file(".", "."); 173 BackupPC::DirOps::find($bpc, {wanted => \&copy_v4_file}, ".", 1); 174 $Backups[$lastIdx]{noFill} = 0 if ( $ErrorCnt == 0 ); 175 printProgress(1); 176 } else { 177 # 178 # Duplicate a V3 backup, and turn it into a filled V4 backup. 179 # We write directly to $DestDir. 180 # 181 if ( !$opts{p} ) { 182 print("__bpc_progress_state__ copy #$lastNum -> #$newNum\n"); 183 } 184 my $view = BackupPC::View->new($bpc, $Host, \@Backups); 185 $Backups[$newIdx]{nFiles} = 0; 186 $Backups[$newIdx]{size} = 0; 187 $Backups[$newIdx]{nFilesExist} = 0; 188 $Backups[$newIdx]{sizeExist} = 0; 189 $Backups[$newIdx]{sizeExistComp} = 0; 190 $Backups[$newIdx]{nFilesNew} = 0; 191 $Backups[$newIdx]{sizeNew} = 0; 192 $Backups[$newIdx]{sizeNewComp} = 0; 193 194 my $deltaInfo = BackupPC::XS::DeltaRefCnt::new($DestDir); 195 196 print("Copying v3 backup #$lastNum to v4 #$newNum\n") if ( $LogLevel >= 1 ); 197 foreach my $shareName ( $view->shareList($lastNum) ) { 198 my $ac = BackupPC::XS::AttribCache::new($Host, $newNum, $shareName, $Compress); 199 $ac->setDeltaInfo($deltaInfo); 200 $bpc->flushXSLibMesgs(); 201 if ( $view->find($lastNum, $shareName, "/", 0, \&copy_v3_file, $ac, $deltaInfo) ) { 202 print(STDERR "BackupPC_backupDuplicate: bad share '$shareName'\n"); 203 $ErrorCnt++; 204 next; 205 } 206 $ac->flush(1); 207 } 208 $Backups[$newIdx]{inodeLast} = $Inode + 1; 209 $deltaInfo->flush(); 210 printProgress(); 211 } 212 213 # 214 # merge in backups (in case it changed) 215 # 216 my @newBackups = $bpc->BackupInfoRead($Host); 217 %{$newBackups[scalar(@newBackups)]} = %{$Backups[$newIdx]}; 218 @Backups = @newBackups; 219 $bpc->BackupInfoWrite($Host, @Backups); 220 BackupPC::Storage->backupInfoWrite("$TopDir/pc/$Host", $Backups[$newIdx]{num}, $Backups[$newIdx], 1); 221 $ErrorCnt += BackupPC::XS::Lib::logErrorCntGet(); 222 $bpc->flushXSLibMesgs(); 223 unlink("$DestDir/refCnt/needFsck.dup") if ( $ErrorCnt == 0 ); 224 my $optsp = " -p" if ( $opts{p} ); 225 $ErrorCnt++ if ( system("$BinDir/BackupPC_refCountUpdate -h $Host -o 0$optsp") != 0 ); 226 print("BackupPC_backupDuplicate: got $ErrorCnt errors and $FileErrorCnt file open errors\n"); 227 print("__bpc_pidEnd__ $$\n") if ( !$opts{p} ); 228 exit($ErrorCnt ? 1 : 0); 229 230 sub copy_v4_file 231 { 232 my($name, $path) = @_; 233 234 printf("Got path = %s, name = %s, e,d,f = %d,%d,%d\n", $path, $name, -e $path, -d $path, -f $path) 235 if ( $LogLevel >= 5 ); 236 if ( -d $path ) { 237 print("Creating directory $DestDir/$path\n") if ( $LogLevel >= 4 ); 238 eval { mkpath("$DestDir/$path", 0, 0777) }; 239 if ( $@ ) { 240 print(STDERR "Can't mkpath $DestDir/$path ($@)\n"); 241 $ErrorCnt++; 242 } 243 # 244 # this is actually the directory count; it's expensive to get the file count since 245 # we'd have to open every attrib file 246 # 247 $FileCnt++; 248 return; 249 } 250 if ( !copy($path, "$DestDir/$path") ) { 251 print(STDERR "Can't copy $path to $DestDir/$path\n"); 252 $ErrorCnt++; 253 } 254 printProgress(1) if ( $FileCnt >= $FileCntNext ); 255 } 256 257 # 258 # Read a modest-sized file 259 # 260 sub readFile 261 { 262 my($path) = @_; 263 my $f = BackupPC::XS::FileZIO::open($path, 0, $Compress); 264 my $data; 265 if ( !defined($f) || $f->read(\$data, 65536) < 0 ) { 266 print(STDERR "Unable to open/read file $path\n"); 267 $f->close() if ( defined($f) ); 268 $FileErrorCnt++; 269 return; 270 } 271 $f->close(); 272 return $data; 273 } 274 275 sub copy_v3_file 276 { 277 my($a, $ac, $deltaInfo) = @_; 278 my $fileName = $a->{relPath}; 279 my $copyFile = 0; 280 281 print("copy_v3_file: $a->{relPath}\n") if ( $LogLevel >= 5 ); 282 283 if ( $a->{type} == BPC_FTYPE_DIR ) { 284 my $path = $ac->getFullMangledPath($a->{relPath}); 285 if ( !-d $path ) { 286 eval { mkpath($path, 0, 0777) }; 287 if ( $@ ) { 288 print("Can't mkpath $path ($@)\n"); 289 $ErrorCnt++; 290 return -1; 291 } 292 print("copy_v3_file: mkpath $path\n") if ( $LogLevel >= 4 ); 293 } 294 } else { 295 printProgress() if ( ++$FileCnt >= $FileCntNext ); 296 } 297 $copyFile = 1 298 if ( $a->{type} == BPC_FTYPE_FILE 299 || $a->{type} == BPC_FTYPE_SYMLINK 300 || $a->{type} == BPC_FTYPE_CHARDEV 301 || $a->{type} == BPC_FTYPE_BLOCKDEV ); 302 # 303 # TODO: confirm charset for v3 symlink contents: do we need to convert to utf8? 304 # 305 # assign inodes; handle hardlinks and nlinks 306 # 307 if ( $a->{type} == BPC_FTYPE_HARDLINK ) { 308 # 309 # old-style hardlink: the file gives the path of the linked-to file, 310 # which we might or might not have processed yet. 311 # 312 my $target = readFile($a->{fullPath}); 313 $target =~ s{^\.?/+}{/}; 314 $target = "/$target" if ( $target !~ m{^/} ); 315 316 print("Got hardlink from $a->{relPath} to $target\n") if ( $LogLevel >= 5 ); 317 if ( defined(my $aCurr = $ac->get($target)) ) { 318 # 319 # Either the target file has been processed, or another hardlink 320 # has created this target entry. Copy the attributes and increment 321 # the link count. Note that normal files have a link count of 0, 322 # os if the link count is 0, we set it to 2. 323 # 324 my $origRelPath = $a->{relPath}; 325 $a = $aCurr; 326 if ( $a->{nlinks} == 0 ) { 327 $a->{nlinks} = 2; 328 } else { 329 $a->{nlinks}++; 330 } 331 $ac->set($origRelPath, $a); 332 $ac->set($target, $a); 333 print("target $target exists, set both inode = $a->{inode}, nlinks = $a->{nlinks}\n") 334 if ( $LogLevel >= 5 ); 335 return; 336 } else { 337 # 338 # Target file not processed. Just set attributes with one link, 339 # so that an inode is created for us and target. 340 # 341 $a->{type} = BPC_FTYPE_FILE; 342 $a->{nlinks} = 1; 343 $a->{inode} = $Inode++; 344 $ac->set($a->{relPath}, $a); 345 $ac->set($target, $a); 346 print("target $target doesn't exist, set both inode = $a->{inode}, nlinks = $a->{nlinks}\n") 347 if ( $LogLevel >= 5 ); 348 return; 349 } 350 } else { 351 my $aCurr = $ac->get($a->{relPath}); 352 if ( defined($aCurr) ) { 353 # 354 # we are processing the target of other hardlinks; keep the inode 355 # and increment the link count. 356 # 357 $a->{inode} = $aCurr->{inode}; 358 $a->{nlinks} = $aCurr->{nlinks} + 1; 359 } else { 360 # 361 # new non-hardlink has a new inode, and no links 362 # 363 $a->{inode} = $Inode++; 364 $a->{nlinks} = 0; 365 } 366 } 367 368 if ( $copyFile ) { 369 my $f = BackupPC::XS::FileZIO::open($a->{fullPath}, 0, $Compress); 370 my($size, $data, $data1MB, $fileSize, $found, @s, $thisRead); 371 if ( !defined($f) ) { 372 print(STDERR "Unable to open file $a->{fullPath}\n"); 373 $FileErrorCnt++; 374 return; 375 } 376 my $md5 = Digest::MD5->new(); 377 do { 378 $thisRead = $f->read(\$data, 1 << 20); 379 if ( $thisRead < 0 ) { 380 print(STDERR "error reading file $a->{fullPath} (read returns $thisRead)\n"); 381 $ErrorCnt++; 382 $f->close(); 383 return; 384 } elsif ( $thisRead > 0 ) { 385 $md5->add($data); 386 $fileSize += length($data); 387 $data1MB .= $data if ( length($data1MB) < (1 << 20) ); 388 } 389 } while ( $thisRead > 0 ); 390 391 my $inode = (stat($a->{fullPath}))[1]; 392 my $digest4 = $md5->digest(); 393 my $digest3 = $bpc->Buffer2MD5_v3(Digest::MD5->new(), $fileSize, \$data1MB); 394 my $path3 = $bpc->MD52Path_v3($digest3, $Compress); 395 my $path4 = $bpc->MD52Path($digest4, $Compress); 396 my $i = -1; 397 398 print("$a->{relPath}: path3 = $path3, path4 = $path4\n") if ( $LogLevel >= 5 ); 399 400 # 401 # see if it's already in the v4 pool 402 # 403 if ( $fileSize == 0 ) { 404 $found = 1; 405 print("$a->{relPath}: empty file in pool by default\n") if ( $LogLevel >= 4 ); 406 } elsif ( (@s = stat($path4)) && $s[1] == $inode ) { 407 $found = 1; 408 print("$a->{relPath}: already in pool4 $path4\n") if ( $LogLevel >= 4 ); 409 } elsif ( !-f $path4 ) { 410 # 411 # look in v3 pool for match 412 # 413 while ( 1 ) { 414 my $testPath = $path3; 415 $testPath .= "_$i" if ( $i >= 0 ); 416 @s = stat($testPath); 417 last if ( !@s ); 418 if ( $s[1] != $inode ) { 419 $i++; 420 next; 421 } 422 # 423 # Found it! 424 # 425 print("$a->{relPath}: found at $testPath; link/move to $path4\n") if ( $LogLevel >= 4 ); 426 $found = 1; 427 my $dir4 = $path4; 428 $dir4 =~ s{(.*)/.*}{$1}; 429 if ( !-d $dir4 ) { 430 print("Creating directory $dir4\n") if ( $LogLevel >= 5 ); 431 eval { mkpath($dir4, 0, 0777) }; 432 if ( $@ ) { 433 print(STDERR "Can't mkpath $dir4 ($@)\n"); 434 $ErrorCnt++; 435 $found = 0; 436 } 437 } 438 if ( !link($testPath, $path4) ) { 439 print(STDERR "Can't link($testPath,$path4)\n"); 440 $ErrorCnt++; 441 $found = 0; 442 } elsif ( unlink($testPath) != 1 ) { 443 print(STDERR "Can't unlink($testPath)\n"); 444 $ErrorCnt++; 445 $found = 0; 446 } 447 last; 448 } 449 } 450 # 451 # check one more time for the V4 pool in case someone else just moved it there 452 # (in which case the link error above is actually not an error). 453 # 454 if ( !$found && (@s = stat($path4)) && $s[1] == $inode ) { 455 $found = 1; 456 print("$a->{relPath}: rediscovered in pool4 $path4; prior link error is benign\n"); 457 $ErrorCnt-- if ( $ErrorCnt > 0 ); # reverse increment above 458 } 459 460 if ( $found ) { 461 $f->close(); 462 $a->{digest} = $digest4; 463 $deltaInfo->update($Compress, $digest4, 1) if ( length($digest4) ); 464 $Backups[$newIdx]{nFiles}++; 465 $Backups[$newIdx]{nFilesExist}++; 466 if ( $a->{type} == BPC_FTYPE_FILE || $a->{type} == BPC_FTYPE_SYMLINK ) { 467 $Backups[$newIdx]{size} += $fileSize; 468 $Backups[$newIdx]{sizeExist} += $fileSize; 469 $Backups[$newIdx]{sizeExistComp} += $s[7]; 470 } 471 } else { 472 # 473 # fall back on copying it to the new pool 474 # 475 $f->rewind(); 476 my $pw = BackupPC::XS::PoolWrite::new($Compress); 477 while ( $f->read(\$data, 1 << 20) > 0 ) { 478 $pw->write(\$data); 479 $size += length($data); 480 } 481 $f->close(); 482 my($match, $digest, $poolSize, $errorCnt) = $pw->close(); 483 if ( $LogLevel >= 5 ) { 484 my $digestStr = unpack("H*", $digest); 485 print( 486 "poolWrite->close $fileName: returned match $match, digest $digestStr, poolSize $poolSize, errCnt $errorCnt\n" 487 ); 488 } 489 $ErrorCnt += $errorCnt; 490 $a->{digest} = $digest; 491 $deltaInfo->update($Compress, $digest, 1) if ( length($digest) ); 492 $bpc->flushXSLibMesgs(); 493 $Backups[$newIdx]{nFiles}++; 494 if ( $match ) { 495 $Backups[$newIdx]{nFilesExist}++; 496 } else { 497 $Backups[$newIdx]{nFilesNew}++; 498 } 499 if ( $a->{type} == BPC_FTYPE_FILE || $a->{type} == BPC_FTYPE_SYMLINK ) { 500 $Backups[$newIdx]{size} += $size; 501 if ( $match ) { 502 $Backups[$newIdx]{sizeExist} += $size; 503 $Backups[$newIdx]{sizeExistComp} += $poolSize; 504 } else { 505 $Backups[$newIdx]{sizeNew} += $size; 506 $Backups[$newIdx]{sizeNewComp} += $poolSize; 507 } 508 } 509 } 510 } elsif ( $a->{type} != BPC_FTYPE_DIR ) { 511 $Backups[$newIdx]{nFiles}++; 512 $Backups[$newIdx]{nFilesExist}++; 513 } 514 print("setting $a->{relPath} attrib (type $a->{type}, inode $a->{inode}, nlinks $a->{nlinks})\n") 515 if ( $LogLevel >= 5 ); 516 $ac->set($a->{relPath}, $a); 517 $bpc->flushXSLibMesgs(); 518 } 519 520 # 521 # Create a needFsck file, so if we are killed and can't recover, we can 522 # make sure an fsck is run next time. 523 # 524 sub createFsckFile 525 { 526 my($destDir) = @_; 527 mkpath("$destDir/refCnt", 0, 0777); 528 my $needFsckFH; 529 if ( !(open($needFsckFH, ">", "$destDir/refCnt/needFsck.dup") && close($needFsckFH)) ) { 530 print("BackupPC_backupDuplicate: can't create $destDir/refCnt/needFsck.dup ($?)\n"); 531 } 532 } 533 534 sub printProgress 535 { 536 my($dirCnt) = @_; 537 538 $FileCntNext = $FileCnt + 100; 539 return if ( $opts{p} ); 540 if ( $dirCnt ) { 541 print("__bpc_progress_fileCnt__ $FileCnt dirs\n"); 542 } else { 543 print("__bpc_progress_fileCnt__ $FileCnt\n"); 544 } 545 }