"Fossies" - the Fresh Open Source Software Archive  

Source code changes of the file "bin/BackupPC_nightly" between
BackupPC-4.3.2.tar.gz and BackupPC-4.4.0.tar.gz

About: BackupPC is a high-performance, enterprise-grade system for backing up Linux and WinXX PCs and laptops to a server’s disk (http/cgi user interface).

[ To the main BackupPC source changes report ]

BackupPC_nightly  (BackupPC-4.3.2):BackupPC_nightly  (BackupPC-4.4.0)
skipping to change at line 79 skipping to change at line 79
# This program is distributed in the hope that it will be useful, # This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details. # GNU General Public License for more details.
# #
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
# #
#======================================================================== #========================================================================
# #
# Version 4.3.2, released 17 Feb 2020. # Version 4.4.0, released 20 Jun 2020.
# #
# See http://backuppc.sourceforge.net. # See http://backuppc.sourceforge.net.
# #
#======================================================================== #========================================================================
use strict; use strict;
no utf8; no utf8;
use lib "__INSTALLDIR__/lib"; use lib "__INSTALLDIR__/lib";
use BackupPC::Lib qw( :BPC_DT_ALL ); use BackupPC::Lib qw( :BPC_DT_ALL );
use BackupPC::XS; use BackupPC::XS;
use BackupPC::DirOps; use BackupPC::DirOps;
use Getopt::Std; use Getopt::Std;
use File::Path; use File::Path;
use Data::Dumper; use Data::Dumper;
die("BackupPC::Lib->new failed\n") if ( !(my $bpc = BackupPC::Lib->new) ); die("BackupPC::Lib->new failed\n") if ( !(my $bpc = BackupPC::Lib->new) );
skipping to change at line 142 skipping to change at line 143
exit(1); exit(1);
} }
my $reply = $bpc->ServerMesg("status hosts"); my $reply = $bpc->ServerMesg("status hosts");
$reply = $1 if ( $reply =~ /(.*)/s ); $reply = $1 if ( $reply =~ /(.*)/s );
eval($reply); eval($reply);
} }
########################################################################### ###########################################################################
# V3 pool: get statistics, and remove files that have only one link. # V3 pool: get statistics, and remove files that have only one link.
########################################################################### ###########################################################################
my $fileCnt; # total number of v3 files my $fileCnt; # total number of v3 files
my $dirCnt; # total number of v3 directories my $dirCnt; # total number of v3 directories
my $blkCnt; # total block size of v3 files my $blkCnt; # total block size of v3 files
my $fileCntRm; # total number of removed v3 files my $fileCntRm; # total number of removed v3 files
my $blkCntRm; # total block size of removed v3 files my $blkCntRm; # total block size of removed v3 files
my $fileCntRep; # total number of v3 file names containing "_", ie: files my $fileCntRep; # total number of v3 file names containing "_", ie: files
# that have repeated md5 checksums # that have repeated md5 checksums
my $fileRepMax; # worse case number of v3 files that have repeated checksums my $fileRepMax; # worse case number of v3 files that have repeated checksu
# (ie: max(nnn+1) for all names xxxxxxxxxxxxxxxx_nnn) ms
my $fileLinkMax; # maximum number of hardlinks on a v3 pool file # (ie: max(nnn+1) for all names xxxxxxxxxxxxxxxx_nnn)
my $fileLinkTotal; # total number of hardlinks on entire v3 pool my $fileLinkMax; # maximum number of hardlinks on a v3 pool file
my $fileCntRename; # number of renamed v3 files (to keep file numbering my $fileLinkTotal; # total number of hardlinks on entire v3 pool
# contiguous) my $fileCntRename; # number of renamed v3 files (to keep file numbering
my %FixList; # list of v3 paths that need to be renamed to avoid # contiguous)
# new holes my %FixList; # list of v3 paths that need to be renamed to avoid
# new holes
if ( $Conf{PoolV3Enabled} ) { if ( $Conf{PoolV3Enabled} ) {
ScanAndCleanV3Pool(); ScanAndCleanV3Pool();
} }
########################################################################### ###########################################################################
# Prior to V3, we need to tell BackupPC that it is now ok to start running # Prior to V3, we need to tell BackupPC that it is now ok to start running
# BackupPC_dump commands. In V3+ they are decoupled, so this isn't actually # BackupPC_dump commands. In V3+ they are decoupled, so this isn't actually
# needed anymore. # needed anymore.
########################################################################### ###########################################################################
printf("BackupPC_nightly lock_off\n"); printf("BackupPC_nightly lock_off\n");
########################################################################### ###########################################################################
# V4 pool: run reference count updating and pool cleaning # V4 pool: run reference count updating and pool cleaning
# This runs in parallel for each subset of the pool # This runs in parallel for each subset of the pool
########################################################################### ###########################################################################
if ( $opts{r} ) { if ( $opts{r} ) {
print("log BackupPC_nightly skipping BackupPC_refCountUpdate\n"); print("log BackupPC_nightly skipping BackupPC_refCountUpdate\n");
} else { } else {
$opts{P} ||= 0; $opts{P} ||= 0;
print("log BackupPC_nightly now running BackupPC_refCountUpdate -m -s -c -P print(
$opts{P} -r $poolRangeStart-$poolRangeEnd\n"); "log BackupPC_nightly now running BackupPC_refCountUpdate -m -s -c -P $o
pts{P} -r $poolRangeStart-$poolRangeEnd\n"
);
system("$BinDir/BackupPC_refCountUpdate -m -s -c -P $opts{P} -r $poolRangeSt art-$poolRangeEnd"); system("$BinDir/BackupPC_refCountUpdate -m -s -c -P $opts{P} -r $poolRangeSt art-$poolRangeEnd");
} }
########################################################################### ###########################################################################
# Send email and generation of backupInfo files for each backup # Send email and generation of backupInfo files for each backup
# Also clean any temp pool/cpool files # Also clean any temp pool/cpool files
########################################################################### ###########################################################################
if ( $opts{m} ) { if ( $opts{m} ) {
CleanV4PoolTempFiles(); CleanV4PoolTempFiles();
print("log BackupPC_nightly now running BackupPC_sendEmail\n"); print("log BackupPC_nightly now running BackupPC_sendEmail\n");
skipping to change at line 201 skipping to change at line 205
exit(0); exit(0);
sub GetPoolStats_V3 sub GetPoolStats_V3
{ {
my($file, $fullPath) = @_; my($file, $fullPath) = @_;
my($inode, $nlinks, $nblocks) = (lstat($fullPath))[1, 3, 12]; my($inode, $nlinks, $nblocks) = (lstat($fullPath))[1, 3, 12];
if ( -d _ ) { if ( -d _ ) {
$dirCnt++; $dirCnt++;
return; return;
} elsif ( ! -f _ ) { } elsif ( !-f _ ) {
return; return;
} }
if ( $nlinks == 1 ) { if ( $nlinks == 1 ) {
$blkCntRm += $nblocks; $blkCntRm += $nblocks;
$fileCntRm++; $fileCntRm++;
# #
# Save the files for later batch deletion. # Save the files for later batch deletion.
# #
# This is so we can remove them in inode order, and additionally # This is so we can remove them in inode order, and additionally
# reduce any remaining chance of race condition of linking to # reduce any remaining chance of race condition of linking to
# pool files vs removing pool files. (Other aspects of the # pool files vs removing pool files. (Other aspects of the
# design should eliminate race conditions.) # design should eliminate race conditions.)
# #
push(@PendingDelete, { push(
inode => $inode, @PendingDelete,
path => $fullPath {
} inode => $inode,
path => $fullPath
}
); );
if ( @PendingDelete > $PendingDeleteMax ) { if ( @PendingDelete > $PendingDeleteMax ) {
processPendingDeletes(0); processPendingDeletes(0);
} }
# #
# We must keep repeated files numbered sequential (ie: files # We must keep repeated files numbered sequential (ie: files
# that have the same checksum are appended with _0, _1 etc). # that have the same checksum are appended with _0, _1 etc).
# There are two cases: we remove the base file xxxx, but xxxx_0 # There are two cases: we remove the base file xxxx, but xxxx_0
# exists, or we remove any file of the form xxxx_nnn. We remember # exists, or we remove any file of the form xxxx_nnn. We remember
# the base name and fix it up later (not in the middle of find). # the base name and fix it up later (not in the middle of find).
skipping to change at line 252 skipping to change at line 258
} }
sub processPendingDeletes sub processPendingDeletes
{ {
my($doAll) = @_; my($doAll) = @_;
my @delete; my @delete;
if ( !$doAll ) { if ( !$doAll ) {
@delete = splice(@PendingDelete, 0, $PendingDeleteMax / 2); @delete = splice(@PendingDelete, 0, $PendingDeleteMax / 2);
} else { } else {
@delete = @PendingDelete; @delete = @PendingDelete;
@PendingDelete = (); @PendingDelete = ();
} }
for my $f ( sort({ $a->{inode} <=> $b->{inode} } @delete) ) { for my $f ( sort({ $a->{inode} <=> $b->{inode} } @delete) ) {
my($nlinks) = (lstat($f->{path}))[3]; my($nlinks) = (lstat($f->{path}))[3];
next if ( $nlinks != 1 ); next if ( $nlinks != 1 );
# print("Deleting $f->{path} ($f->{inode})\n"); # print("Deleting $f->{path} ($f->{inode})\n");
unlink($f->{path}); unlink($f->{path});
} }
} }
sub ScanAndCleanV3Pool() sub ScanAndCleanV3Pool()
{ {
my @hexChars = qw(0 1 2 3 4 5 6 7 8 9 a b c d e f); my @hexChars = qw(0 1 2 3 4 5 6 7 8 9 a b c d e f);
for my $pool ( qw(pool cpool) ) { for my $pool ( qw(pool cpool) ) {
print("__bpc_progress_state__ v3 $pool scan\n") if ( !$opts{p} ); print("__bpc_progress_state__ v3 $pool scan\n") if ( !$opts{p} );
for ( my $i = $poolRangeStart ; $i <= $poolRangeEnd ; $i++ ) { for ( my $i = $poolRangeStart ; $i <= $poolRangeEnd ; $i++ ) {
my $dir = "$hexChars[int($i / 16)]/$hexChars[$i % 16]"; my $dir = "$hexChars[int($i / 16)]/$hexChars[$i % 16]";
# print("Doing $pool/$dir\n") if ( ($i % 16) == 0 ); # print("Doing $pool/$dir\n") if ( ($i % 16) == 0 );
$fileCnt = 0; $fileCnt = 0;
$dirCnt = 0; $dirCnt = 0;
$blkCnt = 0; $blkCnt = 0;
$fileCntRm = 0; $fileCntRm = 0;
$blkCntRm = 0; $blkCntRm = 0;
$fileCntRep = 0; $fileCntRep = 0;
$fileRepMax = 0; $fileRepMax = 0;
$fileLinkMax = 0; $fileLinkMax = 0;
$fileLinkTotal = 0; $fileLinkTotal = 0;
$fileCntRename = 0; $fileCntRename = 0;
%FixList = (); %FixList = ();
print("__bpc_progress_fileCnt__ $i/$poolRangeEnd\n") if ( !$opts{p} ); print("__bpc_progress_fileCnt__ $i/$poolRangeEnd\n") if ( !$opts{p} );
BackupPC::DirOps::find($bpc, {wanted => \&GetPoolStats_V3}, "$TopDir /$pool/$dir") BackupPC::DirOps::find($bpc, {wanted => \&GetPoolStats_V3}, "$TopDir /$pool/$dir")
if ( -d "$TopDir/$pool/$dir" ); if ( -d "$TopDir/$pool/$dir" );
my $kb = $blkCnt / 2; my $kb = $blkCnt / 2;
my $kbRm = $blkCntRm / 2; my $kbRm = $blkCntRm / 2;
# #
# Main BackupPC_nightly counts the top-level directory # Main BackupPC_nightly counts the top-level directory
# #
$dirCnt++ if ( $opts{m} && -d "$TopDir/$pool" && $i == 0 ); $dirCnt++ if ( $opts{m} && -d "$TopDir/$pool" && $i == 0 );
# #
# Also count the next level directories # Also count the next level directories
# #
$dirCnt++ if ( ($i % 16) == 0 $dirCnt++ if ( ($i % 16) == 0 && -d "$TopDir/$pool/$hexChars[int($i
&& -d "$TopDir/$pool/$hexChars[int($i / 16)]" ); / 16)]" );
# #
# We need to process all pending deletes before we do the # We need to process all pending deletes before we do the
# renames # renames
# #
if ( @PendingDelete ) { if ( @PendingDelete ) {
sleep(1); sleep(1);
processPendingDeletes(1); processPendingDeletes(1);
} }
# #
# Now make sure that files with repeated checksums are still # Now make sure that files with repeated checksums are still
# sequentially numbered # sequentially numbered
# #
foreach my $name ( sort(keys(%FixList)) ) { foreach my $name ( sort(keys(%FixList)) ) {
my $rmCnt = $FixList{$name} + 1; my $rmCnt = $FixList{$name} + 1;
my $new = -1; my $new = -1;
for ( my $old = -1 ; ; $old++ ) { for ( my $old = -1 ; ; $old++ ) {
my $oldName = $name; my $oldName = $name;
$oldName .= "_$old" if ( $old >= 0 ); $oldName .= "_$old" if ( $old >= 0 );
if ( !-f $oldName ) { if ( !-f $oldName ) {
# #
# We know we are done when we have missed at least # We know we are done when we have missed at least
# the number of files that were removed from this # the number of files that were removed from this
# base name, plus a couple just to be sure # base name, plus a couple just to be sure
# #
last if ( $rmCnt-- <= 0 ); last if ( $rmCnt-- <= 0 );
next; next;
} }
my $newName = $name; my $newName = $name;
$newName .= "_$new" if ( $new >= 0 ); $newName .= "_$new" if ( $new >= 0 );
$new++; $new++;
next if ( $oldName eq $newName ); next if ( $oldName eq $newName );
rename($oldName, $newName); rename($oldName, $newName);
$fileCntRename++; $fileCntRename++;
} }
} }
print("BackupPC_stats $i = $pool,$fileCnt,$dirCnt,$kb,$kbRm," print( "BackupPC_stats $i = $pool,$fileCnt,$dirCnt,$kb,$kbRm,"
. "$fileCntRm,$fileCntRep,$fileRepMax," . "$fileCntRm,$fileCntRep,$fileRepMax,"
. "$fileCntRename,$fileLinkMax,$fileLinkTotal\ . "$fileCntRename,$fileLinkMax,$fileLinkTotal\n");
n");
} }
} }
sleep(1); sleep(1);
processPendingDeletes(1); processPendingDeletes(1);
} }
# #
# Remove any orphan pool write temp files. The file names is three numbers # Remove any orphan pool write temp files. The file names is three numbers
# separated by periods. The first number is the pid, and we check if that # separated by periods. The first number is the pid, and we check if that
# process is still alive. If not, we delete the file. # process is still alive. If not, we delete the file.
# #
sub CleanV4PoolTempFiles sub CleanV4PoolTempFiles
{ {
my $pidRunning = {}; my $pidRunning = {};
foreach my $pool ( qw(pool cpool) ) { foreach my $pool ( qw(pool cpool) ) {
foreach my $e ( @{BackupPC::DirOps::dirRead($bpc, "$TopDir/$pool")} ) { foreach my $e ( @{BackupPC::DirOps::dirRead($bpc, "$TopDir/$pool")} ) {
next if ( $e->{name} !~ /^(\d+)\.\d+\.\d+$/ || !-f "$TopDir/$pool/$e- next if ( $e->{name} !~ /^(\d+)\.\d+\.\d+$/ || !-f "$TopDir/$pool/$e
>{name}" ); ->{name}" );
my $pid = $1; my $pid = $1;
$pidRunning->{$pid} = kill(0, $pid) ? 1 : 0 if ( !defined($pidRunning $pidRunning->{$pid} = kill(0, $pid) ? 1 : 0 if ( !defined($pidRunnin
->{$pid} ) ); g->{$pid}) );
next if ( $pidRunning->{$pid} ); next if ( $pidRunning->{$pid} );
#print("pid $pid: unlink $TopDir/$pool/$e->{name}\n"); #print("pid $pid: unlink $TopDir/$pool/$e->{name}\n");
unlink("$TopDir/$pool/$e->{name}"); unlink("$TopDir/$pool/$e->{name}");
} }
} }
} }
 End of changes. 14 change blocks. 
44 lines changed or deleted 52 lines changed or added
To the Perl Programs (all 27): section of the BackupPC source changes report or the top of this page
Mainly generated by pkgdiff using rfcdiff
Home  |  About  |  Features  |  All  |  Newest  |  Dox  |  Diffs  |  RSS Feeds  |  Screenshots  |  Comments  |  Imprint  |  Privacy  |  HTTP(S)