"Fossies" - the Fresh Open Source Software Archive  

Source code changes of the file "bin/BackupPC_refCountUpdate" between
BackupPC-4.3.2.tar.gz and BackupPC-4.4.0.tar.gz

About: BackupPC is a high-performance, enterprise-grade system for backing up Linux and WinXX PCs and laptops to a server’s disk (http/cgi user interface).

[ To the main BackupPC source changes report ]

BackupPC_refCountUpdate  (BackupPC-4.3.2):BackupPC_refCountUpdate  (BackupPC-4.4.0)
#!/usr/bin/perl #!/usr/bin/perl
#============================================================= -*-perl-*-
#
# BackupPC_refCountUpdate: Pool reference count updater # BackupPC_refCountUpdate: Pool reference count updater
# #
# DESCRIPTION # DESCRIPTION
# #
# BackupPC_refCountUpdate checks the pool reference counts # BackupPC_refCountUpdate checks the pool reference counts
# #
# Usage: BackupPC_refCountUpdate # Usage: BackupPC_refCountUpdate
# #
# AUTHOR # AUTHOR
# Craig Barratt <cbarratt@users.sourceforge.net> # Craig Barratt <cbarratt@users.sourceforge.net>
skipping to change at line 33 skipping to change at line 31
# This program is distributed in the hope that it will be useful, # This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details. # GNU General Public License for more details.
# #
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
# #
#======================================================================== #========================================================================
# #
# Version 4.3.2, released 17 Feb 2020. # Version 4.4.0, released 20 Jun 2020.
# #
# See http://backuppc.sourceforge.net. # See http://backuppc.sourceforge.net.
# #
#======================================================================== #========================================================================
use strict; use strict;
no utf8; no utf8;
use lib "__INSTALLDIR__/lib"; use lib "__INSTALLDIR__/lib";
use Getopt::Std; use Getopt::Std;
use Fcntl qw(:mode); use Fcntl qw(:mode);
use File::Path; use File::Path;
use Data::Dumper; use Data::Dumper;
use BackupPC::Lib; use BackupPC::Lib;
use BackupPC::XS; use BackupPC::XS;
use BackupPC::DirOps qw( :BPC_DT_ALL ); use BackupPC::DirOps qw( :BPC_DT_ALL );
select(STDOUT); $| = 1; select(STDOUT); $| = 1;
my %opts; my %opts;
if ( !getopts("cFfnmpsvh:r:P:o:", \%opts) || @ARGV != 0 || !(defined($opts{h}) | | $opts{m}) ) { if ( !getopts("cFfnmpsvh:r:P:o:", \%opts) || @ARGV != 0 || !(defined($opts{h}) | | $opts{m}) ) {
print <<EOF; print <<EOF;
Usage: Usage:
BackupPC_refCountUpdate -h HOST [-c] [-f] [-F] [-o N] [-p] [-v] BackupPC_refCountUpdate -h HOST [-c] [-f] [-F] [-o N] [-p] [-v]
With no other args, updates count db on backups with poolCntDelta files With no other args, updates count db on backups with poolCntDelta files
and computers the host's total reference counts. Also builds refCnt for and computes the host's total reference counts. Also builds refCnt for
any >=4.0 backups without refCnts. any >=4.0 backups without refCnts.
-f - do an fsck on this HOST, which involves a rebuild of the -f - do an fsck on this HOST, which involves a rebuild of the
last two backup refCnts. poolCntDelta files are ignored. last two backup refCnts. poolCntDelta files are ignored.
Also forces fsck if requested by needFsck flag files Also forces fsck if requested by needFsck flag files
in TopDir/pc/HOST/refCnt. Equivalent to -o 2. in TopDir/pc/HOST/refCnt. Equivalent to -o 2.
-F - rebuild all the >=4.0 per-backup refCnt files for this -F - rebuild all the >=4.0 per-backup refCnt files for this
host. Equivalent to -o 3. host. Equivalent to -o 3.
-c - compare current count db to new db before replacing -c - compare current count db to new db before replacing
-o N - override \$Conf{RefCntFsck}. -o N - override \$Conf{RefCntFsck}.
-p - don't show progress -p - don't show progress
skipping to change at line 112 skipping to change at line 111
my $Hosts = $bpc->HostInfoRead(); my $Hosts = $bpc->HostInfoRead();
my @Hosts = sort(keys(%$Hosts)); my @Hosts = sort(keys(%$Hosts));
if ( $opts{h} ne "" ) { if ( $opts{h} ne "" ) {
if ( !defined($Hosts->{$opts{h}}) ) { if ( !defined($Hosts->{$opts{h}}) ) {
print(STDERR "BackupPC_refCountUpdate: host $opts{h} doesn't exist\n"); print(STDERR "BackupPC_refCountUpdate: host $opts{h} doesn't exist\n");
exit(1); exit(1);
} }
$bpc->ConfigRead($opts{h}); $bpc->ConfigRead($opts{h});
} }
my %Conf = $bpc->Conf(); my %Conf = $bpc->Conf();
my $PoolStats = {}; my $PoolStats = {};
if ( $opts{v} ) { if ( $opts{v} ) {
$Conf{XferLogLevel} = 8; $Conf{XferLogLevel} = 8;
BackupPC::XS::Lib::logLevelSet(8); BackupPC::XS::Lib::logLevelSet(8);
} }
# #
# Write new-style attrib files (<= 4.0.0beta3 uses old-style), which are 0-lengt h # Write new-style attrib files (<= 4.0.0beta3 uses old-style), which are 0-lengt h
# files with the digest encoded in the file name (eg: attrib_md5HexDigest). We # files with the digest encoded in the file name (eg: attrib_md5HexDigest). We
skipping to change at line 167 skipping to change at line 166
# requested. # requested.
# #
if ( $opts{c} ) { if ( $opts{c} ) {
cleanPoolFiles(); cleanPoolFiles();
} elsif ( $opts{s} ) { } elsif ( $opts{s} ) {
statsPrint(); statsPrint();
} }
} }
print("BackupPC_refCountUpdate total errors: $ErrorCnt\n") if ( $ErrorCnt > 0 || $Conf{XferLogLevel} >= 2 ); print("BackupPC_refCountUpdate total errors: $ErrorCnt\n") if ( $ErrorCnt > 0 || $Conf{XferLogLevel} >= 2 );
print("__bpc_pidEnd__ $$\n") if ( !$opts{p} ); print("__bpc_pidEnd__ $$\n") if ( !$opts{p} );
exit($ErrorCnt ? 1 : 0); exit($ErrorCnt ? 1 : 0);
# #
# For a given host, either update the poolCnt files using the poolCntDelta files , # For a given host, either update the poolCnt files using the poolCntDelta files ,
# or if $forceFsck, completely rebuild the poolCnt files from scratch. # or if $forceFsck, completely rebuild the poolCnt files from scratch.
# #
sub updateHostPoolCnt sub updateHostPoolCnt
{ {
my($host, $forceLast2Fsck, $forceFullFsck, $compareCurr) = @_; my($host, $forceLast2Fsck, $forceFullFsck, $compareCurr) = @_;
my $pcDir = "$TopDir/pc/$host"; my $pcDir = "$TopDir/pc/$host";
my $refCntDir = "$pcDir/refCnt"; my $refCntDir = "$pcDir/refCnt";
my $errorCntSave = $ErrorCnt; my $errorCntSave = $ErrorCnt;
my @fsckFiles; my @fsckFiles;
my @Backups = $bpc->BackupInfoRead($host); my @Backups = $bpc->BackupInfoRead($host);
my $startClock = time; my $startClock = time;
mkdir($refCntDir, 0770) if ( !-d $refCntDir ); mkdir($refCntDir, 0770) if ( !-d $refCntDir );
# #
# Grab the lock to make sure no dumps start. Try a non-blocking request # Grab the lock to make sure no dumps start. Try a non-blocking request
# first, so we can warn the user if we have to wait. # first, so we can warn the user if we have to wait.
# #
my $lockFd = BackupPC::XS::DirOps::lockRangeFile("$refCntDir/LOCK", 0, 1, 0) ; my $lockFd = BackupPC::XS::DirOps::lockRangeFile("$refCntDir/LOCK", 0, 1, 0) ;
if ( $lockFd < 0 ) { if ( $lockFd < 0 ) {
skipping to change at line 258 skipping to change at line 258
print("BackupPC_refCountUpdate: doing fsck on $host #$bkupNum since $bkupRefCntDir doesn't exist\n"); print("BackupPC_refCountUpdate: doing fsck on $host #$bkupNum since $bkupRefCntDir doesn't exist\n");
mkdir($bkupRefCntDir, 0770); mkdir($bkupRefCntDir, 0770);
push(@$bkupList, {bkupNum => $bkupNum, fsck => 1, compress => $compr ess}); push(@$bkupList, {bkupNum => $bkupNum, fsck => 1, compress => $compr ess});
next; next;
} }
my $entries = BackupPC::DirOps::dirRead($bpc, $bkupRefCntDir); my $entries = BackupPC::DirOps::dirRead($bpc, $bkupRefCntDir);
foreach my $e ( @$entries ) { foreach my $e ( @$entries ) {
unlink("$bkupRefCntDir/$e->{name}") if ( $e->{name} =~ /^poolCntNew/ ); unlink("$bkupRefCntDir/$e->{name}") if ( $e->{name} =~ /^poolCntNew/ );
if ( $e->{name} =~ /^needFsck/ || $e->{name} =~ /^tpoolCntDelta/ ) { if ( $e->{name} =~ /^needFsck/ || $e->{name} =~ /^tpoolCntDelta/ ) {
unlink("$bkupRefCntDir/$e->{name}"); unlink("$bkupRefCntDir/$e->{name}");
$gotFsck = 1; $gotFsck = 1;
} }
$gotDelta = 1 if ( $e->{name} =~ /^poolCntDelta/ ); $gotDelta = 1 if ( $e->{name} =~ /^poolCntDelta/ );
$gotPoolCnt = 1 if ( $e->{name} =~ /^poolCnt\./ ); $gotPoolCnt = 1 if ( $e->{name} =~ /^poolCnt\./ );
$gotNoPoolCntOk = 1 if ( $e->{name} =~ /^noPoolCntOk/ ); $gotNoPoolCntOk = 1 if ( $e->{name} =~ /^noPoolCntOk/ );
} }
print("BackupPC_refCountUpdate: host $host #$bkupNum: gotFsck = $gotFsck print( "BackupPC_refCountUpdate: host $host #$bkupNum: gotFsck = $gotFs
, gotDelta = $gotDelta," ck, gotDelta = $gotDelta,"
. " gotPoolCnt = $gotP . " gotPoolCnt = $gotPoolCnt, gotNoPoolCntOk = $gotNoPoolCntOk\n")
oolCnt, gotNoPoolCntOk = $gotNoPoolCntOk\n") if ( $Conf{XferLogLevel} >= 4 );
if ( $Conf{XferLogLevel} >= 4 );
# #
# Apply policy in $ConfRefCntFsck: # Apply policy in $ConfRefCntFsck:
# 0: no additional fsck # 0: no additional fsck
# 1: do an fsck on the last backup if it is from a full backup # 1: do an fsck on the last backup if it is from a full backup
# 2: do an fsck on the last two backups always (handled above) # 2: do an fsck on the last two backups always (handled above)
# 3: do a full fsck on all the backups (handled above) # 3: do a full fsck on all the backups (handled above)
# #
if ( $ConfRefCntFsck == 1 && $i == @Backups - 1 && $Backups[$i]{type} eq "full" ) { if ( $ConfRefCntFsck == 1 && $i == @Backups - 1 && $Backups[$i]{type} eq "full" ) {
print("BackupPC_refCountUpdate: doing fsck on $host #$bkupNum (full) since \$ConfRefCntFsck == 1\n"); print("BackupPC_refCountUpdate: doing fsck on $host #$bkupNum (full) since \$ConfRefCntFsck == 1\n");
$gotFsck = 1; $gotFsck = 1;
skipping to change at line 296 skipping to change at line 296
if ( $forceFullFsck || ($forceLast2Fsck && $i >= @Backups - 2) || $gotFs ck ) { if ( $forceFullFsck || ($forceLast2Fsck && $i >= @Backups - 2) || $gotFs ck ) {
push(@$bkupList, {bkupNum => $bkupNum, fsck => 1, compress => $compr ess}); push(@$bkupList, {bkupNum => $bkupNum, fsck => 1, compress => $compr ess});
} elsif ( $gotDelta ) { } elsif ( $gotDelta ) {
push(@$bkupList, {bkupNum => $bkupNum, fsck => 0, compress => $compr ess}); push(@$bkupList, {bkupNum => $bkupNum, fsck => 0, compress => $compr ess});
} }
} }
while ( (my $b = shift(@$bkupList)) ) { while ( (my $b = shift(@$bkupList)) ) {
my $errorCntSave0 = $ErrorCnt; my $errorCntSave0 = $ErrorCnt;
if ( processOneHostBackup($host, $b->{bkupNum}, $b->{fsck}, $b->{compres s}) && !$b->{fsck} ) { if ( processOneHostBackup($host, $b->{bkupNum}, $b->{fsck}, $b->{compres s}) && !$b->{fsck} ) {
print("BackupPC_refCountUpdate: given errors, redoing host $host #$b print(
->{bkupNum} with fsck (reset errorCnt to $errorCntSave0)\n"); "BackupPC_refCountUpdate: given errors, redoing host $host #$b->
{bkupNum} with fsck (reset errorCnt to $errorCntSave0)\n"
);
$ErrorCnt = $errorCntSave0; $ErrorCnt = $errorCntSave0;
unshift(@$bkupList, {bkupNum => $b->{bkupNum}, fsck => 1, compress = > $b->{compress}}); unshift(@$bkupList, {bkupNum => $b->{bkupNum}, fsck => 1, compress = > $b->{compress}});
} }
} }
# #
# Now add up all the per-backup counts to get the per-host totals # Now add up all the per-backup counts to get the per-host totals
# #
my $errorCntSave0 = $ErrorCnt; my $errorCntSave0 = $ErrorCnt;
print("BackupPC_refCountUpdate: computing totals for host $host\n") if ( $Co nf{XferLogLevel} >= 2 ); print("BackupPC_refCountUpdate: computing totals for host $host\n") if ( $Co nf{XferLogLevel} >= 2 );
my $needFsck = updateHostRefCounts($host); my $needFsck = updateHostRefCounts($host);
my $bkupRedoDone = {}; my $bkupRedoDone = {};
while ( $needFsck ) { while ( $needFsck ) {
my $listText = join(", ", @$needFsck); my $listText = join(", ", @$needFsck);
print("BackupPC_refCountUpdate: due to errors, doing fsck on host $host print(
backups $listText; (reset errorCnt to $errorCntSave0)\n"); "BackupPC_refCountUpdate: due to errors, doing fsck on host $host ba
ckups $listText; (reset errorCnt to $errorCntSave0)\n"
);
$ErrorCnt = $errorCntSave0; $ErrorCnt = $errorCntSave0;
foreach my $bkupNum ( @$needFsck ) { foreach my $bkupNum ( @$needFsck ) {
next if ( $bkupRedoDone->{$bkupNum} ); next if ( $bkupRedoDone->{$bkupNum} );
my $compress = 1; my $compress = 1;
for ( my $i = 0 ; $i < @Backups ; $i++ ) { for ( my $i = 0 ; $i < @Backups ; $i++ ) {
next if ( $Backups[$i]{num} != $bkupNum ); next if ( $Backups[$i]{num} != $bkupNum );
$compress = $Backups[$i]{compress}; $compress = $Backups[$i]{compress};
last; last;
} }
processOneHostBackup($host, $bkupNum, 1, $compress); processOneHostBackup($host, $bkupNum, 1, $compress);
skipping to change at line 337 skipping to change at line 341
# #
# and rename the new count files to replace the current ones # and rename the new count files to replace the current ones
# #
renameNewCntFiles($host, $refCntDir, $compareCurr, "total"); renameNewCntFiles($host, $refCntDir, $compareCurr, "total");
# #
# Now give the lock back # Now give the lock back
# #
BackupPC::XS::DirOps::unlockRangeFile($lockFd); BackupPC::XS::DirOps::unlockRangeFile($lockFd);
printf("BackupPC_refCountUpdate: host %s got %d errors (took %d secs)\n", printf(
$host, $ErrorCnt - $errorCntSave, time - $startClock); "BackupPC_refCountUpdate: host %s got %d errors (took %d secs)\n",
$host,
$ErrorCnt - $errorCntSave,
time - $startClock
);
$bpc->flushXSLibMesgs(); $bpc->flushXSLibMesgs();
} }
sub processOneHostBackup sub processOneHostBackup
{ {
my($host, $bkupNum, $fsck, $compress) = @_; my($host, $bkupNum, $fsck, $compress) = @_;
my $pcDir = "$TopDir/pc/$host"; my $pcDir = "$TopDir/pc/$host";
my $bkupRefCntDir = "$pcDir/$bkupNum/refCnt"; my $bkupRefCntDir = "$pcDir/$bkupNum/refCnt";
my $errorCntSave0 = $ErrorCnt; my $errorCntSave0 = $ErrorCnt;
my $fd; my $fd;
# #
# add a placeholder fsck request, in case we die unexpectedly # add a placeholder fsck request, in case we die unexpectedly
# #
open($fd, ">", "$bkupRefCntDir/needFsck.refCountUpdate") && close($fd); open($fd, ">", "$bkupRefCntDir/needFsck.refCountUpdate") && close($fd);
print("BackupPC_refCountUpdate: processing host $host #$bkupNum (fsck = $fsc k)\n") if ( $Conf{XferLogLevel} >= 2 ); print("BackupPC_refCountUpdate: processing host $host #$bkupNum (fsck = $fsc k)\n") if ( $Conf{XferLogLevel} >= 2 );
skipping to change at line 378 skipping to change at line 386
$ErrorCnt += $err; $ErrorCnt += $err;
$deltaInfo->flush(); $deltaInfo->flush();
$bpc->flushXSLibMesgs(); $bpc->flushXSLibMesgs();
# #
# Update inodeLast for this backup if it is larger than what's stored in backups # Update inodeLast for this backup if it is larger than what's stored in backups
# #
my @backups = $bpc->BackupInfoRead($host); my @backups = $bpc->BackupInfoRead($host);
for ( my $i = 0 ; $i < @backups ; $i++ ) { for ( my $i = 0 ; $i < @backups ; $i++ ) {
next if ( $backups[$i]{num} != $bkupNum || $backups[$i]{inodeLast} > $inodeMax ); next if ( $backups[$i]{num} != $bkupNum || $backups[$i]{inodeLast} > $inodeMax );
$inodeMax += 2; $inodeMax += 2;
print("BackupPC_refCountUpdate: $host #$bkupNum inodeLast set to $in print(
odeMax (was $backups[$i]{inodeLast})\n"); "BackupPC_refCountUpdate: $host #$bkupNum inodeLast set to $inod
eMax (was $backups[$i]{inodeLast})\n");
$backups[$i]{inodeLast} = $inodeMax; $backups[$i]{inodeLast} = $inodeMax;
BackupPC::Storage->backupInfoWrite($pcDir, $bkupNum, $backups[$i], 1 ); BackupPC::Storage->backupInfoWrite($pcDir, $bkupNum, $backups[$i], 1 );
$bpc->BackupInfoWrite($host, @backups); $bpc->BackupInfoWrite($host, @backups);
last; last;
} }
} }
print("__bpc_progress_state__ cntUpdate #$bkupNum\n") if ( !$opts{p} ); print("__bpc_progress_state__ cntUpdate #$bkupNum\n") if ( !$opts{p} );
updateHostDelta2Cnt($host, $bkupRefCntDir, $fsck, 0, $bkupNum); updateHostDelta2Cnt($host, $bkupRefCntDir, $fsck, 0, $bkupNum);
$bpc->flushXSLibMesgs(); $bpc->flushXSLibMesgs();
# #
# delete the noPoolCntOk if there are pool files, or create one # delete the noPoolCntOk if there are pool files, or create one
# if there are none # if there are none
# #
my($gotNoPoolCntOk, $gotPoolCnt); my($gotNoPoolCntOk, $gotPoolCnt);
foreach my $e ( @{BackupPC::DirOps::dirRead($bpc, $bkupRefCntDir)} ) { foreach my $e ( @{BackupPC::DirOps::dirRead($bpc, $bkupRefCntDir)} ) {
$gotPoolCnt = 1 if ( $e->{name} =~ /^poolCnt\./ ); $gotPoolCnt = 1 if ( $e->{name} =~ /^poolCnt\./ );
$gotNoPoolCntOk = 1 if ( $e->{name} =~ /^noPoolCntOk/ ); $gotNoPoolCntOk = 1 if ( $e->{name} =~ /^noPoolCntOk/ );
} }
if ( $gotPoolCnt && $gotNoPoolCntOk ) { if ( $gotPoolCnt && $gotNoPoolCntOk ) {
unlink("$bkupRefCntDir/noPoolCntOk"); unlink("$bkupRefCntDir/noPoolCntOk");
} elsif ( !$gotPoolCnt ) { } elsif ( !$gotPoolCnt ) {
open(my $fd, ">", "$bkupRefCntDir/noPoolCntOk") && close($fd); open(my $fd, ">", "$bkupRefCntDir/noPoolCntOk") && close($fd);
} }
unlink("$bkupRefCntDir/needFsck.refCountUpdate"); unlink("$bkupRefCntDir/needFsck.refCountUpdate");
return 1 if ( $errorCntSave0 != $ErrorCnt ); return 1 if ( $errorCntSave0 != $ErrorCnt );
} }
# #
# Process all the per-backup ref counts to create the total ref count for the ho st. # Process all the per-backup ref counts to create the total ref count for the ho st.
# Returns undef on success, or a listref of backup numbers that need fscks. # Returns undef on success, or a listref of backup numbers that need fscks.
# #
sub updateHostRefCounts sub updateHostRefCounts
{ {
my($host) = @_; my($host) = @_;
my @Backups = $bpc->BackupInfoRead($host); my @Backups = $bpc->BackupInfoRead($host);
my $refCntDestDir = "$TopDir/pc/$host/refCnt"; my $refCntDestDir = "$TopDir/pc/$host/refCnt";
my $needFsck = {}; my $needFsck = {};
print("__bpc_progress_state__ sumUpdate\n") if ( !$opts{p} ); print("__bpc_progress_state__ sumUpdate\n") if ( !$opts{p} );
for ( my $refCntFile = 0 ; $refCntFile < 128 ; $refCntFile++ ) { for ( my $refCntFile = 0 ; $refCntFile < 128 ; $refCntFile++ ) {
print("__bpc_progress_fileCnt__ $refCntFile/128\n") if ( !$opts{p} && ($ refCntFile & 0x7) == 0 ); print("__bpc_progress_fileCnt__ $refCntFile/128\n") if ( !$opts{p} && ($ refCntFile & 0x7) == 0 );
for ( my $compress = 0 ; $compress < 2 ; $compress++ ) { for ( my $compress = 0 ; $compress < 2 ; $compress++ ) {
my $count = BackupPC::XS::PoolRefCnt::new(); my $count = BackupPC::XS::PoolRefCnt::new();
my $countDirty; my $countDirty;
my $poolCntFileNew = sprintf("%s/poolCntNew.%d.%02x", my $poolCntFileNew = sprintf("%s/poolCntNew.%d.%02x", $refCntDestDir
$refCntDestDir, $compress, $refCntFile , $compress, $refCntFile * 2);
* 2);
for ( my $i = 0 ; $i < @Backups ; $i++ ) { for ( my $i = 0 ; $i < @Backups ; $i++ ) {
my $bkupNum = $Backups[$i]{num}; my $bkupNum = $Backups[$i]{num};
my $refCntBkupDir = "$TopDir/pc/$host/$bkupNum/refCnt"; my $refCntBkupDir = "$TopDir/pc/$host/$bkupNum/refCnt";
next if ( $Backups[$i]{version} eq "" || ($Backups[$i]{version} =~ /^[23]\./ && !-d $refCntBkupDir) ); next if ( $Backups[$i]{version} eq "" || ($Backups[$i]{version} =~ /^[23]\./ && !-d $refCntBkupDir) );
my $poolCntHostFile = sprintf("%s/poolCnt.%d.%02x", $refCntBkupD ir, $compress, $refCntFile * 2); my $poolCntHostFile = sprintf("%s/poolCnt.%d.%02x", $refCntBkupD ir, $compress, $refCntFile * 2);
next if ( !-f $poolCntHostFile ); next if ( !-f $poolCntHostFile );
my $countHost = BackupPC::XS::PoolRefCnt::new(); my $countHost = BackupPC::XS::PoolRefCnt::new();
if ( $countHost->read($poolCntHostFile) ) { if ( $countHost->read($poolCntHostFile) ) {
print("Can't open pool count file $poolCntHostFile\n"); print("Can't open pool count file $poolCntHostFile\n");
$ErrorCnt++; $ErrorCnt++;
next; next;
} }
my($d, $c); my($d, $c);
my $idx = 0; my $idx = 0;
while ( 1 ) { while ( 1 ) {
($d, $c, $idx) = $countHost->iterate($idx); ($d, $c, $idx) = $countHost->iterate($idx);
last if ( !defined($d) ); last if ( !defined($d) );
if ( $c < 0 ) { if ( $c < 0 ) {
printf("BackupPC_refCountUpdate: host %s (%s) digest %s printf(
has negative count (%d); will do fsck on #%d\n", "BackupPC_refCountUpdate: host %s (%s) digest %s has
$host, $poolCntHostFile, unpack("H*", $d negative count (%d); will do fsck on #%d\n",
), $c, $bkupNum); $host, $poolCntHostFile, unpack("H*", $d),
$c, $bkupNum
);
$needFsck->{$bkupNum}++; $needFsck->{$bkupNum}++;
$ErrorCnt++; $ErrorCnt++;
} elsif ( $c > 0 ) {
$count->incr($d, $c);
$countDirty = 1;
} }
$count->incr($d, $c);
$countDirty = 1;
} }
} }
if ( $countDirty && $count->write($poolCntFileNew) ) { if ( $countDirty && $count->write($poolCntFileNew) ) {
print("Can't write new host pool count file $poolCntFileNew\n"); print("Can't write new host pool count file $poolCntFileNew\n");
$ErrorCnt++; $ErrorCnt++;
} }
$count = undef; $count = undef;
$bpc->flushXSLibMesgs(); $bpc->flushXSLibMesgs();
} }
} }
skipping to change at line 489 skipping to change at line 501
# Read all the poolCntDelta files, and update the host's poolCnt files # Read all the poolCntDelta files, and update the host's poolCnt files
# (or overwrite them if $forceFsck) # (or overwrite them if $forceFsck)
# #
my $entries = BackupPC::DirOps::dirRead($bpc, $refCntDir); my $entries = BackupPC::DirOps::dirRead($bpc, $refCntDir);
foreach my $e ( @$entries ) { foreach my $e ( @$entries ) {
unlink("$refCntDir/$e->{name}") if ( $e->{name} =~ /^poolCntNew/ ); unlink("$refCntDir/$e->{name}") if ( $e->{name} =~ /^poolCntNew/ );
} }
foreach my $e ( @$entries ) { foreach my $e ( @$entries ) {
next if ( $e->{name} !~ /^poolCntDelta/ ); next if ( $e->{name} !~ /^poolCntDelta/ );
my $deltaFileName = "$refCntDir/$e->{name}"; my $deltaFileName = "$refCntDir/$e->{name}";
my $compress = 1; my $compress = 1;
$compress = $1 if ( $deltaFileName =~ m{/poolCntDelta_(\d)_} ); $compress = $1 if ( $deltaFileName =~ m{/poolCntDelta_(\d)_} );
$gotDeltas++; $gotDeltas++;
updateHostPoolOneDelta2Cnt($host, $refCntDir, $deltaFileName, !$forceFsc k, $compress, $bkupNum); updateHostPoolOneDelta2Cnt($host, $refCntDir, $deltaFileName, !$forceFsc k, $compress, $bkupNum);
} }
renameNewCntFiles($host, $refCntDir, $compareCurr, "#$bkupNum") if ( $gotDel tas ); renameNewCntFiles($host, $refCntDir, $compareCurr, "#$bkupNum") if ( $gotDel tas );
} }
# #
# Rename all the new pool files, replacing the old ones # Rename all the new pool files, replacing the old ones
# #
sub renameNewCntFiles sub renameNewCntFiles
{ {
my($host, $refCntDir, $compareCurr, $statusText) = @_; my($host, $refCntDir, $compareCurr, $statusText) = @_;
print("__bpc_progress_state__ rename $statusText\n") if ( !$opts{p} ); print("__bpc_progress_state__ rename $statusText\n") if ( !$opts{p} );
for ( my $refCntFile = 0 ; $refCntFile < 128 ; $refCntFile++ ) { for ( my $refCntFile = 0 ; $refCntFile < 128 ; $refCntFile++ ) {
for ( my $compress = 0 ; $compress < 2 ; $compress++ ) { for ( my $compress = 0 ; $compress < 2 ; $compress++ ) {
my $poolCntFileNew = sprintf("%s/poolCntNew.%d.%02x", $refCntDir, my $poolCntFileNew = sprintf("%s/poolCntNew.%d.%02x", $refCntDir, $c
$compress, $refCntFile * 2); ompress, $refCntFile * 2);
my $poolCntFileCur = sprintf("%s/poolCnt.%d.%02x", $refCntDir, my $poolCntFileCur = sprintf("%s/poolCnt.%d.%02x", $refCntDir, $c
$compress, $refCntFile * 2); ompress, $refCntFile * 2);
if ( $compareCurr ) { if ( $compareCurr ) {
$ErrorCnt += poolCountHostNewCompare($host, $compress, $poolCntF ileCur, $poolCntFileNew); $ErrorCnt += poolCountHostNewCompare($host, $compress, $poolCntF ileCur, $poolCntFileNew);
} }
if ( -f $poolCntFileNew ) { if ( -f $poolCntFileNew ) {
if ( !rename($poolCntFileNew, $poolCntFileCur) ) { if ( !rename($poolCntFileNew, $poolCntFileCur) ) {
print("BackupPC_refCountUpdate: can't rename $poolCntFileNew to $poolCntFileCur ($!)\n"); print("BackupPC_refCountUpdate: can't rename $poolCntFileNew to $poolCntFileCur ($!)\n");
unlink($poolCntFileNew); unlink($poolCntFileNew);
$ErrorCnt++; $ErrorCnt++;
next; next;
} }
skipping to change at line 553 skipping to change at line 563
my($delta, $d, $c, $entryCnt); my($delta, $d, $c, $entryCnt);
my $idx = 0; my $idx = 0;
while ( 1 ) { while ( 1 ) {
($d, $c, $idx) = $count->iterate($idx); ($d, $c, $idx) = $count->iterate($idx);
last if ( !defined($d) ); last if ( !defined($d) );
$delta->[vec($d, 0, 8) >> 1]{$d} = $c; $delta->[vec($d, 0, 8) >> 1]{$d} = $c;
$entryCnt++; $entryCnt++;
} }
$count = undef; $count = undef;
print("BackupPC_refCountUpdate: processing host $host #$bkupNum deltaFile $d eltaFileName with $entryCnt entries\n") print("BackupPC_refCountUpdate: processing host $host #$bkupNum deltaFile $d eltaFileName with $entryCnt entries\n")
if ( $Conf{XferLogLevel} >= 2 ); if ( $Conf{XferLogLevel} >= 2 );
for ( my $refCntFile = 0 ; $refCntFile < 128 ; $refCntFile++ ) { for ( my $refCntFile = 0 ; $refCntFile < 128 ; $refCntFile++ ) {
my $poolCntFileNew = sprintf("%s/poolCntNew.%d.%02x", $refCntDir, my $poolCntFileNew = sprintf("%s/poolCntNew.%d.%02x", $refCntDir, $compr
$compress, $refCntFile * 2); ess, $refCntFile * 2);
my $poolCntFileCur = sprintf("%s/poolCnt.%d.%02x", $refCntDir, my $poolCntFileCur = sprintf("%s/poolCnt.%d.%02x", $refCntDir, $compr
$compress, $refCntFile * 2); ess, $refCntFile * 2);
if ( !defined($delta->[$refCntFile]) || !%{$delta->[$refCntFile]} ) { if ( !defined($delta->[$refCntFile]) || !%{$delta->[$refCntFile]} ) {
next if ( -f $poolCntFileNew || !-f $poolCntFileCur ); next if ( -f $poolCntFileNew || !-f $poolCntFileCur );
} }
# #
# Read the existing count # Read the existing count
# #
my $count = BackupPC::XS::PoolRefCnt::new(); my $count = BackupPC::XS::PoolRefCnt::new();
if ( -f $poolCntFileNew ) { if ( -f $poolCntFileNew ) {
if ( $count->read($poolCntFileNew) ) { if ( $count->read($poolCntFileNew) ) {
print("BackupPC_refCountUpdate: can't open new pool count file $ poolCntFileNew\n"); print("BackupPC_refCountUpdate: can't open new pool count file $ poolCntFileNew\n");
skipping to change at line 583 skipping to change at line 591
} else { } else {
# #
# Read in the existing counts if we are accumulating. # Read in the existing counts if we are accumulating.
# #
if ( $accumCurr && -f $poolCntFileCur && $count->read($poolCntFileCu r) ) { if ( $accumCurr && -f $poolCntFileCur && $count->read($poolCntFileCu r) ) {
print("BackupPC_refCountUpdate: can't open cur pool count file $ poolCntFileCur\n"); print("BackupPC_refCountUpdate: can't open cur pool count file $ poolCntFileCur\n");
$ErrorCnt++; $ErrorCnt++;
next; next;
} }
} }
# #
# Apply the deltas and write the file back; it's an error for any count to # Apply the deltas and write the file back; it's an error for any count to
# be negative. # be negative. Remove zero counts.
# #
foreach my $d ( keys(%{$delta->[$refCntFile]}) ) { foreach my $d ( keys(%{$delta->[$refCntFile]}) ) {
my $c = $count->incr($d, $delta->[$refCntFile]{$d}); my $c = $count->incr($d, $delta->[$refCntFile]{$d});
next if ( $c >= 0 ); $count->delete($d) if ( $c == 0 );
printf("BackupPC_refCountUpdate: got a negative count for %s process next if ( $c >= 0 );
ing %s; redoing #%d\n", printf(
unpack("H*", $d), $deltaFileName, $bkupNum); "BackupPC_refCountUpdate: got a negative count for %s processing
%s; redoing #%d\n",
unpack("H*", $d),
$deltaFileName, $bkupNum
);
$ErrorCnt++; $ErrorCnt++;
last; last;
} }
if ( $count->write($poolCntFileNew) ) { if ( $count->write($poolCntFileNew) ) {
print("BackupPC_refCountUpdate: can't write new pool count file $poo lCntFileNew\n"); print("BackupPC_refCountUpdate: can't write new pool count file $poo lCntFileNew\n");
$ErrorCnt++; $ErrorCnt++;
next; next;
} }
} }
$delta = {}; $delta = {};
skipping to change at line 618 skipping to change at line 630
# #
# Compare a new pool count file with the existing one. Returns an error count. # Compare a new pool count file with the existing one. Returns an error count.
# After the compare, renames the new pool count to replace the existing # After the compare, renames the new pool count to replace the existing
# one. # one.
# #
sub poolCountHostNewCompare sub poolCountHostNewCompare
{ {
my($host, $compress, $poolCntFileCur, $poolCntFileNew) = @_; my($host, $compress, $poolCntFileCur, $poolCntFileNew) = @_;
my $errorCnt = 0; my $errorCnt = 0;
my $countNew = BackupPC::XS::PoolRefCnt::new(); my $countNew = BackupPC::XS::PoolRefCnt::new();
my $count = BackupPC::XS::PoolRefCnt::new(); my $count = BackupPC::XS::PoolRefCnt::new();
if ( -f $poolCntFileNew && $countNew->read($poolCntFileNew) ) { if ( -f $poolCntFileNew && $countNew->read($poolCntFileNew) ) {
print("BackupPC_refCountUpdate: can't open new pool count file $poolCntF ileNew\n"); print("BackupPC_refCountUpdate: can't open new pool count file $poolCntF ileNew\n");
$errorCnt++; $errorCnt++;
} }
if ( -f $poolCntFileCur && $count->read($poolCntFileCur) ) { if ( -f $poolCntFileCur && $count->read($poolCntFileCur) ) {
print("BackupPC_refCountUpdate: can't open current pool count file $pool CntFileCur\n"); print("BackupPC_refCountUpdate: can't open current pool count file $pool CntFileCur\n");
$errorCnt++; $errorCnt++;
} }
skipping to change at line 642 skipping to change at line 654
# #
my($digest, $cnt); my($digest, $cnt);
my $idx = 0; my $idx = 0;
while ( 1 ) { while ( 1 ) {
($digest, $cnt, $idx) = $count->iterate($idx); ($digest, $cnt, $idx) = $count->iterate($idx);
last if ( !defined($digest) ); last if ( !defined($digest) );
if ( !defined($countNew->get($digest)) && $cnt > 0 ) { if ( !defined($countNew->get($digest)) && $cnt > 0 ) {
printf("BackupPC_refCountUpdate: host %s digest.%d %s count is %d, b ut should be 0\n", printf("BackupPC_refCountUpdate: host %s digest.%d %s count is %d, b ut should be 0\n",
$host, $compress, unpack("H*", $digest), $cnt); $host, $compress, unpack("H*", $digest), $cnt);
$errorCnt++; $errorCnt++;
next; next;
} }
if ( $cnt != $countNew->get($digest) ) { if ( $cnt != $countNew->get($digest) ) {
printf("BackupPC_refCountUpdate: host %s digest.%d %s count is %d, b printf(
ut should be %d\n", "BackupPC_refCountUpdate: host %s digest.%d %s count is %d, but
$host, $compress, unpack("H*", $digest), should be %d\n",
$cnt, $countNew->get($digest)); $host, $compress, unpack("H*", $digest),
$cnt, $countNew->get($digest)
);
$errorCnt++; $errorCnt++;
} }
$countNew->delete($digest); $countNew->delete($digest);
} }
$idx = 0; $idx = 0;
while ( 1 ) { while ( 1 ) {
($digest, $cnt, $idx) = $countNew->iterate($idx); ($digest, $cnt, $idx) = $countNew->iterate($idx);
last if ( !defined($digest) ); last if ( !defined($digest) );
printf("BackupPC_refCountUpdate: host %s digest.%d %s count missing, but printf(
should be %d\n", "BackupPC_refCountUpdate: host %s digest.%d %s count missing, but sh
$host, $compress, unpack("H*", $digest), $countNew->get( ould be %d\n",
$digest)); $host, $compress,
unpack("H*", $digest),
$countNew->get($digest)
);
$errorCnt++; $errorCnt++;
} }
return $errorCnt; return $errorCnt;
} }
sub updateAllHostPoolCnt sub updateAllHostPoolCnt
{ {
my($forceLast2Fsck, $forceFullFsck) = @_; my($forceLast2Fsck, $forceFullFsck) = @_;
skipping to change at line 692 skipping to change at line 710
return if ( $ErrorCnt ); return if ( $ErrorCnt );
print("__bpc_progress_state__ refCnt pool\n") if ( !$opts{p} ); print("__bpc_progress_state__ refCnt pool\n") if ( !$opts{p} );
for ( my $compress = 0 ; $compress < 2 ; $compress++ ) { for ( my $compress = 0 ; $compress < 2 ; $compress++ ) {
my $poolName = $compress ? "cpool4" : "pool4"; my $poolName = $compress ? "cpool4" : "pool4";
for ( my $refCntFile = $refCntStart ; $refCntFile <= $refCntEnd ; $refCn tFile++ ) { for ( my $refCntFile = $refCntStart ; $refCntFile <= $refCntEnd ; $refCn tFile++ ) {
# #
# Count the number of pool directories # Count the number of pool directories
# #
print("__bpc_progress_fileCnt__ cntUpdate $refCntFile/$refCntEnd\n") if ( !$opts{p} ); print("__bpc_progress_fileCnt__ cntUpdate $refCntFile/$refCntEnd\n") if ( !$opts{p} );
my $poolDir = sprintf("%s/%02x", my $poolDir = sprintf("%s/%02x", $compress ? $bpc->{CPoolDir} : $bpc
$compress ? $bpc->{CPoolDir} : $bpc->{PoolDir} ->{PoolDir}, $refCntFile * 2);
,
$refCntFile * 2);
next if ( !-d $poolDir ); next if ( !-d $poolDir );
$PoolStats->{$poolName}[$refCntFile]{dirCnt}++; $PoolStats->{$poolName}[$refCntFile]{dirCnt}++;
my $entries = BackupPC::DirOps::dirRead($bpc, $poolDir); my $entries = BackupPC::DirOps::dirRead($bpc, $poolDir);
foreach my $e ( @$entries ) { foreach my $e ( @$entries ) {
next if ( $e->{name} !~ /^[\da-f][\da-f]$/ ); next if ( $e->{name} !~ /^[\da-f][\da-f]$/ );
$PoolStats->{$poolName}[$refCntFile]{dirCnt}++; $PoolStats->{$poolName}[$refCntFile]{dirCnt}++;
} }
# #
# For each host update the per-host count # For each host update the per-host count
# #
my $poolCntFile = "$poolDir/poolCnt"; my $poolCntFile = "$poolDir/poolCnt";
my $count = BackupPC::XS::PoolRefCnt::new(); my $count = BackupPC::XS::PoolRefCnt::new();
my $countCopy = BackupPC::XS::PoolRefCnt::new(); my $countCopy = BackupPC::XS::PoolRefCnt::new();
my $countCurr = BackupPC::XS::PoolRefCnt::new(); my $countCurr = BackupPC::XS::PoolRefCnt::new();
$countCurr->read($poolCntFile) if ( -f $poolCntFile ); $countCurr->read($poolCntFile) if ( -f $poolCntFile );
foreach my $host ( @Hosts ) { foreach my $host ( @Hosts ) {
my $refCntDir = "$TopDir/pc/$host/refCnt"; my $refCntDir = "$TopDir/pc/$host/refCnt";
my $hostCntFile = sprintf("%s/poolCnt.%d.%02x", $refCntDir, my $hostCntFile = sprintf("%s/poolCnt.%d.%02x", $refCntDir, $com
$compress, $refCntFile * 2); press, $refCntFile * 2);
my $countHost = BackupPC::XS::PoolRefCnt::new(); my $countHost = BackupPC::XS::PoolRefCnt::new();
$countHost->read($hostCntFile) if ( -f $hostCntFile ); $countHost->read($hostCntFile) if ( -f $hostCntFile );
my($d, $c); my($d, $c);
my $idx = 0; my $idx = 0;
while ( 1 ) { while ( 1 ) {
($d, $c, $idx) = $countHost->iterate($idx); ($d, $c, $idx) = $countHost->iterate($idx);
last if ( !defined($d) ); last if ( !defined($d) );
if ( $c < 0 ) { if ( $c < 0 ) {
printf("BackupPC_refCountUpdate: host %s (%s) digest %s has negative count (%d)\n", printf("BackupPC_refCountUpdate: host %s (%s) digest %s has negative count (%d)\n",
$host, $hostCntFile, unpack("H*", $d), $ c); $host, $hostCntFile, unpack("H*", $d), $c);
$ErrorCnt++; $ErrorCnt++;
} }
if ( !defined($countCurr->get($d)) ) { if ( !defined($countCurr->get($d)) ) {
# #
# add stats for the new pool file # add stats for the new pool file
# #
my $poolFile = $bpc->MD52Path($d, $compress); my $poolFile = $bpc->MD52Path($d, $compress);
my @s = stat($poolFile); my @s = stat($poolFile);
if ( @s ) { if ( @s ) {
my $nBlks = $s[12]; my $nBlks = $s[12];
$PoolStats->{$poolName}[$refCntFile]{blkCnt} += $nBl ks; $PoolStats->{$poolName}[$refCntFile]{blkCnt} += $nBl ks;
if ( $c > 0 ) { if ( $c > 0 ) {
if ( ($s[2] & S_IXOTH) && chmod(0444, $poolFile) != 1 ) { if ( ($s[2] & S_IXOTH) && chmod(0444, $poolFile) != 1 ) {
print("BackupPC_refCountUpdate: can't chmod 0444 $poolFile\n"); print("BackupPC_refCountUpdate: can't chmod 0444 $poolFile\n");
$ErrorCnt++; $ErrorCnt++;
} }
} }
} }
} elsif ( $countCurr->get($d) == 0 && $c > 0 ) { } elsif ( $countCurr->get($d) == 0 && $c > 0 ) {
# #
# remove S_IXOTH flag since this digest is now reference d # remove S_IXOTH flag since this digest is now reference d
# #
my $poolFile = $bpc->MD52Path($d, $compress); my $poolFile = $bpc->MD52Path($d, $compress);
my @s = stat($poolFile); my @s = stat($poolFile);
if ( @s ) { if ( @s ) {
if ( ($s[2] & S_IXOTH) && chmod(0444, $poolFile) != 1 ) { if ( ($s[2] & S_IXOTH) && chmod(0444, $poolFile) != 1 ) {
print("BackupPC_refCountUpdate: can't chmod 0444 $poolFile\n"); print("BackupPC_refCountUpdate: can't chmod 0444 $poolFile\n");
$ErrorCnt++; $ErrorCnt++;
} }
} }
} }
$count->incr($d, $c); $count->incr($d, $c);
$countCopy->incr($d, $c); $countCopy->incr($d, $c);
$countCurr->incr($d, $c); # make sure we only count the new pool file once. $countCurr->incr($d, $c); # make sure we only count the n ew pool file once.
} }
} }
# #
# Add entries for any files in the existing pool count that aren't a lready in count/countCopy. # Add entries for any files in the existing pool count that aren't a lready in count/countCopy.
# #
if ( -f $poolCntFile ) { if ( -f $poolCntFile ) {
my($d, $c); my($d, $c);
my $idx = 0; my $idx = 0;
skipping to change at line 786 skipping to change at line 801
$count->incr($d, 0); $count->incr($d, 0);
$countCopy->incr($d, 0); $countCopy->incr($d, 0);
} }
} }
# #
# Scan the pool to add any missing files that have a zero count # Scan the pool to add any missing files that have a zero count
# #
for ( my $subDir = 0 ; $subDir < 128 ; $subDir++ ) { for ( my $subDir = 0 ; $subDir < 128 ; $subDir++ ) {
my $poolSubDir = sprintf("%s/%02x", $poolDir, $subDir * 2); my $poolSubDir = sprintf("%s/%02x", $poolDir, $subDir * 2);
my $entries = BackupPC::DirOps::dirRead($bpc, $poolSubDir); my $entries = BackupPC::DirOps::dirRead($bpc, $poolSubDir);
foreach my $e ( @$entries ) { foreach my $e ( @$entries ) {
next if ( $e->{name} eq "." || $e->{name} eq ".." ); next if ( $e->{name} eq "." || $e->{name} eq ".." );
if ( $e->{name} !~ /^[\da-f]{32,48}$/ && $e->{name} !~ /lock /i ) { if ( $e->{name} !~ /^[\da-f]{32,48}$/ && $e->{name} !~ /lock /i ) {
print("BackupPC_refCountUpdate: unknown pool file $poolS ubDir/$e->{name} removed\n"); print("BackupPC_refCountUpdate: unknown pool file $poolS ubDir/$e->{name} removed\n");
unlink("$poolSubDir/$e->{name}"); unlink("$poolSubDir/$e->{name}");
next; next;
} }
my $d = pack("H*", $e->{name}); my $d = pack("H*", $e->{name});
my $b2 = vec($d, 0, 16); my $b2 = vec($d, 0, 16);
if ( $refCntFile != (($b2 >> 8) & 0xfe) / 2 || $subDir != (( $b2 >> 0) & 0xfe) / 2 ) { if ( $refCntFile != (($b2 >> 8) & 0xfe) / 2 || $subDir != (( $b2 >> 0) & 0xfe) / 2 ) {
print("BackupPC_refCountUpdate: unexpected pool file $po olSubDir/$e->{name} removed\n"); print("BackupPC_refCountUpdate: unexpected pool file $po olSubDir/$e->{name} removed\n");
unlink("$poolSubDir/$e->{name}"); unlink("$poolSubDir/$e->{name}");
next; next;
} }
#
# check a small random subset of the pool file's actual md5
against its file name
#
if ( rand(100) < $Conf{PoolNightlyDigestCheckPercent} ) {
my $md5 = BackupPC::XS::FileDigest::digest("$poolSubDir/
$e->{name}", $compress);
if ( substr($d, 0, 16) ne $md5 ) {
printf(
"BackupPC_refCountUpdate: ERROR pool file %s has
digest %s instead of %s\n",
"$poolSubDir/$e->{name}",
unpack("H*", $md5),
unpack("H*", substr($d, 0, 16))
);
} else {
print("BackupPC_refCountUpdate: pool file $poolSubDi
r/$e->{name} md5 digest verified\n")
if ( $Conf{XferLogLevel} >= 5 );
}
}
if ( !defined($count->get($d)) ) { if ( !defined($count->get($d)) ) {
my @s = stat("$poolSubDir/$e->{name}"); my @s = stat("$poolSubDir/$e->{name}");
print("BackupPC_refCountUpdate: adding pool file $e->{na me} with count 0 (size $s[7])\n") print("BackupPC_refCountUpdate: adding pool file $e->{na me} with count 0 (size $s[7])\n")
if ( $Conf{XferLogLevel} >= 5 ); if ( $Conf{XferLogLevel} >= 5 );
# #
# add stats for new pool file # add stats for new pool file
# #
my $nBlks = $s[12]; my $nBlks = $s[12];
$PoolStats->{$poolName}[$refCntFile]{blkCnt} += $nBlks; $PoolStats->{$poolName}[$refCntFile]{blkCnt} += $nBlks;
$count->incr($d, 0); $count->incr($d, 0);
} else { } else {
$countCopy->delete($d); $countCopy->delete($d);
} }
} }
skipping to change at line 830 skipping to change at line 862
# #
# Normally we only update relative changes to the pool size (when a new pool # Normally we only update relative changes to the pool size (when a new pool
# file is added, or an old one is deleted), which is a lot more effi cient. # file is added, or an old one is deleted), which is a lot more effi cient.
# #
# So decide when to do a full pool size scan. $refCntFile goes from 0..127, # So decide when to do a full pool size scan. $refCntFile goes from 0..127,
# phase ($opts{P}) is 0..15 and $Conf{PoolSizeNightlyUpdatePeriod} i s # phase ($opts{P}) is 0..15 and $Conf{PoolSizeNightlyUpdatePeriod} i s
# 0, 1, 2, 4, 8, 16. # 0, 1, 2, 4, 8, 16.
# #
my $fullPoolScan; my $fullPoolScan;
if ( $Conf{PoolSizeNightlyUpdatePeriod} > 0 && defined($opts{P}) ) { if ( $Conf{PoolSizeNightlyUpdatePeriod} > 0 && defined($opts{P}) ) {
$fullPoolScan = (int($refCntFile / 8) % $Conf{PoolSizeNightlyUpd $fullPoolScan = (int($refCntFile / 8) % $Conf{PoolSizeNightlyUpd
atePeriod}) atePeriod}) ==
== ($opts{P} % $Conf{PoolSizeNightlyUpdatePer ($opts{P} % $Conf{PoolSizeNightlyUpdatePeriod});
iod});
} }
# print("BackupPC_refCountUpdate: computing full $poolName size for $
refCntFile (phase = $opts{P}," # print("BackupPC_refCountUpdate: computing full $poolNam
# . " \$Conf{PoolSizeNightlyUpdatePeriod} = $Conf{PoolSizeNightly e size for $refCntFile (phase = $opts{P},"
UpdatePeriod})\n") # . " \$Conf{PoolSizeNightlyUpdatePeriod} = $Conf{Poo
# if ( $fullPoolScan ); lSizeNightlyUpdatePeriod})\n")
# if ( $fullPoolScan );
my $blkCnt = 0; # size of pool files
my $fileCntRep = 0; # total number of pool files with repeated md my $blkCnt = 0; # size of pool files
5 checksums my $fileCntRep = 0; # total number of pool files with repeated
# (ie: digest > 16 bytes; first instance isn' md5 checksums
t counted) # (ie: digest > 16 bytes; first instance i
my $fileRepMax = 0; # worse case chain length of pool files that sn't counted)
have repeated my $fileRepMax = 0; # worse case chain length of pool files th
# checksums (ie: max(NNN) for all digests xxx at have repeated
xxxxxxxxxxxxxNNN) # checksums (ie: max(NNN) for all digests
my $fileLinkMax = 0; # maximum number of links on a pool file xxxxxxxxxxxxxxxxNNN)
my $fileLinkTotal = 0; # total number of links on entire pool my $fileLinkMax = 0; # maximum number of links on a pool file
my $fileLinkTotal = 0; # total number of links on entire pool
my($digest, $cnt); my($digest, $cnt);
my $idx = 0; my $idx = 0;
my $poolFileCnt = 0; my $poolFileCnt = 0;
while ( 1 ) { while ( 1 ) {
($digest, $cnt, $idx) = $count->iterate($idx); ($digest, $cnt, $idx) = $count->iterate($idx);
last if ( !defined($digest) ); last if ( !defined($digest) );
$poolFileCnt++; $poolFileCnt++;
$fileLinkTotal += $cnt; $fileLinkTotal += $cnt;
$fileLinkMax = $cnt if ( $fileLinkMax < $cnt && $digest ne $E mptyMD5 ); $fileLinkMax = $cnt if ( $fileLinkMax < $cnt && $digest ne $Empt yMD5 );
if ( $fullPoolScan ) { if ( $fullPoolScan ) {
my $poolFile = $bpc->MD52Path($digest, $compress); my $poolFile = $bpc->MD52Path($digest, $compress);
my @s = stat($poolFile); my @s = stat($poolFile);
$blkCnt += $s[12] if ( @s ); $blkCnt += $s[12] if ( @s );
} }
next if ( length($digest) <= 16 ); next if ( length($digest) <= 16 );
my $ext = $bpc->digestExtGet($digest); my $ext = $bpc->digestExtGet($digest);
$fileCntRep++; $fileCntRep++;
$fileRepMax = $ext if ( $fileRepMax < $ext ); $fileRepMax = $ext if ( $fileRepMax < $ext );
} }
$PoolStats->{$poolName}[$refCntFile]{blkCnt} = $blkCnt if ( $ fullPoolScan ); $PoolStats->{$poolName}[$refCntFile]{blkCnt} = $blkCnt if ( $ fullPoolScan );
$PoolStats->{$poolName}[$refCntFile]{fileCnt} = $poolFileCnt; $PoolStats->{$poolName}[$refCntFile]{fileCnt} = $poolFileCnt;
$PoolStats->{$poolName}[$refCntFile]{fileLinkMax} = $fileLinkMax; $PoolStats->{$poolName}[$refCntFile]{fileLinkMax} = $fileLinkMax;
skipping to change at line 914 skipping to change at line 947
sub cleanPoolFiles sub cleanPoolFiles
{ {
for ( my $compress = 0 ; $compress < 2 ; $compress++ ) { for ( my $compress = 0 ; $compress < 2 ; $compress++ ) {
my $poolName = $compress ? "cpool4" : "pool4"; my $poolName = $compress ? "cpool4" : "pool4";
for ( my $refCntFile = $refCntStart ; $refCntFile <= $refCntEnd ; $refCn tFile++ ) { for ( my $refCntFile = $refCntStart ; $refCntFile <= $refCntEnd ; $refCn tFile++ ) {
# #
# Read the existing count # Read the existing count
# #
my $dirty = 0; my $dirty = 0;
my $poolDir = sprintf("%s/%02x", my $poolDir = sprintf("%s/%02x", $compress ? $bpc->{CPoolDir} :
$compress ? $bpc->{CPoolDir} : $bpc->{Pool $bpc->{PoolDir}, $refCntFile * 2);
Dir},
$refCntFile * 2);
my $poolCntFile = "$poolDir/poolCnt"; my $poolCntFile = "$poolDir/poolCnt";
my $count = BackupPC::XS::PoolRefCnt::new(); my $count = BackupPC::XS::PoolRefCnt::new();
# #
# Grab a lock to make sure BackupPC_dump won't unmark and use a pend ing # Grab a lock to make sure BackupPC_dump won't unmark and use a pend ing
# delete file. # delete file.
# #
my $lockFd = BackupPC::XS::DirOps::lockRangeFile("$poolDir/LOCK", 0, 1, 1); my $lockFd = BackupPC::XS::DirOps::lockRangeFile("$poolDir/LOCK", 0, 1, 1);
if ( -f $poolCntFile && $count->read($poolCntFile) ) { if ( -f $poolCntFile && $count->read($poolCntFile) ) {
print("BackupPC_refCountUpdate: can't read pool count file $pool CntFile\n"); print("BackupPC_refCountUpdate: can't read pool count file $pool CntFile\n");
$dirty = 1; $dirty = 1;
$ErrorCnt++; $ErrorCnt++;
} }
my($digest, $cnt); my($digest, $cnt);
my $idx = 0; my $idx = 0;
my $minMtime = time() - 7 * 24 * 3600; my $minMtime = time() - 7 * 24 * 3600;
while ( 1 ) { while ( 1 ) {
($digest, $cnt, $idx) = $count->iterate($idx); ($digest, $cnt, $idx) = $count->iterate($idx);
last if ( !defined($digest) ); last if ( !defined($digest) );
next if ( $cnt > 0 ); next if ( $cnt > 0 );
my $poolFile = $bpc->MD52Path($digest, $compress); my $poolFile = $bpc->MD52Path($digest, $compress);
my @s = stat($poolFile); my @s = stat($poolFile);
next if ( !@s || $s[7] == 0 || $s[9] >= $minMtime ); next if ( !@s || $s[7] == 0 || $s[9] >= $minMtime );
my $mode = $s[2]; my $mode = $s[2];
my $nBlks = $s[12]; my $nBlks = $s[12];
if ( $mode & S_IXOTH ) { if ( $mode & S_IXOTH ) {
# #
# figure out the next file in the sequence # figure out the next file in the sequence
# #
my $ext = $bpc->digestExtGet($digest); my $ext = $bpc->digestExtGet($digest);
my($nextDigest, $nextPoolFile) = $bpc->digestConcat($digest, my($nextDigest, $nextPoolFile) = $bpc->digestConcat($digest,
$ext + 1 $ext + 1, $compress);
, $compress);
if ( !-f $nextPoolFile ) { if ( !-f $nextPoolFile ) {
# #
# last in the chain (or no chain) - just delete it # last in the chain (or no chain) - just delete it
# #
print("BackupPC_refCountUpdate: removing pool file $pool print("BackupPC_refCountUpdate: removing pool file $pool
File\n") if ( $Conf{XferLogLevel} >= 4 ); File\n")
if ( $Conf{XferLogLevel} >= 4 );
if ( unlink($poolFile) != 1 ) { if ( unlink($poolFile) != 1 ) {
print("BackupPC_refCountUpdate: can't remove $poolFi le\n"); print("BackupPC_refCountUpdate: can't remove $poolFi le\n");
$ErrorCnt++; $ErrorCnt++;
next; next;
} }
} else { } else {
# #
# in a chain of pool files we can't delete so # in a chain of pool files we can't delete so
# we replace the file with an empty file. # we replace the file with an empty file.
# first remove S_IXOTH mode # first remove S_IXOTH mode
# #
print("BackupPC_refCountUpdate: zeroing pool file $poolF print("BackupPC_refCountUpdate: zeroing pool file $poolF
ile (next $nextPoolFile exists)\n") if ( $Conf{XferLogLevel} >= 4 ); ile (next $nextPoolFile exists)\n")
if ( $Conf{XferLogLevel} >= 4 );
if ( chmod(0644, $poolFile) != 1 ) { if ( chmod(0644, $poolFile) != 1 ) {
print("BackupPC_refCountUpdate: can't chmod 0644 $po olFile\n"); print("BackupPC_refCountUpdate: can't chmod 0644 $po olFile\n");
$ErrorCnt++; $ErrorCnt++;
} }
if ( open(my $fh, ">", $poolFile) ) { if ( open(my $fh, ">", $poolFile) ) {
close($fh); close($fh);
} else { } else {
print("BackupPC_refCountUpdate: can't truncate $pool File\n"); print("BackupPC_refCountUpdate: can't truncate $pool File\n");
$ErrorCnt++; $ErrorCnt++;
next; next;
} }
} }
$count->delete($digest); $count->delete($digest);
$dirty = 1; $dirty = 1;
# #
# update stats # update stats
# #
$PoolStats->{$poolName}[$refCntFile]{fileCnt}--; $PoolStats->{$poolName}[$refCntFile]{fileCnt}--;
$PoolStats->{$poolName}[$refCntFile]{blkCnt} -= $nBlks; $PoolStats->{$poolName}[$refCntFile]{blkCnt} -= $nBlks;
$PoolStats->{$poolName}[$refCntFile]{fileCntRm}++; $PoolStats->{$poolName}[$refCntFile]{fileCntRm}++;
$PoolStats->{$poolName}[$refCntFile]{blkCntRm} += $nBlks; $PoolStats->{$poolName}[$refCntFile]{blkCntRm} += $nBlks;
} else { } else {
# #
# mark the pool file so no one links to it # mark the pool file so no one links to it
# #
print("BackupPC_refCountUpdate: marking pool file $poolFile\ n") if ( $Conf{XferLogLevel} >= 4 ); print("BackupPC_refCountUpdate: marking pool file $poolFile\ n") if ( $Conf{XferLogLevel} >= 4 );
if ( chmod(0445, $poolFile) != 1 ) { if ( chmod(0445, $poolFile) != 1 ) {
print("BackupPC_refCountUpdate: can't chmod 0445 $poolFi le\n"); print("BackupPC_refCountUpdate: can't chmod 0445 $poolFi le\n");
$ErrorCnt++; $ErrorCnt++;
skipping to change at line 1023 skipping to change at line 1056
} }
} }
} }
} }
sub statsPrintSingle sub statsPrintSingle
{ {
my($poolName, $refCntFile) = @_; my($poolName, $refCntFile) = @_;
my $s = $PoolStats->{$poolName}[$refCntFile]; my $s = $PoolStats->{$poolName}[$refCntFile];
my $kb = $s->{blkCnt} >= 0 ? int($s->{blkCnt} / 2 + 0.5) : int($s->{bl kCnt} / 2 - 0.5); my $kb = $s->{blkCnt} >= 0 ? int($s->{blkCnt} / 2 + 0.5) : int($s->{blkCnt } / 2 - 0.5);
my $kbRm = $s->{blkCntRm} >= 0 ? int($s->{blkCntRm} / 2 + 0.5) : int($s->{bl kCntRm} / 2 - 0.5); my $kbRm = $s->{blkCntRm} >= 0 ? int($s->{blkCntRm} / 2 + 0.5) : int($s->{bl kCntRm} / 2 - 0.5);
printf("BackupPC_stats4 %d = %s,%d,%d,%d,%d,%d,%d,%d,%d,%d\n", printf(
$refCntFile, $poolName, "BackupPC_stats4 %d = %s,%d,%d,%d,%d,%d,%d,%d,%d,%d\n",
$s->{fileCnt}, $s->{dirCnt}, $kb, $kbRm, $refCntFile, $poolName, $s->{fileCnt}, $s->{dirCnt},
$s->{fileCntRm}, $s->{fileCntRep}, $s->{fileRepMax}, $kb, $kbRm, $s->{fileCntRm}, $s->{fileCntRep},
$s->{fileLinkMax}, $s->{fileLinkTotal}); $s->{fileRepMax}, $s->{fileLinkMax}, $s->{fileLinkTotal}
);
} }
sub statsPrint sub statsPrint
{ {
foreach my $poolName ( qw(pool4 cpool4) ) { foreach my $poolName ( qw(pool4 cpool4) ) {
for ( my $refCntFile = $refCntStart ; $refCntFile <= $refCntEnd ; $refCn tFile++ ) { for ( my $refCntFile = $refCntStart ; $refCntFile <= $refCntEnd ; $refCn tFile++ ) {
statsPrintSingle($poolName, $refCntFile); statsPrintSingle($poolName, $refCntFile);
} }
} }
} }
 End of changes. 61 change blocks. 
124 lines changed or deleted 163 lines changed or added
To the Perl Programs (all 27): section of the BackupPC source changes report or the top of this page
Mainly generated by pkgdiff using rfcdiff
Home  |  About  |  Features  |  All  |  Newest  |  Dox  |  Diffs  |  RSS Feeds  |  Screenshots  |  Comments  |  Imprint  |  Privacy  |  HTTP(S)