From bf1e6fbf9550004dd1584ff3f76f2ed1929372ab Mon Sep 17 00:00:00 2001 From: Jacob Bachmeyer Date: Fri, 14 Oct 2022 22:19:29 -0500 Subject: [PATCH] Check file sizes while scanning directory in scan_incoming Excessively large directive and signature files are now detected while scanning the incoming files directory and promptly removed afterward; the testsuite is adjusted accordingly. --- gatekeeper.pl | 41 +++++++++++++++------------- testsuite/gatekeeper.all/00_idle.exp | 20 +++++--------- testsuite/lib/gatekeeper.exp | 16 ++--------- 3 files changed, 31 insertions(+), 46 deletions(-) diff --git a/gatekeeper.pl b/gatekeeper.pl index 7ef1d53..4706e3c 100755 --- a/gatekeeper.pl +++ b/gatekeeper.pl @@ -880,11 +880,17 @@ sub scan_incoming { my $ent = $1; # if we get here, $RE_filename_here matched above # $_ remains tainted, but $ent is an untainted (and safe) copy + # Examine the file; this populates an internal cache in perl. + unless (stat($ent)) { + ftp_syslog('warning', "could not stat($ent), skipping"); + next ENT + } + # Do not consider files that have been modified in the last 2 minutes. # This is an extra safety check to avoid trying to process files that - # are still being uploaded. + # are still being uploaded. (use stat cache) if (TSTAMPCHECK) { - if ((stat($ent))[9] >= $time_bar) { + if ((stat(_))[9] >= $time_bar) { ftp_syslog('debug', "DEBUG: " ."$ent has been modified in the last 2 minutes, skipping") if DEBUG; @@ -892,6 +898,17 @@ sub scan_incoming { } } + # check for overlength directives and signatures (use stat cache) + if (/[.]directive[.]asc$/ && -f _ && ((-s _) >= MAX_DIRECTIVE_SIZE)) { + ftp_syslog('info', "Overlength directive file ($ent) trashcanned"); + push @trash, File::Spec->catfile($directory, $ent); + next ENT + } elsif (/[.]sig$/ && -f _ && ((-s _) >= MAX_SIGNATURE_SIZE)) { + ftp_syslog('info', "Overlength signature file ($ent) trashcanned"); + push @trash, File::Spec->catfile($directory, $ent); + next ENT + } + ftp_syslog('debug', "DEBUG: uploaded file to check: $ent") if DEBUG; $possible{$ent} = 1; } @@ -956,14 +973,10 @@ sub scan_incoming { ftp_syslog('debug', "DEBUG: " ."considering $ent for processing") if DEBUG; - # work on this triple, if all three files exist, and the signature - # and directive files aren't huge. We want to exclude huge files - # here, before even reading the directive file; otherwise, perl could - # consume lots of memory reading it. + # Work on this triple, if all three files are accepted. Overlength + # directive and signature files were discarded in the initial scan. if (exists($possible{$base}) && exists($possible{$sig}) - && exists($possible{$directive}) - && (-s "$directory/$directive" < 50*1024) - && (-s "$directory/$sig" < 50*1024)) { + && exists($possible{$directive})) { push (@ret, { "directive" => $directive, "sig" => $sig, "upload" => $base, "directive_only" => 0 }); ftp_syslog('info', "processing [$directive:$sig:$base]"); @@ -1019,16 +1032,6 @@ sub scan_incoming { ."to $scratchpad/$base failed: $!",0); } delete $possible{$base}; - } elsif ((-f $directive) && ((-s $directive) >= MAX_DIRECTIVE_SIZE)) { - rename ("$directory/$directive", "$directory/.$directive"); - ftp_syslog('info', "directive file ($directive) larger than 50KB"); - fatal("The directive file $directive is larger than 50KB. " - ."This can not be correct, ignoring upload.",0); - } elsif ((-f $sig) && ((-s $sig) >= MAX_SIGNATURE_SIZE)) { - rename ("$directory/$sig", "$directory/.$sig"); - ftp_syslog('info', "signature file ($sig) larger than 50KB"); - fatal("The signature file $sig is larger than 50KB. " - ."This can not be correct, ignoring upload.",0); } } diff --git a/testsuite/gatekeeper.all/00_idle.exp b/testsuite/gatekeeper.all/00_idle.exp index 8f23bf6..355af3a 100644 --- a/testsuite/gatekeeper.all/00_idle.exp +++ b/testsuite/gatekeeper.all/00_idle.exp @@ -352,11 +352,10 @@ proc check_oversize_part { oversize_directive oversize_signature } { set msglist { start "ftp-upload start message" + nowork "ftp-upload 'nothing to do' message" scan,oversize.bin "scan found main file" consider,oversize.bin "considered main file" - scan,oversize.bin.sig "scan found signature file" - scan,oversize.bin.directive.asc "scan found directive file" } set filelist { oversize.bin } @@ -380,9 +379,10 @@ proc check_oversize_part { oversize_directive oversize_signature } { "3 minutes ago" lappend msglist oversize-directive,oversize.bin.directive.asc \ "detected oversize directive file" - lappend filelist .oversize.bin.directive.asc } else { lappend filelist oversize.bin.directive.asc + lappend msglist scan,oversize.bin.directive.asc \ + "scan found directive file" } if { $oversize_signature } { set chan [open [file join $tenv incoming oversize.bin.sig] a] @@ -391,17 +391,11 @@ proc check_oversize_part { oversize_directive oversize_signature } { close $chan age_file [file join $tenv incoming oversize.bin.sig] \ "3 minutes ago" - if { ! $oversize_directive } { - # a signature will not be checked if the directive is oversize - # TODO: fix this so both will be reported if appropriate - lappend msglist oversize-signature,oversize.bin.sig \ - "detected oversize signature file" - lappend filelist .oversize.bin.sig - } else { - lappend filelist oversize.bin.sig - } + lappend msglist oversize-signature,oversize.bin.sig \ + "detected oversize signature file" } else { lappend filelist oversize.bin.sig + lappend msglist scan,oversize.bin.sig "scan found signature file" } start_test_services $tenv @@ -418,7 +412,7 @@ proc check_oversize_part { oversize_directive oversize_signature } { in-stage stage pub archive } empty analyze_log $tenv $Name $msglist - analyze_mail $tenv $Name to { ftp-upload-script@gnu.org } + analyze_no_mail $tenv $Name close_test_environment $tenv } diff --git a/testsuite/lib/gatekeeper.exp b/testsuite/lib/gatekeeper.exp index 9c4daae..4e9a9be 100644 --- a/testsuite/lib/gatekeeper.exp +++ b/testsuite/lib/gatekeeper.exp @@ -588,29 +588,17 @@ proc analyze_log { base_dir name assess } { exp_continue } -re {^gatekeeper\[[0-9]+\]: \(Test\)\ - directive file \(([^ ]+)\) larger than[^\r\n]+} { + Overlength directive file \(([^ ]+)\) [^\r\n]+} { # from scan_incoming, on finding an oversize directive set A(oversize-directive,$expect_out(1,string)) 1 exp_continue } -re {^gatekeeper\[[0-9]+\]: \(Test\)\ - The directive file [^ ]+ is larger than[^\r\n]+} { - # from scan_incoming, on finding an oversize directive - # already noted above; both messages are produced - exp_continue - } - -re {^gatekeeper\[[0-9]+\]: \(Test\)\ - signature file \(([^ ]+)\) larger than[^\r\n]+} { + Overlength signature file \(([^ ]+)\) [^\r\n]+} { # from scan_incoming, on finding an oversize signature set A(oversize-signature,$expect_out(1,string)) 1 exp_continue } - -re {^gatekeeper\[[0-9]+\]: \(Test\)\ - The signature file [^ ]+ is larger than[^\r\n]+} { - # from scan_incoming, on finding an oversize signature - # already noted above; both messages are produced - exp_continue - } -re {^gatekeeper\[[0-9]+\]: \(Test\)\ DEBUG: found keyring [^\r\n]+} { -- 2.25.1