Check file sizes while scanning directory in scan_incoming

Excessively large directive and signature files are now detected while
scanning the incoming files directory and promptly removed afterward;
the testsuite is adjusted accordingly.

diff --git a/gatekeeper.pl b/gatekeeper.pl
index 7ef1d535bfd934e70fa491550e78717ae08953db..4706e3ca6440a027ba4531d59e679074b41d96cb 100755 (executable)
--- a/gatekeeper.pl
+++ b/gatekeeper.pl
@@ -880,11 +880,17 @@ sub scan_incoming {
     my $ent = $1;      # if we get here, $RE_filename_here matched above
     # $_ remains tainted, but $ent is an untainted (and safe) copy
 
+    # Examine the file; this populates an internal cache in perl.
+    unless (stat($ent)) {
+      ftp_syslog('warning', "could not stat($ent), skipping");
+      next ENT
+    }
+
     # Do not consider files that have been modified in the last 2 minutes.
     # This is an extra safety check to avoid trying to process files that
-    # are still being uploaded.
+    # are still being uploaded.  (use stat cache)
     if (TSTAMPCHECK) {
-      if ((stat($ent))[9] >= $time_bar) {
+      if ((stat(_))[9] >= $time_bar) {
        ftp_syslog('debug', "DEBUG: "
                   ."$ent has been modified in the last 2 minutes, skipping")
          if DEBUG;
@@ -892,6 +898,17 @@ sub scan_incoming {
       }
     }
 
+    # check for overlength directives and signatures (use stat cache)
+    if (/[.]directive[.]asc$/ && -f _ && ((-s _) >= MAX_DIRECTIVE_SIZE)) {
+      ftp_syslog('info', "Overlength directive file ($ent) trashcanned");
+      push @trash, File::Spec->catfile($directory, $ent);
+      next ENT
+    } elsif (/[.]sig$/ && -f _ && ((-s _) >= MAX_SIGNATURE_SIZE)) {
+      ftp_syslog('info', "Overlength signature file ($ent) trashcanned");
+      push @trash, File::Spec->catfile($directory, $ent);
+      next ENT
+    }
+
     ftp_syslog('debug', "DEBUG: uploaded file to check: $ent") if DEBUG;
     $possible{$ent} = 1;
   }
@@ -956,14 +973,10 @@ sub scan_incoming {
     ftp_syslog('debug', "DEBUG: "
               ."considering $ent for processing") if DEBUG;
 
-    # work on this triple, if all three files exist, and the signature
-    # and directive files aren't huge.  We want to exclude huge files
-    # here, before even reading the directive file; otherwise, perl could
-    # consume lots of memory reading it.
+    # Work on this triple, if all three files are accepted.  Overlength
+    # directive and signature files were discarded in the initial scan.
     if (exists($possible{$base}) && exists($possible{$sig})
-       && exists($possible{$directive})
-       && (-s "$directory/$directive" < 50*1024)
-       && (-s "$directory/$sig" < 50*1024)) {
+       && exists($possible{$directive})) {
       push (@ret, { "directive" => $directive, "sig" => $sig,
                    "upload" => $base, "directive_only" => 0 });
       ftp_syslog('info', "processing [$directive:$sig:$base]");
@@ -1019,16 +1032,6 @@ sub scan_incoming {
                   ."to $scratchpad/$base failed: $!",0);
       }
       delete $possible{$base};
-    } elsif ((-f $directive) && ((-s $directive) >= MAX_DIRECTIVE_SIZE)) {
-      rename ("$directory/$directive", "$directory/.$directive");
-      ftp_syslog('info', "directive file ($directive) larger than 50KB");
-      fatal("The directive file $directive is larger than 50KB. "
-           ."This can not be correct, ignoring upload.",0);
-    } elsif ((-f $sig) && ((-s $sig) >= MAX_SIGNATURE_SIZE)) {
-      rename ("$directory/$sig", "$directory/.$sig");
-      ftp_syslog('info', "signature file ($sig) larger than 50KB");
-      fatal("The signature file $sig is larger than 50KB. "
-           ."This can not be correct, ignoring upload.",0);
     }
   }
 

diff --git a/testsuite/gatekeeper.all/00_idle.exp b/testsuite/gatekeeper.all/00_idle.exp
index 8f23bf615e60f9fc19f32d1540590aaa6a4f3a91..355af3aa7aec29fab29b3ff0683e6ad791f54487 100644 (file)
--- a/testsuite/gatekeeper.all/00_idle.exp
+++ b/testsuite/gatekeeper.all/00_idle.exp
@@ -352,11 +352,10 @@ proc check_oversize_part { oversize_directive oversize_signature } {
 
     set msglist {
        start "ftp-upload start message"
+       nowork "ftp-upload 'nothing to do' message"
 
        scan,oversize.bin "scan found main file"
        consider,oversize.bin "considered main file"
-       scan,oversize.bin.sig "scan found signature file"
-       scan,oversize.bin.directive.asc "scan found directive file"
     }
 
     set filelist { oversize.bin }
@@ -380,9 +379,10 @@ proc check_oversize_part { oversize_directive oversize_signature } {
            "3 minutes ago"
        lappend msglist oversize-directive,oversize.bin.directive.asc \
            "detected oversize directive file"
-       lappend filelist .oversize.bin.directive.asc
     } else {
        lappend filelist oversize.bin.directive.asc
+       lappend msglist scan,oversize.bin.directive.asc \
+           "scan found directive file"
     }
     if { $oversize_signature } {
        set chan [open [file join $tenv incoming oversize.bin.sig] a]
@@ -391,17 +391,11 @@ proc check_oversize_part { oversize_directive oversize_signature } {
        close $chan
        age_file [file join $tenv incoming oversize.bin.sig] \
            "3 minutes ago"
-       if { ! $oversize_directive } {
-           # a signature will not be checked if the directive is oversize
-           # TODO: fix this so both will be reported if appropriate
-           lappend msglist oversize-signature,oversize.bin.sig \
-               "detected oversize signature file"
-           lappend filelist .oversize.bin.sig
-       } else {
-           lappend filelist oversize.bin.sig
-       }
+       lappend msglist oversize-signature,oversize.bin.sig \
+           "detected oversize signature file"
     } else {
        lappend filelist oversize.bin.sig
+       lappend msglist scan,oversize.bin.sig "scan found signature file"
     }
 
     start_test_services $tenv
@@ -418,7 +412,7 @@ proc check_oversize_part { oversize_directive oversize_signature } {
        in-stage stage pub archive
     } empty
     analyze_log $tenv $Name $msglist
-    analyze_mail $tenv $Name to { ftp-upload-script@gnu.org }
+    analyze_no_mail $tenv $Name
 
     close_test_environment $tenv
 }

diff --git a/testsuite/lib/gatekeeper.exp b/testsuite/lib/gatekeeper.exp
index 9c4daaec06f4d0914318451d721ed6e4dacab206..4e9a9bea81e108df299c5f9c185353953ace5aeb 100644 (file)
--- a/testsuite/lib/gatekeeper.exp
+++ b/testsuite/lib/gatekeeper.exp
@@ -588,29 +588,17 @@ proc analyze_log { base_dir name assess } {
                     exp_continue
                 }
        -re {^gatekeeper\[[0-9]+\]: \(Test\)\
-                directive file \(([^ ]+)\) larger than[^\r\n]+} {
+                Overlength directive file \(([^ ]+)\) [^\r\n]+} {
                     # from scan_incoming, on finding an oversize directive
                     set A(oversize-directive,$expect_out(1,string)) 1
                     exp_continue
                 }
        -re {^gatekeeper\[[0-9]+\]: \(Test\)\
-                The directive file [^ ]+ is larger than[^\r\n]+} {
-                    # from scan_incoming, on finding an oversize directive
-                    # already noted above; both messages are produced
-                    exp_continue
-                }
-       -re {^gatekeeper\[[0-9]+\]: \(Test\)\
-                signature file \(([^ ]+)\) larger than[^\r\n]+} {
+                Overlength signature file \(([^ ]+)\) [^\r\n]+} {
                     # from scan_incoming, on finding an oversize signature
                     set A(oversize-signature,$expect_out(1,string)) 1
                     exp_continue
                 }
-       -re {^gatekeeper\[[0-9]+\]: \(Test\)\
-                The signature file [^ ]+ is larger than[^\r\n]+} {
-                    # from scan_incoming, on finding an oversize signature
-                    # already noted above; both messages are produced
-                    exp_continue
-                }
 
        -re {^gatekeeper\[[0-9]+\]: \(Test\)\
                 DEBUG: found keyring [^\r\n]+} {