release-process/scripts/mk_exim_release

   1 #!/usr/bin/env perl
   2 # Copyright (c) The Exim Maintainers 2016
   3
   4 use strict;
   5 use warnings;
   6 use Carp;
   7 use File::Copy;
   8 use File::Spec;
   9 use File::Path;
  10 use File::Temp;
  11 use FindBin;
  12 use Getopt::Long;
  13 use IO::File;
  14 use Pod::Usage;
  15
  16 my $debug   = 0;
  17 my $verbose = 0;
  18
  19 # ------------------------------------------------------------------
  20
  21 sub get_and_check_version {
  22     my $release = shift;
  23     my $context = shift;
  24
  25     # make sure this looks like a real release version
  26     # which should (currently) be 4.xx[.y] or 4.xx[.y]_RCx
  27     unless ( $release =~ /^(?<release>(?<major>4\.\d\d)(?:\.(?<minor>\d+))?(?<rc>_RC\d+)?)$/ ) {
  28         croak "The given version number does not look right - $release";
  29     }
  30     $context->{release}  = $+{release};
  31     $context->{major} = $+{major};
  32     $context->{minor} = $+{minor};
  33     $context->{candidatev} = $+{rc};
  34
  35     ($context->{trelease} = $+{release}) =~ s/_RC\d+//;
  36 }
  37
  38 # ------------------------------------------------------------------
  39
  40 sub build_tag {
  41     my $context = shift;
  42
  43     # The CVS tag consists of exim-$version where $version
  44     # is the version number with . replaced with _
  45     my $modversion = $context->{release};
  46     $modversion =~ tr/0-9RC/_/cs;
  47
  48     return sprintf( 'exim-%s', $modversion );
  49 }
  50
  51 # ------------------------------------------------------------------
  52
  53 sub deal_with_working_directory {
  54     my $context = shift;
  55     my $delete  = shift;
  56
  57     # Set default directory
  58     $context->{directory} ||= File::Spec->rel2abs( sprintf( 'exim-packaging-%s', $context->{release} ) );
  59     my $directory = $context->{directory};
  60
  61     # ensure the working directory is not in place
  62     if ( -d $directory ) {
  63         if ($delete) {
  64             print "Deleting existing $directory\n" if ($verbose);
  65             rmtree( $directory, { verbose => $debug } );
  66         }
  67         if ( -d $directory ) {
  68             croak "Working directory $directory exists";
  69         }
  70     }
  71
  72     # create base directory
  73     mkpath( $context->{directory}, { verbose => ( $verbose || $debug ) } );
  74
  75     # set and create subdirectories
  76     foreach (qw(release_tree pkgs pkgdirs docbook tmp)) {
  77         $context->{$_} = File::Spec->catdir( $context->{directory}, $_ );
  78         mkpath( $context->{$_}, { verbose => ( $verbose || $debug ) } );
  79     }
  80 }
  81
  82 # ------------------------------------------------------------------
  83
  84 sub export_git_tree {
  85     my $context = shift;
  86
  87     # build git command
  88     my $archive_file = sprintf( '%s/%s-%s.tar', $context->{tmp}, $context->{pkgname}, $context->{release} );
  89     $context->{tmp_archive_file} = $archive_file;
  90     my @cmd = ( 'git', 'archive', '--format=tar', "--output=$archive_file", $context->{tag} );
  91     # run git command
  92     print( "Running: ", join( ' ', @cmd ), "\n" ) if ($verbose);
  93     system(@cmd) == 0 || croak "Export failed";
  94 }
  95
  96 # ------------------------------------------------------------------
  97
  98 sub unpack_tree {
  99     my $context = shift;
 100
 101     die "Cannot see archive file\n" unless ( -f $context->{tmp_archive_file} );
 102     my @cmd = ( 'tar', 'xf', $context->{tmp_archive_file}, '-C', $context->{release_tree} );
 103
 104     # run  command
 105     print( "Running: ", join( ' ', @cmd ), "\n" ) if ($verbose);
 106     system(@cmd) == 0 || croak "Unpack failed";
 107 }
 108
 109 # ------------------------------------------------------------------
 110
 111 sub make_version_script {
 112     my $context = shift;
 113
 114     my $variant = substr( $context->{release}, length($context->{trelease}) );
 115     if ( $context->{release} ne $context->{trelease} . $variant ) {
 116         die "Broken version numbering, I'm buggy";
 117     }
 118
 119     my $srcdir    = File::Spec->catdir( $context->{release_tree}, 'src', 'src' );
 120     chdir $srcdir or die "chdir $srcdir: $!\n";
 121
 122     if ( -f "version.sh" ) {
 123         print( "WARNING: version.sh already exists - leaving it in place\n" );
 124         return;
 125     }
 126
 127     # Currently (25. Feb. 2016) the mk_exim_release.pl up to now can't
 128     # deal with security releases.!? So we need a current
 129     # mk_exim_release.pl. But if we use a current (master), the
 130     # reversion script returns wrong version info (it's running inside
 131     # the Git tree and uses git --describe, which always returns the
 132     # current version of master.) I do not want to change the old
 133     # reversion scripts (in 4.86.1, 4.85.1).
 134     #
 135     # Thus we've to provide the version.sh, based on the info we have
 136     # about the release. If reversion finds this, it doesn't try to find
 137     # it's own way to get a valid version number from the git.
 138     #
 139     # 4.89 series: the logic here did not handle _RC<N> thus breaking RC
 140     # status in versions.  nb: candidatev in context should be same as $variant
 141     # in local context.
 142     my $stamp = $context->{minor} ? '_'.$context->{minor} : '';
 143     $stamp .= $context->{candidatev} if $context->{candidatev};
 144     #
 145     open(my $v, '>', 'version.sh') or die "Can't open '>version.sh' $!\n";
 146     print {$v} <<__;
 147 # initial version automatically generated from $0
 148 EXIM_RELEASE_VERSION=$context->{major}
 149 EXIM_VARIANT_VERSION=$stamp
 150 EXIM_COMPILE_NUMBER=0
 151 __
 152     close($v);
 153     unlink 'version.h';
 154     return;
 155
 156     # Later, if we get the reversion script fixed, we can call it again.
 157     # For now (25. Feb. 2016) we'll leave it unused.
 158     my @cmd = ("../scripts/reversion", "release", $context->{tag});
 159     print( "Running: ", join( ' ', @cmd ), "\n" ) if ($verbose);
 160     system(@cmd) == 0 || croak "reversion failed";
 161
 162     unlink "version.h";
 163
 164     -f "version.sh" or die "failed to create version.sh";
 165 }
 166
 167 # ------------------------------------------------------------------
 168
 169 sub build_html_documentation {
 170     my $context = shift;
 171
 172     my $genpath   = $context->{webgen_base} . '/script/gen.pl';
 173     my $templates = $context->{webgen_base} . '/templates';
 174     my $dir       = File::Spec->catdir( $context->{release_tree}, 'html' );
 175     my $spec      = File::Spec->catfile( $context->{docbook}, 'spec.xml' );
 176     my $filter    = File::Spec->catfile( $context->{docbook}, 'filter.xml' );
 177
 178     mkdir($dir);
 179
 180     my @cmd = (
 181         $genpath,   '--spec',    $spec,                '--filter',
 182         $filter,    '--latest',  $context->{trelease}, '--tmpl',
 183         $templates, '--docroot', $dir,                 '--localstatic'
 184     );
 185     push @cmd, '--verbose' if $verbose or $debug;
 186
 187     print "Executing ", join( ' ', @cmd ), "\n";
 188     system(@cmd);
 189
 190     # move directory into right place
 191     my $sourcedir = File::Spec->catdir( $context->{docbook}, 'filter.xml' );
 192
 193     rename(
 194         File::Spec->catdir( $dir,                sprintf( 'exim-html-%s', $context->{trelease} ) ),
 195         File::Spec->catdir( $context->{pkgdirs}, sprintf( 'exim-html-%s', $context->{release} ) )
 196     );
 197 }
 198
 199 # ------------------------------------------------------------------
 200
 201 sub copy_docbook_files {
 202     my $context = shift;
 203
 204     # where the generated docbook files can be found
 205     my $docdir = File::Spec->catdir( $context->{release_tree}, 'doc', 'doc-docbook' );
 206
 207     # where the website docbook source dir is - push files to here
 208     my $webdir = File::Spec->catdir( $context->{webgen_base}, 'docbook', $context->{trelease} );
 209     mkpath( $webdir, { verbose => ( $verbose || $debug ) } );
 210
 211     foreach my $file ( 'spec.xml', 'filter.xml' ) {
 212         my $from  = File::Spec->catfile( $docdir,             $file );
 213         my $to    = File::Spec->catfile( $context->{docbook}, $file );
 214         my $webto = File::Spec->catfile( $webdir,             $file );
 215         copy( $from, $to );
 216         copy( $from, $webto );
 217     }
 218 }
 219
 220 # ------------------------------------------------------------------
 221
 222 sub build_documentation {
 223     my $context = shift;
 224
 225     my $docdir = File::Spec->catdir( $context->{release_tree}, 'doc', 'doc-docbook' );
 226     # documentation building gets the truncated release, without RC
 227     system("cd '$docdir' && ./OS-Fixups && $context->{make_cmd} EXIM_VER=$context->{trelease} everything") == 0
 228       || croak "Doc build failed";
 229
 230     copy_docbook_files($context);
 231     build_html_documentation($context) if $context->{web};
 232 }
 233
 234 # ------------------------------------------------------------------
 235
 236 sub move_text_docs_into_pkg {
 237     my $context = shift;
 238
 239     my $old_docdir = File::Spec->catdir( $context->{release_tree}, 'doc', 'doc-docbook' );
 240     my $old_txtdir = File::Spec->catdir( $context->{release_tree}, 'doc', 'doc-txt' );
 241     my $new_docdir = File::Spec->catdir( $context->{eximpkgdir}, 'doc' );
 242     mkpath( $new_docdir, { verbose => ( $verbose || $debug ) } );
 243
 244     # move generated documents from docbook stuff
 245     foreach my $file (qw/exim.8 spec.txt filter.txt/) {
 246         die "Empty file \"$file\"\n" if -z File::Spec->catfile( $old_docdir, $file );
 247         move( File::Spec->catfile( $old_docdir, $file ), File::Spec->catfile( $new_docdir, $file ) );
 248     }
 249
 250     # move text documents across
 251     foreach my $file ( glob( File::Spec->catfile( $old_txtdir, '*' ) ) ) {
 252
 253         # skip a few we dont want
 254         my $fn = ( File::Spec->splitpath($file) )[2];
 255         next
 256           if ( ( $fn eq 'ABOUT' )
 257             || ( $fn eq 'ChangeLog.0' )
 258             || ( $fn eq 'test-harness.txt' )
 259             # Debian issue re licensing of RFCs
 260             || ( $fn =~ /^draft-ietf-.*/ )
 261             || ( $fn =~ /^rfc.*/ )
 262              );
 263         move( $file, File::Spec->catfile( $new_docdir, $fn ) );
 264     }
 265 }
 266
 267 # ------------------------------------------------------------------
 268
 269 sub build_pspdfinfo_directory {
 270     my $context = shift;
 271
 272     ##foreach my $format (qw/pdf postscript texinfo info/) {
 273     foreach my $format (qw/pdf postscript/) {
 274         my $target = File::Spec->catdir( $context->{pkgdirs}, sprintf( 'exim-%s-%s', $format, $context->{release} ), 'doc' );
 275         mkpath( $target, { verbose => ( $verbose || $debug ) } );
 276
 277         # move documents across
 278         foreach my $file (
 279             glob(
 280                 File::Spec->catfile(
 281                     $context->{release_tree},
 282                     'doc',
 283                     'doc-docbook',
 284                     (
 285                         ( $format eq 'postscript' )
 286                         ? '*.ps'
 287                         : ( '*.' . $format )
 288                     )
 289                 )
 290             )
 291           )
 292         {
 293             move( $file, File::Spec->catfile( $target, ( File::Spec->splitpath($file) )[2] ) );
 294         }
 295     }
 296 }
 297
 298 # ------------------------------------------------------------------
 299
 300 sub build_main_package_directory {
 301     my $context = shift;
 302
 303     # build the exim package directory path
 304     $context->{eximpkgdir} = File::Spec->catdir( $context->{pkgdirs}, sprintf( 'exim-%s', $context->{release} ) );
 305
 306     # initially we move the exim-src directory to the new directory name
 307     rename( File::Spec->catdir( $context->{release_tree}, 'src' ), $context->{eximpkgdir} )
 308       || croak "Rename of src dir failed - $!";
 309
 310     # add Local subdirectory
 311     mkpath( File::Spec->catdir( $context->{eximpkgdir}, 'Local' ), { verbose => ( $verbose || $debug ) } );
 312
 313     # now add the text docs
 314     move_text_docs_into_pkg($context);
 315 }
 316
 317 # ------------------------------------------------------------------
 318
 319 sub build_package_directories {
 320     my $context = shift;
 321
 322     build_main_package_directory($context);
 323     build_pspdfinfo_directory($context) if $context->{build_docs};
 324 }
 325
 326 # ------------------------------------------------------------------
 327
 328 sub do_cleanup {
 329     my $context = shift;
 330
 331     print "Cleaning up\n" if ($verbose);
 332     chdir( $context->{directory} ) || die;
 333     rmtree( $context->{release_tree}, { verbose => $debug } );
 334     rmtree( $context->{docbook},      { verbose => $debug } );
 335     rmtree( $context->{pkgdirs},      { verbose => $debug } );
 336 }
 337
 338 # ------------------------------------------------------------------
 339
 340 # We prefer gtar to tar if gtar exists in $PATH
 341
 342 sub fix_paths_tar {
 343     my $context = shift;
 344     my $tar = $context->{tar_cmd};
 345
 346     return unless $tar eq 'tar';
 347
 348     foreach my $d (File::Spec->path()) {
 349         my $p = File::Spec->catfile($d, 'gtar');
 350         if (-x $p) {
 351             $context->{tar_cmd} = $p;
 352             print "Switched tar command to: $p\n" if ($verbose);
 353             return;
 354         }
 355     }
 356 }
 357
 358 # ------------------------------------------------------------------
 359
 360 sub create_tar_files {
 361     my $context = shift;
 362
 363     my $pkgs    = $context->{pkgs};
 364     my $pkgdirs = $context->{pkgdirs};
 365     my $tar     = $context->{tar_cmd};
 366     if ($verbose) {
 367         foreach my $c (keys %{ $context->{compressors} }) {
 368             print "Compression: $c\t$context->{compressors}{$c}\n";
 369         }
 370     }
 371
 372     # We ideally do not want local system user information in release tarballs;
 373     # those are artifacts of use of tar for backups and have no place in
 374     # software release packaging; if someone extracts as root, then they should
 375     # get sane file ownerships.
 376     my $ownership = "";
 377     if (`tar --help 2>&1` =~ /^\s*--owner=/m) {
 378         $ownership .= " --owner=$context->{tar_perms}{user} --group=$context->{tar_perms}{group}";
 379         # on this GNU tar, --numeric-owner works during creation too
 380         $ownership .= " --numeric-owner";
 381     }
 382
 383     # See also environment variables set in main, tuning compression levels
 384     my @COMPRESSIONS = (
 385         # compressors-dict-key, file-extension, flags-as-string
 386         [ "gzip", "gz", "--gzip" ],
 387         [ "bzip2", "bz2", "--bzip2" ],
 388         [ "lzip", "lz", "--lzip" ],
 389         [ "xz", "xz", "--xz" ],
 390     );
 391
 392     foreach my $dir ( glob( File::Spec->catdir( $pkgdirs, ( 'exim*-' . $context->{release} ) ) ) ) {
 393         my $dirname = ( File::Spec->splitdir($dir) )[-1];
 394         foreach my $comp (@COMPRESSIONS) {
 395             my ($compkey, $extension, $flags) = @{$comp};
 396             next unless $context->{compressors}{$compkey};
 397             print "Creating: ${pkgs}/${dirname}.tar.${extension}\n" if ($verbose || $debug);
 398             system("$tar cf  ${pkgs}/${dirname}.tar.${extension} ${flags} ${ownership} -C ${pkgdirs} ${dirname}");
 399         }
 400     }
 401
 402 }
 403
 404 # ------------------------------------------------------------------
 405 MAIN: {
 406
 407     $0 =~ m|^(?:\./)?release-process/scripts/|
 408     or die "$0: please call this script from the root of the Exim project sources\n";
 409
 410     my $context = {
 411         pkgname     => 'exim',
 412         orig_dir    => File::Spec->curdir(),
 413         tmp_dir     => File::Temp->newdir(),
 414         webgen_base => "$FindBin::Bin/../../../exim-website",
 415         tar_cmd     => 'tar',
 416         tar_perms   => {
 417                 user    => '0',
 418                 group   => '0',
 419         },
 420         make_cmd    => 'make',
 421         compressors => {
 422                 gzip    => 1,
 423                 bzip2   => 1,
 424                 xz      => 1,
 425                 lzip    => 0,
 426         },
 427         build_docs   => 1,
 428         web          => 1,
 429     };
 430     my $delete;
 431     my $cleanup = 1;
 432     ##$ENV{'PATH'} = '/opt/local/bin:' . $ENV{'PATH'};
 433     # We are creating files for mass distribution, so work harder to make smaller files.
 434     $ENV{'GZIP'} = '-9';
 435     $ENV{'BZIP2'} = '-9';
 436     # xz documents minimum file sizes for levels higher than -6 to be useful and each
 437     # requires more RAM on the decompressing system.  Exim tarball currently 24MiB so
 438     # using -8.
 439     $ENV{'XZ_DEFAULTS'} = '-8';
 440
 441     GetOptions(
 442         'directory=s'   => \$context->{directory},
 443         'webgen_base=s' => \$context->{webgen_base},
 444         'tar=s'         => \$context->{tar_cmd},
 445         'make=s'        => \$context->{make_cmd},
 446         'lzip!'         => \$context->{compressors}{lzip},
 447         'verbose!'      => \$verbose,
 448         'debug!'        => \$debug,
 449         'help|?'        => sub { pod2usage(-verbose => 1, -exit => 0) },
 450         'man!'          => sub { pod2usage(-verbose => 2, -exit => 0, -noperldoc => system('perldoc -V >/dev/null 2>&1')) },
 451         'delete!'       => \$delete,
 452         'cleanup!'      => \$cleanup,
 453         'build-docs!'   => \$context->{build_docs},
 454         'web!'          => \$context->{web},
 455     ) and @ARGV == 1 or pod2usage;
 456
 457     umask(022);
 458     get_and_check_version( shift, $context );
 459     fix_paths_tar($context);
 460     $context->{tag} = build_tag($context);
 461     deal_with_working_directory( $context, $delete );
 462     export_git_tree($context);
 463     chdir( $context->{directory} ) || die;
 464     unpack_tree($context);
 465     make_version_script($context);
 466     build_documentation($context) if $context->{build_docs};
 467     build_package_directories($context);
 468     create_tar_files($context);
 469     do_cleanup($context) if ($cleanup);
 470 }
 471
 472 1;
 473
 474 __END__
 475
 476 =head1 NAME
 477
 478 mk_exim_release - Build an exim release
 479
 480 =head1 SYNOPSIS
 481
 482  mk_exim_release [options] version
 483
 484 =head1 DESCRIPTION
 485
 486 B<mk_exim_release> builds an exim release.
 487
 488 Starting in a populated git repo that has already been tagged for
 489 release it builds docs, packages etc.  Parameter is the version number
 490 to build as - ie 4.72 4.72RC1, 4.86.1, etc
 491
 492 After creating the release files, they should be signed. There is another
 493 helper for creating the signatures:
 494 F<release-process/scripts/sign_exim_packages>.
 495
 496 Call B<mk_exim_release> about like this:
 497
 498     release-process/scripts/mk_exim_release 4.99
 499
 500
 501 =head1 OPTIONS
 502
 503 =over 4
 504
 505 =item B<--[no]debug>
 506
 507 Forces debug mode. (default: no debug info)
 508
 509 =item B<--[no]delete>
 510
 511 Delete a pre-existing package directory at start. (default: don't delete)
 512
 513 =item B<--directory> I<dir>
 514
 515 Change the name of the package directory (default: F<< exim-packaging-<version> >>)
 516
 517 =item B<--[no]help>
 518
 519 Display short help and exit cleanly. (default: don't do that)
 520
 521 =item B<--[no]lzip>
 522
 523 Control the creation of B<lzip> tarballs. (default: do not use lzip)
 524
 525 =item B<--make> I<cmd>
 526
 527 Force the use of a specific C<make> command. This may be necessary if C<make> is not
 528 C<gmake> (default: C<make>)
 529
 530 =item B<--[no]man>
 531
 532 Display man page and exit cleanly. (default: don't do that)
 533
 534 =item B<--tar> I<cmd>
 535
 536 Use to override the path to the C<tar> command.  Need GNU tar in case
 537 I<lzip> is selected. (default: C<gtar>, if not found, use C<tar>)
 538
 539 =item B<--[no]web>
 540
 541 Control the creation of the website. For creation of the website, the F<../exim-website>
 542 directory must exist. (default: create the website)
 543
 544 =item B<--verbose>
 545
 546 Force verbose mode. (default: no verbosity)
 547
 548 =back
 549
 550 =head1 AUTHOR
 551
 552 Nigel Metheringham <Nigel.Metheringham@dev.intechnology.co.uk>,
 553 some changes by Heiko Schlittermann <hs@schlittermann.de>
 554
 555 =head1 COPYRIGHT
 556
 557 Copyright 2010-2016 Exim Maintainers. All rights reserved.
 558
 559 =cut
 560 # vim: set sw=4 et :