[exim.git] / release-process / scripts / mk_exim_release

#!/usr/bin/env perl
# Copyright (c) The Exim Maintainers 2016-2018

use strict;
use warnings;
use Carp;
use Cwd qw'abs_path';
use File::Basename;
use File::Path qw(make_path remove_tree);
use File::Temp;
use Getopt::Long;
use IO::File;
use Pod::Usage;
use Digest::SHA;
use if $ENV{DEBUG} => 'Smart::Comments';

my $ME = basename $0;


my $debug   = 0;
my $verbose = 0;

# MAJOR.MINOR[.SECURITY[.FIXES]][-RCX]
# 4    .90    .0        .22      -RC1
my $version_pattern = qr/
    (?<release>
           (?<major>\d)         # 4
         \.(?<minor>\d\d)       #  .90
      (?:\.(?<security>\d+)     #     .0
      (?:\.(?<fixes>)\d+)?)?    #       .22
       (?:-(?<rc>RC\d+)?)?      #          -RC1
    )
/x;

# ------------------------------------------------------------------

package Context {
    use strict;     # not strictly necessary yet, until in an own package
    use warnings;   # not strictly necessary yet, ...
    use File::Spec::Functions qw'splitpath catfile catdir splitdir';
    use File::Path qw'make_path remove_tree';
    use File::Copy;
    use Cwd qw'abs_path';
    use Carp;

    package PWD {
        use Cwd;
        sub TIESCALAR { bless do {\my $x} }
        sub FETCH { cwd }
    }

    tie my $cwd => 'PWD' or die;

    sub new {
        my $class = shift;
        return bless { @_ } => $class;
    }

    sub check_version {
        my $context = shift;
        my $version = shift;

        croak "The given version number does not look right - $version"
            if not $version =~ /$version_pattern/;

        $context->{v}{release}  = $+{release};
        $context->{v}{major} = $+{major};
        $context->{v}{minor} = $+{minor};
        $context->{v}{security} = $+{security};
        $context->{v}{rc} = $+{rc};

        die "$ME: This script doesn't work for versions prior 4.92-RCx. "
           ."Please checkout an older version.\n"
            if $context->{v}{major} < 4
            or $context->{v}{major} == 4 && $context->{v}{minor} < 92;

        ### v: $context->{v}

        # find a valid vcs tag matching the version
        my $pattern = "$context->{pkgname}-$context->{v}{release}" =~ s/[-_.]/[-_.]/gr;
        chomp(my @tags = qx{git tag --list '$pattern'});

        croak "The given version is ambigous, tags: @tags\n" if @tags > 1;
        croak "The given version does not exist (no such tag: exim-$version)\n" if @tags == 0;

        $context->{git_tag} = $tags[0];

        # target_release: the release we aim to reach with release candidates
        # FIXME: re-construct from the parsed version number
        ($context->{v}{target_release} = $context->{v}{release}) =~ s/-RC\d+//;
    }


    # We prefer gtar to tar if gtar exists in $PATH
    sub override_tar_cmd {
        my $context = shift;
        my $tar = $context->{tar_cmd};

        return unless $tar eq 'tar';

        foreach my $d (File::Spec->path()) {
            my $p = catfile($d, 'gtar');
            if (-x $p) {
                $context->{tar_cmd} = $p;
                print "Switched tar command to: $p\n" if $verbose;
                return;
            }
        }
    }

    sub prepare_working_directory {
        my $context = shift;
        my $workspace = $context->{workspace};

        if (not defined $workspace) {
            $workspace = $context->{workspace} = File::Temp->newdir(File::Spec->tmpdir . '/exim-packaging-XXXX');
        }
        else {
            # ensure the working directory is not in place
            if (-e $workspace) {
                if ($context->{delete}) {
                    print "Deleting existing $workspace\n" if $verbose;
                    remove_tree $workspace, { verbose => $verbose || $debug };
                }
                else {
                    croak "Working directory $workspace exists" if -e $workspace;
                }
            }

            # create base directory
            make_path( $context->{directory}, { verbose => $verbose || $debug } );
        }

        # Set(!) and create subdirectories
        foreach (qw(vcs_export pkg_tars pkg_trees tmp)) {   # {dookbook}
            make_path(
                $context->{d}{$_} = catdir($workspace, $_),
                { verbose => $verbose || $debug });
        }
    }

    sub export_git_tree {
        my $context = shift;

        # build git command
        my $archive_file = $context->{tmp_archive_file} = sprintf'%s/%s-%s.tar', $context->{d}{tmp}, $context->{pkgname}, $context->{v}{release};
        ### $archive_file
        my @cmd = ( 'git', 'archive', '--format=tar', "--output=$archive_file", $context->{git_tag} );
        ### @cmd
        # run git command
        print "[$cwd] Running: @cmd\n" if $verbose;
        0 == system @cmd or croak "Export failed";
    }

    sub unpack_tree {
        # TODO: Why can't we combine the export_git_tree with the
        # unpack_tree function?
        my $context = shift;

        ### $context
        die "Cannot see archive file\n" unless -f $context->{tmp_archive_file};
        my @cmd = ('tar',
            xf => $context->{tmp_archive_file},
            -C => $context->{d}{vcs_export} );

        # run  command
        print "[$cwd] Running: @cmd\n" if $verbose;
        system @cmd and croak "Unpack failed\n";

    }

    sub make_version_script {
        my $context = shift;

        #my $variant = substr( $context->{v}{release}, length($context->{v}{target_release}) );
        #if ( $context->{v}{release} ne $context->{v}{target_release} . $variant ) {
        #    die "Broken version numbering, I'm buggy";
        #}


        # Work
        if (not my $pid = fork // die "$ME: Cannot fork: $!\n") {

            my $source_tree    = catdir($context->{d}{vcs_export}, 'src', 'src');
            ### $source_tree

            chdir $source_tree or die "chdir $source_tree: $!\n";


            croak "WARNING: version.sh already exists - leaving it in place\n"
                if -f 'version.sh';

            # Currently (25. Feb. 2016) the mk_exim_release.pl up to now can't
            # deal with security releases.!? So we need a current
            # mk_exim_release.pl. But if we use a current (master), the
            # reversion script returns wrong version info (it's running inside
            # the Git tree and uses git --describe, which always returns the
            # current version of master.) I do not want to change the old
            # reversion scripts (in 4.86.1, 4.85.1).
            #
            # Thus we've to provide the version.sh, based on the info we have
            # about the release. If reversion finds this, it doesn't try to find
            # it's own way to get a valid version number from the git.
            #
            # 4.89 series: the logic here did not handle _RC<N> thus breaking RC
            # status in versions.  nb: rc in context should be same as $variant
            # in local context.

            #my $stamp = $context->{minor} ? '_'.$context->{minor} : '';
            #$stamp .= $context->{rc} if $context->{rc};
            my $variant = $context->{v}{rc} // '';

            print "[$cwd] create version.sh\n" if $verbose;
            open(my $v, '>', 'version.sh') or die "Can't open version.sh for writing: $!\n";
            print {$v} <<__;
# initial version automatically generated by $0
EXIM_RELEASE_VERSION=$context->{v}{target_release}
EXIM_VARIANT_VERSION=$variant
EXIM_COMPILE_NUMBER=0
# echo "[[[ \$EXIM_RELEASE_VERSION | \$EXIM_VARIANT_VERSION | \$EXIM_COMPILE_NUMBER ]]]"
__
            close $v  or die "$0: Can not close $source_tree/version.h: $!\n";
            unlink 'version.h' or die "$ME: Can not unlink $source_tree/version.h: $!\n"
                if -f 'version.h';

            # Later, if we get the reversion script fixed, we can call it again.
            # For now (25. Feb. 2016) we'll leave it unused.
            #my @cmd = ('../scripts/reversion', 'release', $context->{git_tag});

            my @cmd = ('../scripts/reversion', 'release');
            print "[$cwd] Running: @cmd\n" if $verbose;
            system(@cmd) and croak "reversion failed";

            die "$ME: failed to create version.sh"
                unless -f 'version.sh';

            exit 0;
        }
        else {
            $pid == waitpid($pid, 0) or die "$0: waidpid: $!\n";
            exit $? >> 8 if $?;
        }
    }

    sub build_documentation {
        my $context = shift;
        my $docdir = catdir $context->{d}{vcs_export}, 'doc', 'doc-docbook';

        # documentation building does a chdir, so we'll do it in a
        # subprocess
        if (not my $pid = fork // die "$ME: Can't fork: $!\n") {
            chdir $docdir or die "$ME: Can't chdir to $docdir: $!\n";
            system('./OS-Fixups') == 0 or exit $?;
            exec $context->{make_cmd},
                "EXIM_VER=$context->{v}{target_release}" .
                    ($context->{v}{rc} ? "-$context->{v}{rc}" : ''),
                    'everything';
            die "$ME: [$cwd] Cannot exec $context->{make_cmd}: $!\n";
        }
        else {
            waitpid($pid, 0);
            exit $? >> 8 if $?;
        }

        $context->copy_docbook_files;
    }

    sub copy_docbook_files {
        my $context = shift;

        # where the generated docbook files can be found
        my $docdir = catdir $context->{d}{vcs_export}, 'doc', 'doc-docbook';

        foreach ('spec.xml', 'filter.xml') {
            my $from = catfile $docdir, $_;
            my $to = catdir $context->{d}{tmp}; # {dookbook}
            copy $from => $to    or die $@;
        }
    }

    sub build_html_documentation {
        my $context = shift;

        # where the website docbook source dir is - push the generated
        # files there
        {
            my $webdir = catdir $context->{website_base}, 'docbook', $context->{v}{target_release};
            make_path $webdir, { verbose => $verbose + $debug };
            copy catfile($context->{d}{vcs_export}, 'doc', 'doc-docbook', $_)
                => $webdir or die $@
                for 'spec.xml', 'filter.xml';
        }

        my $gen    = catfile $context->{website_base}, 'script/gen';
        my $outdir = catdir $context->{d}{pkg_trees}, "exim-html-$context->{v}{release}";

        make_path $outdir, { verbose => $verbose || $debug };

        my @cmd = (
            $gen,
            '--spec'    => catfile($context->{d}{tmp}, 'spec.xml'),     # {dookbook}
            '--filter'  => catfile($context->{d}{tmp}, 'filter.xml'),   # {dookbok}
            '--latest'  => $context->{v}{target_release},
            '--docroot' => $outdir,
            '--localstatic',
            ($verbose || $debug ? '--verbose' : ()),
        );

        print "[$cwd] Executing @cmd\n";
        0 == system @cmd or exit $? >> 8;

    }

    sub sign {
        my $context = shift;
        foreach my $tar (glob "$context->{d}{pkg_tars}/*") {
            system gpg =>
            '--quiet', '--batch',
            defined $context->{gpg}{key}
                ? ('--local-user' => $context->{gpg}{key})
                : (),
            '--detach-sig', '--armor', $tar;
        }
    }

    sub move_to_outdir {
        my $context = shift;
        make_path $context->{OUTDIR}, { verbose => $verbose || $debug };
        move $_ => $context->{OUTDIR} or die $@
            for glob "$context->{d}{pkg_tars}/*";
    }

    sub build_src_package_directory {
        my $context = shift;

        # build the exim package directory path
        $context->{d}{src} = catdir $context->{d}{pkg_trees}, "exim-$context->{v}{release}";

        # initially we move the exim-src directory to the new directory name
        move
            catdir( $context->{d}{vcs_export}, 'src')
            => $context->{d}{src}
        or croak "Move of src dir failed - $!";

        # add Local subdirectory
        make_path( catdir( $context->{d}{src}, 'Local' ), { verbose => $verbose || $debug } );

        # now add the text docs
        $context->move_text_docs_into_pkg;
    }

    sub build_doc_packages_directory {
        my $context = shift;

        ##foreach my $format (qw/pdf postscript texinfo info/) {
        foreach my $format (qw/pdf postscript/) {
            my $target = catdir $context->{d}{pkg_trees}, "exim-$format-$context->{v}{release}", 'doc';
            make_path( $target, { verbose => $verbose || $debug } );

            # move documents across
            foreach my $file (
                glob(
                    catfile(
                        $context->{d}{vcs_export},
                        'doc',
                        'doc-docbook',
                        (
                            ( $format eq 'postscript' )
                            ? '*.ps'
                            : ( '*.' . $format )
                        )
                    )
                )
            )
            {
                move( $file, catfile( $target, ( splitpath($file) )[2] ) );
            }
        }
    }

    sub move_text_docs_into_pkg {
        my $context = shift;

        my $old_docdir = catdir( $context->{d}{vcs_export}, 'doc', 'doc-docbook' );
        my $old_txtdir = catdir( $context->{d}{vcs_export}, 'doc', 'doc-txt' );
        my $new_docdir = catdir( $context->{d}{src}, 'doc' );
        make_path( $new_docdir, { verbose => $verbose || $debug } );

        # move generated documents from docbook stuff
        foreach my $file (qw/exim.8 spec.txt filter.txt/) {
            die "Empty file \"$file\"\n" if -z catfile( $old_docdir, $file );
            move( catfile( $old_docdir, $file ), catfile( $new_docdir, $file ) );
        }

        # move text documents across
        foreach my $file ( glob( catfile( $old_txtdir, '*' ) ) ) {

            # skip a few we dont want
            my $fn = ( splitpath($file) )[2];
            next
            if ( ( $fn eq 'ABOUT' )
                || ( $fn eq 'ChangeLog.0' )
                || ( $fn eq 'test-harness.txt' )
                # Debian issue re licensing of RFCs
                || ( $fn =~ /^draft-ietf-.*/ )
                || ( $fn =~ /^rfc.*/ )
                );
            move( $file, catfile( $new_docdir, $fn ) );
        }
    }

    sub create_tar_files {
        my $context = shift;

        my $pkg_tars    = $context->{d}{pkg_tars};
        my $pkg_trees = $context->{d}{pkg_trees};
        my $tar     = $context->{tar_cmd};
        if ($verbose) {
            foreach my $c (keys %{ $context->{compressors} }) {
                print "Compression: $c\t$context->{compressors}{$c}\n";
            }
        }

        # We ideally do not want local system user information in release tarballs;
        # those are artifacts of use of tar for backups and have no place in
        # software release packaging; if someone extracts as root, then they should
        # get sane file ownerships.
        my @ownership = (
            '--owner' => $context->{tar_perms}{user},
            '--group' => $context->{tar_perms}{group},
            # on this GNU tar, --numeric-owner works during creation too
            '--numeric-owner'
        ) if qx/tar --help 2>&1/ =~ /^\s*--owner=/m;

        # See also environment variables set in main, tuning compression levels
        my %COMPRESSION = (
            gzip  => { extension => 'gz',  flags => '--gzip' },
            bzip2 => { extension => 'bz2', flags => '--bzip2' },
            lzip  => { extension => 'lz',  flags => '--lzip' },
            xz    => { extension => 'xz',  flags => '--xz' },
        );


        my (%size, %sha256);

        foreach my $dir ( glob( catdir( $pkg_trees, ( 'exim*-' . $context->{v}{release} ) ) ) ) {
            my $dirname = ( splitdir($dir) )[-1];
            foreach my $comp (keys %COMPRESSION) {
                next unless $context->{compressors}{$comp};

                my $basename = "$dirname.tar.$COMPRESSION{$comp}{extension}";
                my $outfile = catfile $pkg_tars, $basename;

                print "Creating: $outfile\n" if $verbose || $debug;
                0 == system($tar,
                    cf => $outfile,
                        $COMPRESSION{$comp}{flags},
                        @ownership, -C => $pkg_trees, $dirname)
                    or exit $? >> 8;

                # calculate size and md5sum
                $size{$basename} = -s $outfile;
                $sha256{$basename} = do {
                    my $sha = Digest::SHA->new(256);
                    $sha->addfile($outfile);
                    $sha->hexdigest;
                };
            }
        }

        # write the sizes file
        open my $sizes, '>', $_ = catfile $pkg_tars, 'sizes.txt'
            or die "$ME: Can't open `$_': $!\n";

        print $sizes join "\n",
            (map { "SIZE($_) = $size{$_}" } sort keys %size),
            (map { "SHA256($_) = $sha256{$_}" } sort keys %sha256);

        close($sizes) or die "$ME: Can't close $_: $!\n";
    }

    sub do_cleanup {
        my $context = shift;

        print "Cleaning up\n" if $verbose;
        remove_tree $context->{d}{tmp}, { verbose => $debug + $verbose };
    }

}

MAIN: {

    # some of these settings are useful only if we're in the
    # exim-projekt-root, but the check, if we're, is deferred
    my $context = Context->new(
        pkgname     => 'exim',
        website_base => abs_path('../exim-website'),
        tar_cmd     => 'tar',
        tar_perms   => {
                user    => '0',
                group   => '0',
        },
        make_cmd    => 'make',
        compressors => {
                gzip    => 1,
                bzip2   => 1,
                xz      => 1,
                lzip    => 0,
        },
        docs         => 1,
        web          => 1,
        delete       => 0,
        cleanup      => 1,
        gpg => {
            sign         => 1,
            key          => undef,
        },
    );

    ##$ENV{'PATH'} = '/opt/local/bin:' . $ENV{'PATH'};
    # We are creating files for mass distribution, so work harder to make smaller files.
    $ENV{GZIP} = -9;
    $ENV{BZIP2} = -9;
    # xz documents minimum file sizes for levels higher than -6 to be useful and each
    # requires more RAM on the decompressing system.  Exim tarball currently 24MiB so
    # using -8.
    $ENV{XZ_DEFAULTS} = -8;

    GetOptions(
        $context,
        qw(workspace|tmp=s outdir=s website_base|webgen_base=s tar_cmd=s make_cmd=s docs|build-docs! web|build-web!
           delete! cleanup!),
        'sign!'         => \$context->{gpg}{sign},
        'key=s'         => \$context->{gpg}{key},
        'lzip!'         => \$context->{compressors}{lzip},
        'verbose!'      => \$verbose,
        'debug!'        => \$debug,
        'help|?'        => sub { pod2usage(-verbose => 1, -exit => 0) },
        'man!'          => sub { pod2usage(-verbose => 2, -exit => 0, -noperldoc => system('perldoc -V >/dev/null 2>&1')) },
    ) and @ARGV == 2 or pod2usage;

    -f '.exim-project-root'
        or die "$ME: please call this script from the root of the Exim project sources\n";

    $context->{OUTDIR} = pop @ARGV;

    if ($context->{gpg}{sign}) {
        $context->{gpg}{key} //= do { chomp($_ = qx/git config user.signingkey/); $_ }
            || $ENV{EXIM_KEY}
            || do {
                warn "$ME: No GPG key, using default\n";
                undef;
            }
    }


    warn "$ME: changed umask to 022\n" if umask(022) != 022;

    $context->check_version(shift);
    $context->override_tar_cmd;
    $context->prepare_working_directory;
    $context->export_git_tree;
    $context->unpack_tree;
    $context->make_version_script;
    $context->build_documentation if $context->{docs};
    $context->build_html_documentation if $context->{docs} && $context->{web};

    $context->build_src_package_directory;
    $context->build_doc_packages_directory if $context->{docs};

    $context->create_tar_files;
    $context->sign if $context->{gpg}{sign};
    $context->move_to_outdir;
    $context->do_cleanup if $context->{cleanup};

    ### $context
}

1;

__END__

=head1 NAME

mk_exim_release - Build an exim release

=head1 SYNOPSIS

 mk_exim_release [options] version PKG-DIRECTORY

=head1 DESCRIPTION

B<mk_exim_release> builds an exim release.

Starting in a populated git repo that has already been tagged for
release it builds docs, packages etc.  Parameter is the version number
to build as - ie 4.72 4.72-RC1, 4.86.1, etc, without any prefix.

This scripts expects to find a tag "exim-<version>".

After creating the release files, they should be signed. There is another
helper for creating the signatures:
F<release-process/scripts/sign_exim_packages>.

Call B<mk_exim_release> about like this:

    release-process/scripts/mk_exim_release 4.99 OUT-DIR


=head1 OPTIONS

=over 4

=item B<--[no]cleanup>

Do (or do not) cleanup the tmp directory at exit (default: do cleanup)

=item B<--[no]debug>

Forces debug mode. (default: no debug info)

=item B<--[no]delete>

Delete a pre-existing tmp- and package-directory at start. (default: don't delete)

=item B<--tmpdir> I<dir>

Change the name of the tmp directory (default: temporary directory)

=item B<--[no]help>

Display short help and exit cleanly. (default: don't do that)

=item B<--[no]lzip>

Control the creation of B<lzip> tarballs. (default: do not use lzip)

=item B<--make> I<cmd>

Force the use of a specific C<make> command. This may be necessary if C<make> is not
C<gmake> (default: C<make>)

=item B<--[no]man>

Display man page and exit cleanly. (default: don't do that)

=item B<--tar> I<cmd>

Use to override the path to the C<tar> command.  Need GNU tar in case
I<lzip> is selected. (default: C<gtar>, if not found, use C<tar>)

=item B<--[no]sign>

Sign the created archive files (and the sizes.txt). (default: sign)

=item B<--key> I<GPG key>

Use this GPG key for signing. If nothing is specified the first one of this list
is used:

=over 8

=item - git config user.signingkey

=item - environment C<EXIM_KEY>

=item - default GPG key

=back

=item B<--[no]web>

Control the creation of the website. For creation of the website, the F<../exim-website>
(but see the B<website-base> option) directory must exist. (default: create the website)

=item B<--website-base> I<dir>

Base directory for the web site generation (default: F<../exim-website>)

=item B<--verbose>

Force verbose mode. (default: no verbosity)

=back

=head1 AUTHOR

Nigel Metheringham <Nigel.Metheringham@dev.intechnology.co.uk>,
some changes by Heiko Schlittermann <hs@schlittermann.de>

=head1 COPYRIGHT

Copyright 2010-2016 Exim Maintainers. All rights reserved.

=cut
# vim: set sw=4 et :
Commit	Line	Data
	1	#!/usr/bin/env perl
	2	# Copyright (c) The Exim Maintainers 2016-2018
	3
	4	use strict;
	5	use warnings;
	6	use Carp;
	7	use Cwd qw'abs_path';
	8	use File::Basename;
	9	use File::Path qw(make_path remove_tree);
	10	use File::Temp;
	11	use Getopt::Long;
	12	use IO::File;
	13	use Pod::Usage;
	14	use Digest::SHA;
	15	use if $ENV{DEBUG} => 'Smart::Comments';
	16
	17	my $ME = basename $0;
	18
	19
	20	my $debug = 0;
	21	my $verbose = 0;
	22
	23	# MAJOR.MINOR[.SECURITY[.FIXES]][-RCX]
	24	# 4 .90 .0 .22 -RC1
	25	my $version_pattern = qr/
	26	(?<release>
	27	(?<major>\d) # 4
	28	\.(?<minor>\d\d) # .90
	29	(?:\.(?<security>\d+) # .0
	30	(?:\.(?<fixes>)\d+)?)? # .22
	31	(?:-(?<rc>RC\d+)?)? # -RC1
	32	)
	33	/x;
	34
	35	# ------------------------------------------------------------------
	36
	37	package Context {
	38	use strict; # not strictly necessary yet, until in an own package
	39	use warnings; # not strictly necessary yet, ...
	40	use File::Spec::Functions qw'splitpath catfile catdir splitdir';
	41	use File::Path qw'make_path remove_tree';
	42	use File::Copy;
	43	use Cwd qw'abs_path';
	44	use Carp;
	45
	46	package PWD {
	47	use Cwd;
	48	sub TIESCALAR { bless do {\my $x} }
	49	sub FETCH { cwd }
	50	}
	51
	52	tie my $cwd => 'PWD' or die;
	53
	54	sub new {
	55	my $class = shift;
	56	return bless { @_ } => $class;
	57	}
	58
	59	sub check_version {
	60	my $context = shift;
	61	my $version = shift;
	62
	63	croak "The given version number does not look right - $version"
	64	if not $version =~ /$version_pattern/;
	65
	66	$context->{v}{release} = $+{release};
	67	$context->{v}{major} = $+{major};
	68	$context->{v}{minor} = $+{minor};
	69	$context->{v}{security} = $+{security};
	70	$context->{v}{rc} = $+{rc};
	71
	72	die "$ME: This script doesn't work for versions prior 4.92-RCx. "
	73	."Please checkout an older version.\n"
	74	if $context->{v}{major} < 4
	75	or $context->{v}{major} == 4 && $context->{v}{minor} < 92;
	76
	77	### v: $context->{v}
	78
	79	# find a valid vcs tag matching the version
	80	my $pattern = "$context->{pkgname}-$context->{v}{release}" =~ s/[-_.]/[-_.]/gr;
	81	chomp(my @tags = qx{git tag --list '$pattern'});
	82
	83	croak "The given version is ambigous, tags: @tags\n" if @tags > 1;
	84	croak "The given version does not exist (no such tag: exim-$version)\n" if @tags == 0;
	85
	86	$context->{git_tag} = $tags[0];
	87
	88	# target_release: the release we aim to reach with release candidates
	89	# FIXME: re-construct from the parsed version number
	90	($context->{v}{target_release} = $context->{v}{release}) =~ s/-RC\d+//;
	91	}
	92
	93
	94	# We prefer gtar to tar if gtar exists in $PATH
	95	sub override_tar_cmd {
	96	my $context = shift;
	97	my $tar = $context->{tar_cmd};
	98
	99	return unless $tar eq 'tar';
	100
	101	foreach my $d (File::Spec->path()) {
	102	my $p = catfile($d, 'gtar');
	103	if (-x $p) {
	104	$context->{tar_cmd} = $p;
	105	print "Switched tar command to: $p\n" if $verbose;
	106	return;
	107	}
	108	}
	109	}
	110
	111	sub prepare_working_directory {
	112	my $context = shift;
	113	my $workspace = $context->{workspace};
	114
	115	if (not defined $workspace) {
	116	$workspace = $context->{workspace} = File::Temp->newdir(File::Spec->tmpdir . '/exim-packaging-XXXX');
	117	}
	118	else {
	119	# ensure the working directory is not in place
	120	if (-e $workspace) {
	121	if ($context->{delete}) {
	122	print "Deleting existing $workspace\n" if $verbose;
	123	remove_tree $workspace, { verbose => $verbose \|\| $debug };
	124	}
	125	else {
	126	croak "Working directory $workspace exists" if -e $workspace;
	127	}
	128	}
	129
	130	# create base directory
	131	make_path( $context->{directory}, { verbose => $verbose \|\| $debug } );
	132	}
	133
	134	# Set(!) and create subdirectories
	135	foreach (qw(vcs_export pkg_tars pkg_trees tmp)) { # {dookbook}
	136	make_path(
	137	$context->{d}{$_} = catdir($workspace, $_),
	138	{ verbose => $verbose \|\| $debug });
	139	}
	140	}
	141
	142	sub export_git_tree {
	143	my $context = shift;
	144
	145	# build git command
	146	my $archive_file = $context->{tmp_archive_file} = sprintf'%s/%s-%s.tar', $context->{d}{tmp}, $context->{pkgname}, $context->{v}{release};
	147	### $archive_file
	148	my @cmd = ( 'git', 'archive', '--format=tar', "--output=$archive_file", $context->{git_tag} );
	149	### @cmd
	150	# run git command
	151	print "[$cwd] Running: @cmd\n" if $verbose;
	152	0 == system @cmd or croak "Export failed";
	153	}
	154
	155	sub unpack_tree {
	156	# TODO: Why can't we combine the export_git_tree with the
	157	# unpack_tree function?
	158	my $context = shift;
	159
	160	### $context
	161	die "Cannot see archive file\n" unless -f $context->{tmp_archive_file};
	162	my @cmd = ('tar',
	163	xf => $context->{tmp_archive_file},
	164	-C => $context->{d}{vcs_export} );
	165
	166	# run command
	167	print "[$cwd] Running: @cmd\n" if $verbose;
	168	system @cmd and croak "Unpack failed\n";
	169
	170	}
	171
	172	sub make_version_script {
	173	my $context = shift;
	174
	175	#my $variant = substr( $context->{v}{release}, length($context->{v}{target_release}) );
	176	#if ( $context->{v}{release} ne $context->{v}{target_release} . $variant ) {
	177	# die "Broken version numbering, I'm buggy";
	178	#}
	179
	180
	181	# Work
	182	if (not my $pid = fork // die "$ME: Cannot fork: $!\n") {
	183
	184	my $source_tree = catdir($context->{d}{vcs_export}, 'src', 'src');
	185	### $source_tree
	186
	187	chdir $source_tree or die "chdir $source_tree: $!\n";
	188
	189
	190	croak "WARNING: version.sh already exists - leaving it in place\n"
	191	if -f 'version.sh';
	192
	193	# Currently (25. Feb. 2016) the mk_exim_release.pl up to now can't
	194	# deal with security releases.!? So we need a current
	195	# mk_exim_release.pl. But if we use a current (master), the
	196	# reversion script returns wrong version info (it's running inside
	197	# the Git tree and uses git --describe, which always returns the
	198	# current version of master.) I do not want to change the old
	199	# reversion scripts (in 4.86.1, 4.85.1).
	200	#
	201	# Thus we've to provide the version.sh, based on the info we have
	202	# about the release. If reversion finds this, it doesn't try to find
	203	# it's own way to get a valid version number from the git.
	204	#
	205	# 4.89 series: the logic here did not handle _RC<N> thus breaking RC
	206	# status in versions. nb: rc in context should be same as $variant
	207	# in local context.
	208
	209	#my $stamp = $context->{minor} ? '_'.$context->{minor} : '';
	210	#$stamp .= $context->{rc} if $context->{rc};
	211	my $variant = $context->{v}{rc} // '';
	212
	213	print "[$cwd] create version.sh\n" if $verbose;
	214	open(my $v, '>', 'version.sh') or die "Can't open version.sh for writing: $!\n";
	215	print {$v} <<__;
	216	# initial version automatically generated by $0
	217	EXIM_RELEASE_VERSION=$context->{v}{target_release}
	218	EXIM_VARIANT_VERSION=$variant
	219	EXIM_COMPILE_NUMBER=0
	220	# echo "[[[ \$EXIM_RELEASE_VERSION \| \$EXIM_VARIANT_VERSION \| \$EXIM_COMPILE_NUMBER ]]]"
	221	__
	222	close $v or die "$0: Can not close $source_tree/version.h: $!\n";
	223	unlink 'version.h' or die "$ME: Can not unlink $source_tree/version.h: $!\n"
	224	if -f 'version.h';
	225
	226	# Later, if we get the reversion script fixed, we can call it again.
	227	# For now (25. Feb. 2016) we'll leave it unused.
	228	#my @cmd = ('../scripts/reversion', 'release', $context->{git_tag});
	229
	230	my @cmd = ('../scripts/reversion', 'release');
	231	print "[$cwd] Running: @cmd\n" if $verbose;
	232	system(@cmd) and croak "reversion failed";
	233
	234	die "$ME: failed to create version.sh"
	235	unless -f 'version.sh';
	236
	237	exit 0;
	238	}
	239	else {
	240	$pid == waitpid($pid, 0) or die "$0: waidpid: $!\n";
	241	exit $? >> 8 if $?;
	242	}
	243	}
	244
	245	sub build_documentation {
	246	my $context = shift;
	247	my $docdir = catdir $context->{d}{vcs_export}, 'doc', 'doc-docbook';
	248
	249	# documentation building does a chdir, so we'll do it in a
	250	# subprocess
	251	if (not my $pid = fork // die "$ME: Can't fork: $!\n") {
	252	chdir $docdir or die "$ME: Can't chdir to $docdir: $!\n";
	253	system('./OS-Fixups') == 0 or exit $?;
	254	exec $context->{make_cmd},
	255	"EXIM_VER=$context->{v}{target_release}" .
	256	($context->{v}{rc} ? "-$context->{v}{rc}" : ''),
	257	'everything';
	258	die "$ME: [$cwd] Cannot exec $context->{make_cmd}: $!\n";
	259	}
	260	else {
	261	waitpid($pid, 0);
	262	exit $? >> 8 if $?;
	263	}
	264
	265	$context->copy_docbook_files;
	266	}
	267
	268	sub copy_docbook_files {
	269	my $context = shift;
	270
	271	# where the generated docbook files can be found
	272	my $docdir = catdir $context->{d}{vcs_export}, 'doc', 'doc-docbook';
	273
	274	foreach ('spec.xml', 'filter.xml') {
	275	my $from = catfile $docdir, $_;
	276	my $to = catdir $context->{d}{tmp}; # {dookbook}
	277	copy $from => $to or die $@;
	278	}
	279	}
	280
	281	sub build_html_documentation {
	282	my $context = shift;
	283
	284	# where the website docbook source dir is - push the generated
	285	# files there
	286	{
	287	my $webdir = catdir $context->{website_base}, 'docbook', $context->{v}{target_release};
	288	make_path $webdir, { verbose => $verbose + $debug };
	289	copy catfile($context->{d}{vcs_export}, 'doc', 'doc-docbook', $_)
	290	=> $webdir or die $@
	291	for 'spec.xml', 'filter.xml';
	292	}
	293
	294	my $gen = catfile $context->{website_base}, 'script/gen';
	295	my $outdir = catdir $context->{d}{pkg_trees}, "exim-html-$context->{v}{release}";
	296
	297	make_path $outdir, { verbose => $verbose \|\| $debug };
	298
	299	my @cmd = (
	300	$gen,
	301	'--spec' => catfile($context->{d}{tmp}, 'spec.xml'), # {dookbook}
	302	'--filter' => catfile($context->{d}{tmp}, 'filter.xml'), # {dookbok}
	303	'--latest' => $context->{v}{target_release},
	304	'--docroot' => $outdir,
	305	'--localstatic',
	306	($verbose \|\| $debug ? '--verbose' : ()),
	307	);
	308
	309	print "[$cwd] Executing @cmd\n";
	310	0 == system @cmd or exit $? >> 8;
	311
	312	}
	313
	314	sub sign {
	315	my $context = shift;
	316	foreach my $tar (glob "$context->{d}{pkg_tars}/*") {
	317	system gpg =>
	318	'--quiet', '--batch',
	319	defined $context->{gpg}{key}
	320	? ('--local-user' => $context->{gpg}{key})
	321	: (),
	322	'--detach-sig', '--armor', $tar;
	323	}
	324	}
	325
	326	sub move_to_outdir {
	327	my $context = shift;
	328	make_path $context->{OUTDIR}, { verbose => $verbose \|\| $debug };
	329	move $_ => $context->{OUTDIR} or die $@
	330	for glob "$context->{d}{pkg_tars}/*";
	331	}
	332
	333	sub build_src_package_directory {
	334	my $context = shift;
	335
	336	# build the exim package directory path
	337	$context->{d}{src} = catdir $context->{d}{pkg_trees}, "exim-$context->{v}{release}";
	338
	339	# initially we move the exim-src directory to the new directory name
	340	move
	341	catdir( $context->{d}{vcs_export}, 'src')
	342	=> $context->{d}{src}
	343	or croak "Move of src dir failed - $!";
	344
	345	# add Local subdirectory
	346	make_path( catdir( $context->{d}{src}, 'Local' ), { verbose => $verbose \|\| $debug } );
	347
	348	# now add the text docs
	349	$context->move_text_docs_into_pkg;
	350	}
	351
	352	sub build_doc_packages_directory {
	353	my $context = shift;
	354
	355	##foreach my $format (qw/pdf postscript texinfo info/) {
	356	foreach my $format (qw/pdf postscript/) {
	357	my $target = catdir $context->{d}{pkg_trees}, "exim-$format-$context->{v}{release}", 'doc';
	358	make_path( $target, { verbose => $verbose \|\| $debug } );
	359
	360	# move documents across
	361	foreach my $file (
	362	glob(
	363	catfile(
	364	$context->{d}{vcs_export},
	365	'doc',
	366	'doc-docbook',
	367	(
	368	( $format eq 'postscript' )
	369	? '*.ps'
	370	: ( '*.' . $format )
	371	)
	372	)
	373	)
	374	)
	375	{
	376	move( $file, catfile( $target, ( splitpath($file) )[2] ) );
	377	}
	378	}
	379	}
	380
	381	sub move_text_docs_into_pkg {
	382	my $context = shift;
	383
	384	my $old_docdir = catdir( $context->{d}{vcs_export}, 'doc', 'doc-docbook' );
	385	my $old_txtdir = catdir( $context->{d}{vcs_export}, 'doc', 'doc-txt' );
	386	my $new_docdir = catdir( $context->{d}{src}, 'doc' );
	387	make_path( $new_docdir, { verbose => $verbose \|\| $debug } );
	388
	389	# move generated documents from docbook stuff
	390	foreach my $file (qw/exim.8 spec.txt filter.txt/) {
	391	die "Empty file \"$file\"\n" if -z catfile( $old_docdir, $file );
	392	move( catfile( $old_docdir, $file ), catfile( $new_docdir, $file ) );
	393	}
	394
	395	# move text documents across
	396	foreach my $file ( glob( catfile( $old_txtdir, '*' ) ) ) {
	397
	398	# skip a few we dont want
	399	my $fn = ( splitpath($file) )[2];
	400	next
	401	if ( ( $fn eq 'ABOUT' )
	402	\|\| ( $fn eq 'ChangeLog.0' )
	403	\|\| ( $fn eq 'test-harness.txt' )
	404	# Debian issue re licensing of RFCs
	405	\|\| ( $fn =~ /^draft-ietf-.*/ )
	406	\|\| ( $fn =~ /^rfc.*/ )
	407	);
	408	move( $file, catfile( $new_docdir, $fn ) );
	409	}
	410	}
	411
	412	sub create_tar_files {
	413	my $context = shift;
	414
	415	my $pkg_tars = $context->{d}{pkg_tars};
	416	my $pkg_trees = $context->{d}{pkg_trees};
	417	my $tar = $context->{tar_cmd};
	418	if ($verbose) {
	419	foreach my $c (keys %{ $context->{compressors} }) {
	420	print "Compression: $c\t$context->{compressors}{$c}\n";
	421	}
	422	}
	423
	424	# We ideally do not want local system user information in release tarballs;
	425	# those are artifacts of use of tar for backups and have no place in
	426	# software release packaging; if someone extracts as root, then they should
	427	# get sane file ownerships.
	428	my @ownership = (
	429	'--owner' => $context->{tar_perms}{user},
	430	'--group' => $context->{tar_perms}{group},
	431	# on this GNU tar, --numeric-owner works during creation too
	432	'--numeric-owner'
	433	) if qx/tar --help 2>&1/ =~ /^\s*--owner=/m;
	434
	435	# See also environment variables set in main, tuning compression levels
	436	my %COMPRESSION = (
	437	gzip => { extension => 'gz', flags => '--gzip' },
	438	bzip2 => { extension => 'bz2', flags => '--bzip2' },
	439	lzip => { extension => 'lz', flags => '--lzip' },
	440	xz => { extension => 'xz', flags => '--xz' },
	441	);
	442
	443
	444	my (%size, %sha256);
	445
	446	foreach my $dir ( glob( catdir( $pkg_trees, ( 'exim*-' . $context->{v}{release} ) ) ) ) {
	447	my $dirname = ( splitdir($dir) )[-1];
	448	foreach my $comp (keys %COMPRESSION) {
	449	next unless $context->{compressors}{$comp};
	450
	451	my $basename = "$dirname.tar.$COMPRESSION{$comp}{extension}";
	452	my $outfile = catfile $pkg_tars, $basename;
	453
	454	print "Creating: $outfile\n" if $verbose \|\| $debug;
	455	0 == system($tar,
	456	cf => $outfile,
	457	$COMPRESSION{$comp}{flags},
	458	@ownership, -C => $pkg_trees, $dirname)
	459	or exit $? >> 8;
	460
	461	# calculate size and md5sum
	462	$size{$basename} = -s $outfile;
	463	$sha256{$basename} = do {
	464	my $sha = Digest::SHA->new(256);
	465	$sha->addfile($outfile);
	466	$sha->hexdigest;
	467	};
	468	}
	469	}
	470
	471	# write the sizes file
	472	open my $sizes, '>', $_ = catfile $pkg_tars, 'sizes.txt'
	473	or die "$ME: Can't open `$_': $!\n";
	474
	475	print $sizes join "\n",
	476	(map { "SIZE($_) = $size{$_}" } sort keys %size),
	477	(map { "SHA256($_) = $sha256{$_}" } sort keys %sha256);
	478
	479	close($sizes) or die "$ME: Can't close $_: $!\n";
	480	}
	481
	482	sub do_cleanup {
	483	my $context = shift;
	484
	485	print "Cleaning up\n" if $verbose;
	486	remove_tree $context->{d}{tmp}, { verbose => $debug + $verbose };
	487	}
	488
	489	}
	490
	491	MAIN: {
	492
	493	# some of these settings are useful only if we're in the
	494	# exim-projekt-root, but the check, if we're, is deferred
	495	my $context = Context->new(
	496	pkgname => 'exim',
	497	website_base => abs_path('../exim-website'),
	498	tar_cmd => 'tar',
	499	tar_perms => {
	500	user => '0',
	501	group => '0',
	502	},
	503	make_cmd => 'make',
	504	compressors => {
	505	gzip => 1,
	506	bzip2 => 1,
	507	xz => 1,
	508	lzip => 0,
	509	},
	510	docs => 1,
	511	web => 1,
	512	delete => 0,
	513	cleanup => 1,
	514	gpg => {
	515	sign => 1,
	516	key => undef,
	517	},
	518	);
	519
	520	##$ENV{'PATH'} = '/opt/local/bin:' . $ENV{'PATH'};
	521	# We are creating files for mass distribution, so work harder to make smaller files.
	522	$ENV{GZIP} = -9;
	523	$ENV{BZIP2} = -9;
	524	# xz documents minimum file sizes for levels higher than -6 to be useful and each
	525	# requires more RAM on the decompressing system. Exim tarball currently 24MiB so
	526	# using -8.
	527	$ENV{XZ_DEFAULTS} = -8;
	528
	529	GetOptions(
	530	$context,
	531	qw(workspace\|tmp=s outdir=s website_base\|webgen_base=s tar_cmd=s make_cmd=s docs\|build-docs! web\|build-web!
	532	delete! cleanup!),
	533	'sign!' => \$context->{gpg}{sign},
	534	'key=s' => \$context->{gpg}{key},
	535	'lzip!' => \$context->{compressors}{lzip},
	536	'verbose!' => \$verbose,
	537	'debug!' => \$debug,
	538	'help\|?' => sub { pod2usage(-verbose => 1, -exit => 0) },
	539	'man!' => sub { pod2usage(-verbose => 2, -exit => 0, -noperldoc => system('perldoc -V >/dev/null 2>&1')) },
	540	) and @ARGV == 2 or pod2usage;
	541
	542	-f '.exim-project-root'
	543	or die "$ME: please call this script from the root of the Exim project sources\n";
	544
	545	$context->{OUTDIR} = pop @ARGV;
	546
	547	if ($context->{gpg}{sign}) {
	548	$context->{gpg}{key} //= do { chomp($_ = qx/git config user.signingkey/); $_ }
	549	\|\| $ENV{EXIM_KEY}
	550	\|\| do {
	551	warn "$ME: No GPG key, using default\n";
	552	undef;
	553	}
	554	}
	555
	556
	557	warn "$ME: changed umask to 022\n" if umask(022) != 022;
	558
	559	$context->check_version(shift);
	560	$context->override_tar_cmd;
	561	$context->prepare_working_directory;
	562	$context->export_git_tree;
	563	$context->unpack_tree;
	564	$context->make_version_script;
	565	$context->build_documentation if $context->{docs};
	566	$context->build_html_documentation if $context->{docs} && $context->{web};
	567
	568	$context->build_src_package_directory;
	569	$context->build_doc_packages_directory if $context->{docs};
	570
	571	$context->create_tar_files;
	572	$context->sign if $context->{gpg}{sign};
	573	$context->move_to_outdir;
	574	$context->do_cleanup if $context->{cleanup};
	575
	576	### $context
	577	}
	578
	579	1;
	580
	581	__END__
	582
	583	=head1 NAME
	584
	585	mk_exim_release - Build an exim release
	586
	587	=head1 SYNOPSIS
	588
	589	mk_exim_release [options] version PKG-DIRECTORY
	590
	591	=head1 DESCRIPTION
	592
	593	B<mk_exim_release> builds an exim release.
	594
	595	Starting in a populated git repo that has already been tagged for
	596	release it builds docs, packages etc. Parameter is the version number
	597	to build as - ie 4.72 4.72-RC1, 4.86.1, etc, without any prefix.
	598
	599	This scripts expects to find a tag "exim-<version>".
	600
	601	After creating the release files, they should be signed. There is another
	602	helper for creating the signatures:
	603	F<release-process/scripts/sign_exim_packages>.
	604
	605	Call B<mk_exim_release> about like this:
	606
	607	release-process/scripts/mk_exim_release 4.99 OUT-DIR
	608
	609
	610	=head1 OPTIONS
	611
	612	=over 4
	613
	614	=item B<--[no]cleanup>
	615
	616	Do (or do not) cleanup the tmp directory at exit (default: do cleanup)
	617
	618	=item B<--[no]debug>
	619
	620	Forces debug mode. (default: no debug info)
	621
	622	=item B<--[no]delete>
	623
	624	Delete a pre-existing tmp- and package-directory at start. (default: don't delete)
	625
	626	=item B<--tmpdir> I<dir>
	627
	628	Change the name of the tmp directory (default: temporary directory)
	629
	630	=item B<--[no]help>
	631
	632	Display short help and exit cleanly. (default: don't do that)
	633
	634	=item B<--[no]lzip>
	635
	636	Control the creation of B<lzip> tarballs. (default: do not use lzip)
	637
	638	=item B<--make> I<cmd>
	639
	640	Force the use of a specific C<make> command. This may be necessary if C<make> is not
	641	C<gmake> (default: C<make>)
	642
	643	=item B<--[no]man>
	644
	645	Display man page and exit cleanly. (default: don't do that)
	646
	647	=item B<--tar> I<cmd>
	648
	649	Use to override the path to the C<tar> command. Need GNU tar in case
	650	I<lzip> is selected. (default: C<gtar>, if not found, use C<tar>)
	651
	652	=item B<--[no]sign>
	653
	654	Sign the created archive files (and the sizes.txt). (default: sign)
	655
	656	=item B<--key> I<GPG key>
	657
	658	Use this GPG key for signing. If nothing is specified the first one of this list
	659	is used:
	660
	661	=over 8
	662
	663	=item - git config user.signingkey
	664
	665	=item - environment C<EXIM_KEY>
	666
	667	=item - default GPG key
	668
	669	=back
	670
	671	=item B<--[no]web>
	672
	673	Control the creation of the website. For creation of the website, the F<../exim-website>
	674	(but see the B<website-base> option) directory must exist. (default: create the website)
	675
	676	=item B<--website-base> I<dir>
	677
	678	Base directory for the web site generation (default: F<../exim-website>)
	679
	680	=item B<--verbose>
	681
	682	Force verbose mode. (default: no verbosity)
	683
	684	=back
	685
	686	=head1 AUTHOR
	687
	688	Nigel Metheringham <Nigel.Metheringham@dev.intechnology.co.uk>,
	689	some changes by Heiko Schlittermann <hs@schlittermann.de>
	690
	691	=head1 COPYRIGHT
	692
	693	Copyright 2010-2016 Exim Maintainers. All rights reserved.
	694
	695	=cut
	696	# vim: set sw=4 et :