mk_exim_release: more perlish
[exim.git] / release-process / scripts / mk_exim_release
1 #!/usr/bin/env perl
2 # Copyright (c) The Exim Maintainers 2016-2018
3
4 use strict;
5 use warnings;
6 use Carp;
7 use Cwd qw'abs_path';
8 use File::Basename;
9 use File::Path qw(make_path remove_tree);
10 use File::Temp;
11 use Getopt::Long;
12 use IO::File;
13 use Pod::Usage;
14 use Digest::SHA;
15 use if $ENV{DEBUG} => 'Smart::Comments';
16
17 my $ME = basename $0;
18
19
20 my $debug = 0;
21 my $verbose = 0;
22
23 # MAJOR.MINOR[.SECURITY[.FIXES]][-RCX]
24 # 4 .90 .0 .22 -RC1
25 my $version_pattern = qr/
26 (?<release>
27 (?<major>\d) # 4
28 \.(?<minor>\d\d) # .90
29 (?:\.(?<security>\d+) # .0
30 (?:\.(?<fixes>)\d+)?)? # .22
31 (?:-(?<rc>RC\d+)?)? # -RC1
32 )
33 /x;
34
35 # ------------------------------------------------------------------
36
37 package Context {
38 use strict; # not strictly necessary yet, until in an own package
39 use warnings; # not strictly necessary yet, ...
40 use File::Spec::Functions qw'splitpath catfile catdir splitdir';
41 use File::Path qw'make_path remove_tree';
42 use File::Copy;
43 use Cwd qw'abs_path';
44 use Carp;
45
46 package PWD {
47 use Cwd;
48 sub TIESCALAR { bless do {\my $x} }
49 sub FETCH { cwd }
50 }
51
52 tie my $cwd => 'PWD' or die;
53
54 sub new {
55 my $class = shift;
56 return bless { @_ } => $class;
57 }
58
59 sub check_version {
60 my $context = shift;
61 my $version = shift;
62
63 croak "The given version number does not look right - $version"
64 if not $version =~ /$version_pattern/;
65
66 $context->{v}{release} = $+{release};
67 $context->{v}{major} = $+{major};
68 $context->{v}{minor} = $+{minor};
69 $context->{v}{security} = $+{security};
70 $context->{v}{rc} = $+{rc};
71
72 die "$ME: This script doesn't work for versions prior 4.92-RCx. "
73 ."Please checkout an older version.\n"
74 if $context->{v}{major} < 4
75 or $context->{v}{major} == 4 && $context->{v}{minor} < 92;
76
77 ### v: $context->{v}
78
79 # find a valid vcs tag matching the version
80 my $pattern = "$context->{pkgname}-$context->{v}{release}" =~ s/[-_.]/[-_.]/gr;
81 chomp(my @tags = qx{git tag --list '$pattern'});
82
83 croak "The given version is ambigous, tags: @tags\n" if @tags > 1;
84 croak "The given version does not exist (no such tag: exim-$version)\n" if @tags == 0;
85
86 $context->{git_tag} = $tags[0];
87
88 # target_release: the release we aim to reach with release candidates
89 # FIXME: re-construct from the parsed version number
90 ($context->{v}{target_release} = $context->{v}{release}) =~ s/-RC\d+//;
91 }
92
93
94 # We prefer gtar to tar if gtar exists in $PATH
95 sub override_tar_cmd {
96 my $context = shift;
97 my $tar = $context->{tar_cmd};
98
99 return unless $tar eq 'tar';
100
101 foreach my $d (File::Spec->path()) {
102 my $p = catfile($d, 'gtar');
103 if (-x $p) {
104 $context->{tar_cmd} = $p;
105 print "Switched tar command to: $p\n" if $verbose;
106 return;
107 }
108 }
109 }
110
111 sub prepare_working_directory {
112 my $context = shift;
113 my $workspace = $context->{workspace};
114
115 if (not defined $workspace) {
116 $workspace = $context->{workspace} = File::Temp->newdir(File::Spec->tmpdir . '/exim-packaging-XXXX');
117 }
118 else {
119 # ensure the working directory is not in place
120 if (-e $workspace) {
121 if ($context->{delete}) {
122 print "Deleting existing $workspace\n" if $verbose;
123 remove_tree $workspace, { verbose => $verbose || $debug };
124 }
125 else {
126 croak "Working directory $workspace exists" if -e $workspace;
127 }
128 }
129
130 # create base directory
131 make_path( $context->{directory}, { verbose => $verbose || $debug } );
132 }
133
134 # Set(!) and create subdirectories
135 foreach (qw(vcs_export pkg_tars pkg_trees tmp)) { # {dookbook}
136 make_path(
137 $context->{d}{$_} = catdir($workspace, $_),
138 { verbose => $verbose || $debug });
139 }
140 }
141
142 sub export_git_tree {
143 my $context = shift;
144
145 # build git command
146 my $archive_file = $context->{tmp_archive_file} = sprintf'%s/%s-%s.tar', $context->{d}{tmp}, $context->{pkgname}, $context->{v}{release};
147 ### $archive_file
148 my @cmd = ( 'git', 'archive', '--format=tar', "--output=$archive_file", $context->{git_tag} );
149 ### @cmd
150 # run git command
151 print "[$cwd] Running: @cmd\n" if $verbose;
152 0 == system @cmd or croak "Export failed";
153 }
154
155 sub unpack_tree {
156 # TODO: Why can't we combine the export_git_tree with the
157 # unpack_tree function?
158 my $context = shift;
159
160 ### $context
161 die "Cannot see archive file\n" unless -f $context->{tmp_archive_file};
162 my @cmd = ('tar',
163 xf => $context->{tmp_archive_file},
164 -C => $context->{d}{vcs_export} );
165
166 # run command
167 print "[$cwd] Running: @cmd\n" if $verbose;
168 system @cmd and croak "Unpack failed\n";
169
170 }
171
172 sub make_version_script {
173 my $context = shift;
174
175 #my $variant = substr( $context->{v}{release}, length($context->{v}{target_release}) );
176 #if ( $context->{v}{release} ne $context->{v}{target_release} . $variant ) {
177 # die "Broken version numbering, I'm buggy";
178 #}
179
180
181 # Work
182 if (not my $pid = fork // die "$ME: Cannot fork: $!\n") {
183
184 my $source_tree = catdir($context->{d}{vcs_export}, 'src', 'src');
185 ### $source_tree
186
187 chdir $source_tree or die "chdir $source_tree: $!\n";
188
189
190 croak "WARNING: version.sh already exists - leaving it in place\n"
191 if -f 'version.sh';
192
193 # Currently (25. Feb. 2016) the mk_exim_release.pl up to now can't
194 # deal with security releases.!? So we need a current
195 # mk_exim_release.pl. But if we use a current (master), the
196 # reversion script returns wrong version info (it's running inside
197 # the Git tree and uses git --describe, which always returns the
198 # current version of master.) I do not want to change the old
199 # reversion scripts (in 4.86.1, 4.85.1).
200 #
201 # Thus we've to provide the version.sh, based on the info we have
202 # about the release. If reversion finds this, it doesn't try to find
203 # it's own way to get a valid version number from the git.
204 #
205 # 4.89 series: the logic here did not handle _RC<N> thus breaking RC
206 # status in versions. nb: rc in context should be same as $variant
207 # in local context.
208
209 #my $stamp = $context->{minor} ? '_'.$context->{minor} : '';
210 #$stamp .= $context->{rc} if $context->{rc};
211 my $variant = $context->{v}{rc} // '';
212
213 print "[$cwd] create version.sh\n" if $verbose;
214 open(my $v, '>', 'version.sh') or die "Can't open version.sh for writing: $!\n";
215 print {$v} <<__;
216 # initial version automatically generated by $0
217 EXIM_RELEASE_VERSION=$context->{v}{target_release}
218 EXIM_VARIANT_VERSION=$variant
219 EXIM_COMPILE_NUMBER=0
220 # echo "[[[ \$EXIM_RELEASE_VERSION | \$EXIM_VARIANT_VERSION | \$EXIM_COMPILE_NUMBER ]]]"
221 __
222 close $v or die "$0: Can not close $source_tree/version.h: $!\n";
223 unlink 'version.h' or die "$ME: Can not unlink $source_tree/version.h: $!\n"
224 if -f 'version.h';
225
226 # Later, if we get the reversion script fixed, we can call it again.
227 # For now (25. Feb. 2016) we'll leave it unused.
228 #my @cmd = ('../scripts/reversion', 'release', $context->{git_tag});
229
230 my @cmd = ('../scripts/reversion', 'release');
231 print "[$cwd] Running: @cmd\n" if $verbose;
232 system(@cmd) and croak "reversion failed";
233
234 die "$ME: failed to create version.sh"
235 unless -f 'version.sh';
236
237 exit 0;
238 }
239 else {
240 $pid == waitpid($pid, 0) or die "$0: waidpid: $!\n";
241 exit $? >> 8 if $?;
242 }
243 }
244
245 sub build_documentation {
246 my $context = shift;
247 my $docdir = catdir $context->{d}{vcs_export}, 'doc', 'doc-docbook';
248
249 # documentation building does a chdir, so we'll do it in a
250 # subprocess
251 if (not my $pid = fork // die "$ME: Can't fork: $!\n") {
252 chdir $docdir or die "$ME: Can't chdir to $docdir: $!\n";
253 system('./OS-Fixups') == 0 or exit $?;
254 exec $context->{make_cmd},
255 "EXIM_VER=$context->{v}{target_release}" .
256 ($context->{v}{rc} ? "-$context->{v}{rc}" : ''),
257 'everything';
258 die "$ME: [$cwd] Cannot exec $context->{make_cmd}: $!\n";
259 }
260 else {
261 waitpid($pid, 0);
262 exit $? >> 8 if $?;
263 }
264
265 $context->copy_docbook_files;
266 $context->build_html_documentation if $context->{web};
267 }
268
269 sub copy_docbook_files {
270 my $context = shift;
271
272 # where the generated docbook files can be found
273 my $docdir = catdir $context->{d}{vcs_export}, 'doc', 'doc-docbook';
274
275 foreach ('spec.xml', 'filter.xml') {
276 my $from = catfile $docdir, $_;
277 my $to = catdir $context->{d}{tmp}; # {dookbook}
278 copy $from => $to or die $@;
279 }
280 }
281
282 sub build_html_documentation {
283 my $context = shift;
284
285 # where the website docbook source dir is - push the generated
286 # files there
287 {
288 my $webdir = catdir $context->{website_base}, 'docbook', $context->{v}{target_release};
289 make_path $webdir, { verbose => $verbose + $debug };
290 copy catfile($context->{d}{vcs_export}, 'doc', 'doc-docbook', $_)
291 => $webdir or die $@
292 for 'spec.xml', 'filter.xml';
293 }
294
295 my $gen = catfile $context->{website_base}, 'script/gen';
296 my $outdir = catdir $context->{d}{pkg_trees}, "exim-html-$context->{v}{release}";
297
298 make_path $outdir, { verbose => $verbose || $debug };
299
300 my @cmd = (
301 $gen,
302 '--spec' => catfile($context->{d}{tmp}, 'spec.xml'), # {dookbook}
303 '--filter' => catfile($context->{d}{tmp}, 'filter.xml'), # {dookbok}
304 '--latest' => $context->{v}{target_release},
305 '--docroot' => $outdir,
306 '--localstatic',
307 ($verbose || $debug ? '--verbose' : ()),
308 );
309
310 print "[$cwd] Executing @cmd\n";
311 0 == system @cmd or exit $? >> 8;
312
313 }
314
315 sub sign {
316 my $context = shift;
317 foreach my $tar (glob "$context->{d}{pkg_tars}/*") {
318 system gpg =>
319 '--quiet', '--batch',
320 defined $context->{gpg}{key}
321 ? ('--local-user' => $context->{gpg}{key})
322 : (),
323 '--detach-sig', '--armor', $tar;
324 }
325 }
326
327 sub move_to_outdir {
328 my $context = shift;
329 make_path $context->{OUTDIR}, { verbose => $verbose || $debug };
330 move $_ => $context->{OUTDIR} or die $@
331 for glob "$context->{d}{pkg_tars}/*";
332 }
333
334 sub build_src_package_directory {
335 my $context = shift;
336
337 # build the exim package directory path
338 $context->{d}{src} = catdir $context->{d}{pkg_trees}, "exim-$context->{v}{release}";
339
340 # initially we move the exim-src directory to the new directory name
341 move
342 catdir( $context->{d}{vcs_export}, 'src')
343 => $context->{d}{src}
344 or croak "Move of src dir failed - $!";
345
346 # add Local subdirectory
347 make_path( catdir( $context->{d}{src}, 'Local' ), { verbose => $verbose || $debug } );
348
349 # now add the text docs
350 $context->move_text_docs_into_pkg;
351 }
352
353 sub build_doc_packages_directory {
354 my $context = shift;
355
356 ##foreach my $format (qw/pdf postscript texinfo info/) {
357 foreach my $format (qw/pdf postscript/) {
358 my $target = catdir $context->{d}{pkg_trees}, "exim-$format-$context->{v}{release}", 'doc';
359 make_path( $target, { verbose => $verbose || $debug } );
360
361 # move documents across
362 foreach my $file (
363 glob(
364 catfile(
365 $context->{d}{vcs_export},
366 'doc',
367 'doc-docbook',
368 (
369 ( $format eq 'postscript' )
370 ? '*.ps'
371 : ( '*.' . $format )
372 )
373 )
374 )
375 )
376 {
377 move( $file, catfile( $target, ( splitpath($file) )[2] ) );
378 }
379 }
380 }
381
382 sub move_text_docs_into_pkg {
383 my $context = shift;
384
385 my $old_docdir = catdir( $context->{d}{vcs_export}, 'doc', 'doc-docbook' );
386 my $old_txtdir = catdir( $context->{d}{vcs_export}, 'doc', 'doc-txt' );
387 my $new_docdir = catdir( $context->{d}{src}, 'doc' );
388 make_path( $new_docdir, { verbose => $verbose || $debug } );
389
390 # move generated documents from docbook stuff
391 foreach my $file (qw/exim.8 spec.txt filter.txt/) {
392 die "Empty file \"$file\"\n" if -z catfile( $old_docdir, $file );
393 move( catfile( $old_docdir, $file ), catfile( $new_docdir, $file ) );
394 }
395
396 # move text documents across
397 foreach my $file ( glob( catfile( $old_txtdir, '*' ) ) ) {
398
399 # skip a few we dont want
400 my $fn = ( splitpath($file) )[2];
401 next
402 if ( ( $fn eq 'ABOUT' )
403 || ( $fn eq 'ChangeLog.0' )
404 || ( $fn eq 'test-harness.txt' )
405 # Debian issue re licensing of RFCs
406 || ( $fn =~ /^draft-ietf-.*/ )
407 || ( $fn =~ /^rfc.*/ )
408 );
409 move( $file, catfile( $new_docdir, $fn ) );
410 }
411 }
412
413 sub create_tar_files {
414 my $context = shift;
415
416 my $pkg_tars = $context->{d}{pkg_tars};
417 my $pkg_trees = $context->{d}{pkg_trees};
418 my $tar = $context->{tar_cmd};
419 if ($verbose) {
420 foreach my $c (keys %{ $context->{compressors} }) {
421 print "Compression: $c\t$context->{compressors}{$c}\n";
422 }
423 }
424
425 # We ideally do not want local system user information in release tarballs;
426 # those are artifacts of use of tar for backups and have no place in
427 # software release packaging; if someone extracts as root, then they should
428 # get sane file ownerships.
429 my @ownership = (
430 '--owner' => $context->{tar_perms}{user},
431 '--group' => $context->{tar_perms}{group},
432 # on this GNU tar, --numeric-owner works during creation too
433 '--numeric-owner'
434 ) if qx/tar --help 2>&1/ =~ /^\s*--owner=/m;
435
436 # See also environment variables set in main, tuning compression levels
437 my %COMPRESSION = (
438 gzip => { extension => 'gz', flags => '--gzip' },
439 bzip2 => { extension => 'bz2', flags => '--bzip2' },
440 lzip => { extension => 'lz', flags => '--lzip' },
441 xz => { extension => 'xz', flags => '--xz' },
442 );
443
444
445 my (%size, %sha256);
446
447 foreach my $dir ( glob( catdir( $pkg_trees, ( 'exim*-' . $context->{v}{release} ) ) ) ) {
448 my $dirname = ( splitdir($dir) )[-1];
449 foreach my $comp (keys %COMPRESSION) {
450 next unless $context->{compressors}{$comp};
451
452 my $basename = "$dirname.tar.$COMPRESSION{$comp}{extension}";
453 my $outfile = catfile $pkg_tars, $basename;
454
455 print "Creating: $outfile\n" if $verbose || $debug;
456 0 == system($tar,
457 cf => $outfile,
458 $COMPRESSION{$comp}{flags},
459 @ownership, -C => $pkg_trees, $dirname)
460 or exit $? >> 8;
461
462 # calculate size and md5sum
463 $size{$basename} = -s $outfile;
464 $sha256{$basename} = do {
465 my $sha = Digest::SHA->new(256);
466 $sha->addfile($outfile);
467 $sha->hexdigest;
468 };
469 }
470 }
471
472 # write the sizes file
473 open my $sizes, '>', $_ = catfile $pkg_tars, 'sizes.txt'
474 or die "$ME: Can't open `$_': $!\n";
475
476 print $sizes join "\n",
477 (map { "SIZE($_) = $size{$_}" } sort keys %size),
478 (map { "SHA256($_) = $sha256{$_}" } sort keys %sha256);
479
480 close($sizes) or die "$ME: Can't close $_: $!\n";
481 }
482
483 sub do_cleanup {
484 my $context = shift;
485
486 print "Cleaning up\n" if $verbose;
487 remove_tree $context->{d}{tmp}, { verbose => $debug + $verbose };
488 }
489
490 }
491
492 MAIN: {
493
494 # some of these settings are useful only if we're in the
495 # exim-projekt-root, but the check, if we're, is deferred
496 my $context = Context->new(
497 pkgname => 'exim',
498 website_base => abs_path('../exim-website'),
499 tar_cmd => 'tar',
500 tar_perms => {
501 user => '0',
502 group => '0',
503 },
504 make_cmd => 'make',
505 compressors => {
506 gzip => 1,
507 bzip2 => 1,
508 xz => 1,
509 lzip => 0,
510 },
511 docs => 1,
512 web => 1,
513 delete => 0,
514 cleanup => 1,
515 gpg => {
516 sign => 1,
517 key => undef,
518 },
519 );
520
521 ##$ENV{'PATH'} = '/opt/local/bin:' . $ENV{'PATH'};
522 # We are creating files for mass distribution, so work harder to make smaller files.
523 $ENV{GZIP} = -9;
524 $ENV{BZIP2} = -9;
525 # xz documents minimum file sizes for levels higher than -6 to be useful and each
526 # requires more RAM on the decompressing system. Exim tarball currently 24MiB so
527 # using -8.
528 $ENV{XZ_DEFAULTS} = -8;
529
530 GetOptions(
531 $context,
532 qw(workspace|tmp=s outdir=s website_base|webgen_base=s tar_cmd=s make_cmd=s docs|build-docs! web|build-web!
533 delete! cleanup!),
534 'sign!' => \$context->{gpg}{sign},
535 'key=s' => \$context->{gpg}{key},
536 'lzip!' => \$context->{compressors}{lzip},
537 'verbose!' => \$verbose,
538 'debug!' => \$debug,
539 'help|?' => sub { pod2usage(-verbose => 1, -exit => 0) },
540 'man!' => sub { pod2usage(-verbose => 2, -exit => 0, -noperldoc => system('perldoc -V >/dev/null 2>&1')) },
541 ) and @ARGV == 2 or pod2usage;
542
543 -f '.exim-project-root'
544 or die "$ME: please call this script from the root of the Exim project sources\n";
545
546 $context->{OUTDIR} = pop @ARGV;
547
548 if ($context->{gpg}{sign}) {
549 $context->{gpg}{key} //= do { chomp($_ = qx/git config user.signingkey/); $_ }
550 || $ENV{EXIM_KEY}
551 || do {
552 warn "$ME: No GPG key, using default\n";
553 undef;
554 }
555 }
556
557
558 warn "$ME: changed umask to 022\n" if umask(022) != 022;
559
560 $context->check_version(shift);
561 $context->override_tar_cmd;
562 $context->prepare_working_directory;
563 $context->export_git_tree;
564 $context->unpack_tree;
565 $context->make_version_script;
566 $context->build_documentation if $context->{docs};
567
568 $context->build_src_package_directory;
569 $context->build_doc_packages_directory if $context->{docs};
570
571 $context->create_tar_files;
572 $context->sign if $context->{gpg}{sign};
573 $context->move_to_outdir;
574 $context->do_cleanup if $context->{cleanup};
575
576 ### $context
577 }
578
579 1;
580
581 __END__
582
583 =head1 NAME
584
585 mk_exim_release - Build an exim release
586
587 =head1 SYNOPSIS
588
589 mk_exim_release [options] version PKG-DIRECTORY
590
591 =head1 DESCRIPTION
592
593 B<mk_exim_release> builds an exim release.
594
595 Starting in a populated git repo that has already been tagged for
596 release it builds docs, packages etc. Parameter is the version number
597 to build as - ie 4.72 4.72-RC1, 4.86.1, etc, without any prefix.
598
599 This scripts expects to find a tag "exim-<version>".
600
601 After creating the release files, they should be signed. There is another
602 helper for creating the signatures:
603 F<release-process/scripts/sign_exim_packages>.
604
605 Call B<mk_exim_release> about like this:
606
607 release-process/scripts/mk_exim_release 4.99 OUT-DIR
608
609
610 =head1 OPTIONS
611
612 =over 4
613
614 =item B<--[no]cleanup>
615
616 Do (or do not) cleanup the tmp directory at exit (default: do cleanup)
617
618 =item B<--[no]debug>
619
620 Forces debug mode. (default: no debug info)
621
622 =item B<--[no]delete>
623
624 Delete a pre-existing tmp- and package-directory at start. (default: don't delete)
625
626 =item B<--tmpdir> I<dir>
627
628 Change the name of the tmp directory (default: temporary directory)
629
630 =item B<--[no]help>
631
632 Display short help and exit cleanly. (default: don't do that)
633
634 =item B<--[no]lzip>
635
636 Control the creation of B<lzip> tarballs. (default: do not use lzip)
637
638 =item B<--make> I<cmd>
639
640 Force the use of a specific C<make> command. This may be necessary if C<make> is not
641 C<gmake> (default: C<make>)
642
643 =item B<--[no]man>
644
645 Display man page and exit cleanly. (default: don't do that)
646
647 =item B<--tar> I<cmd>
648
649 Use to override the path to the C<tar> command. Need GNU tar in case
650 I<lzip> is selected. (default: C<gtar>, if not found, use C<tar>)
651
652 =item B<--[no]sign>
653
654 Sign the created archive files (and the sizes.txt). (default: sign)
655
656 =item B<--key> I<GPG key>
657
658 Use this GPG key for signing. If nothing is specified the first one of this list
659 is used:
660
661 =over 8
662
663 =item - git config user.signingkey
664
665 =item - environment C<EXIM_KEY>
666
667 =item - default GPG key
668
669 =back
670
671 =item B<--[no]web>
672
673 Control the creation of the website. For creation of the website, the F<../exim-website>
674 (but see the B<website-base> option) directory must exist. (default: create the website)
675
676 =item B<--website-base> I<dir>
677
678 Base directory for the web site generation (default: F<../exim-website>)
679
680 =item B<--verbose>
681
682 Force verbose mode. (default: no verbosity)
683
684 =back
685
686 =head1 AUTHOR
687
688 Nigel Metheringham <Nigel.Metheringham@dev.intechnology.co.uk>,
689 some changes by Heiko Schlittermann <hs@schlittermann.de>
690
691 =head1 COPYRIGHT
692
693 Copyright 2010-2016 Exim Maintainers. All rights reserved.
694
695 =cut
696 # vim: set sw=4 et :