5bda34c69f2ac317afbb78ad68b225383886ffa4
[exim.git] / release-process / scripts / mk_exim_release
1 #!/usr/bin/env perl
2 # Copyright (c) The Exim Maintainers 2016-2018
3
4 use strict;
5 use warnings;
6 use Carp;
7 use Cwd qw'abs_path';
8 use File::Basename;
9 use File::Path qw(make_path remove_tree);
10 use File::Temp;
11 use Getopt::Long;
12 use IO::File;
13 use Pod::Usage;
14 use if $ENV{DEBUG} => 'Smart::Comments';
15
16 my $ME = basename $0;
17
18
19 my $debug = 0;
20 my $verbose = 0;
21
22 # MAJOR.MINOR[.SECURITY[.FIXES]][-RCX]
23 # 4 .90 .0 .22 -RC1
24 my $version_pattern = qr/
25 (?<release>
26 (?<major>\d) # 4
27 \.(?<minor>\d\d) # .90
28 (?:\.(?<security>\d+) # .0
29 (?:\.(?<fixes>)\d+)?)? # .22
30 (?:-(?<rc>RC\d+)?)? # -RC1
31 )
32 /x;
33
34 # ------------------------------------------------------------------
35
36 package Context {
37 use strict; # not strictly necessary yet, until in an own package
38 use warnings; # not strictly necessary yet, ...
39 use File::Spec::Functions qw'splitpath catfile catdir splitdir';
40 use File::Path qw'make_path remove_tree';
41 use File::Copy;
42 use Cwd qw'abs_path';
43 use Carp;
44
45 package PWD {
46 use Cwd;
47 sub TIESCALAR { bless do {\my $x} }
48 sub FETCH { cwd }
49 }
50
51 tie my $cwd => 'PWD' or die;
52
53 sub new {
54 my $class = shift;
55 return bless { @_ } => $class;
56 }
57
58 sub check_version {
59 my $context = shift;
60 my $version = shift;
61
62 croak "The given version number does not look right - $version"
63 if not $version =~ /$version_pattern/;
64
65 $context->{v}{release} = $+{release};
66 $context->{v}{major} = $+{major};
67 $context->{v}{minor} = $+{minor};
68 $context->{v}{security} = $+{security};
69 $context->{v}{rc} = $+{rc};
70
71 die "$ME: This script doesn't work for versions prior 4.92-RCx. Please checkout an older version.\n"
72 if $context->{v}{major} < 4
73 or $context->{v}{major} == 4 && $context->{v}{minor} < 92;
74
75 ### v: $context->{v}
76
77 # find a valid vcs tag matching the version
78 my $pattern = "$context->{pkgname}-$context->{v}{release}" =~ s/[-_.]/[-_.]/gr;
79 chomp(my @tags = qx{git tag --list '$pattern'});
80
81 croak "The given version is ambigous, tags: @tags\n" if @tags > 1;
82 croak "The given version does not exist (no such tag: exim-$version)\n" if @tags == 0;
83
84 $context->{git_tag} = $tags[0];
85
86 # target_release: the release we aim to reach with release candidates
87 # FIXME: re-construct from the parsed version number
88 ($context->{v}{target_release} = $context->{v}{release}) =~ s/-RC\d+//;
89 }
90
91
92 # We prefer gtar to tar if gtar exists in $PATH
93 sub override_tar_cmd {
94 my $context = shift;
95 my $tar = $context->{tar_cmd};
96
97 return unless $tar eq 'tar';
98
99 foreach my $d (File::Spec->path()) {
100 my $p = catfile($d, 'gtar');
101 if (-x $p) {
102 $context->{tar_cmd} = $p;
103 print "Switched tar command to: $p\n" if $verbose;
104 return;
105 }
106 }
107 }
108
109 sub prepare_working_directory {
110 my $context = shift;
111 my $workspace = $context->{workspace};
112
113 if (not defined $workspace) {
114 $workspace = $context->{workspace} = File::Temp->newdir(File::Spec->tmpdir . '/exim-packaging-XXXX');
115 }
116 else {
117 # ensure the working directory is not in place
118 if (-e $workspace) {
119 if ($context->{delete}) {
120 print "Deleting existing $workspace\n" if $verbose;
121 remove_tree $workspace, { verbose => $verbose || $debug };
122 }
123 else {
124 croak "Working directory $workspace exists" if -e $workspace;
125 }
126 }
127
128 # create base directory
129 make_path( $context->{directory}, { verbose => $verbose || $debug } );
130 }
131
132 # Set(!) and create subdirectories
133 foreach (qw(vcs_export pkg_tars pkg_trees tmp)) { # {dookbook}
134 make_path(
135 $context->{d}{$_} = catdir($workspace, $_),
136 { verbose => $verbose || $debug });
137 }
138 }
139
140 sub export_git_tree {
141 my $context = shift;
142
143 # build git command
144 my $archive_file = $context->{tmp_archive_file} = sprintf'%s/%s-%s.tar', $context->{d}{tmp}, $context->{pkgname}, $context->{v}{release};
145 ### $archive_file
146 my @cmd = ( 'git', 'archive', '--format=tar', "--output=$archive_file", $context->{git_tag} );
147 ### @cmd
148 # run git command
149 print "[$cwd] Running: @cmd\n" if $verbose;
150 0 == system @cmd or croak "Export failed";
151 }
152
153 sub unpack_tree {
154 # TODO: Why can't we combine the export_git_tree with the
155 # unpack_tree function?
156 my $context = shift;
157
158 ### $context
159 die "Cannot see archive file\n" unless -f $context->{tmp_archive_file};
160 my @cmd = ('tar',
161 xf => $context->{tmp_archive_file},
162 -C => $context->{d}{vcs_export} );
163
164 # run command
165 print "[$cwd] Running: @cmd\n" if $verbose;
166 system @cmd and croak "Unpack failed\n";
167
168 }
169
170 sub make_version_script {
171 my $context = shift;
172
173 #my $variant = substr( $context->{v}{release}, length($context->{v}{target_release}) );
174 #if ( $context->{v}{release} ne $context->{v}{target_release} . $variant ) {
175 # die "Broken version numbering, I'm buggy";
176 #}
177
178
179 # Work
180 if (not my $pid = fork // die "$ME: Cannot fork: $!\n") {
181
182 my $source_tree = catdir($context->{d}{vcs_export}, 'src', 'src');
183 ### $source_tree
184
185 chdir $source_tree or die "chdir $source_tree: $!\n";
186
187
188 croak "WARNING: version.sh already exists - leaving it in place\n"
189 if -f 'version.sh';
190
191 # Currently (25. Feb. 2016) the mk_exim_release.pl up to now can't
192 # deal with security releases.!? So we need a current
193 # mk_exim_release.pl. But if we use a current (master), the
194 # reversion script returns wrong version info (it's running inside
195 # the Git tree and uses git --describe, which always returns the
196 # current version of master.) I do not want to change the old
197 # reversion scripts (in 4.86.1, 4.85.1).
198 #
199 # Thus we've to provide the version.sh, based on the info we have
200 # about the release. If reversion finds this, it doesn't try to find
201 # it's own way to get a valid version number from the git.
202 #
203 # 4.89 series: the logic here did not handle _RC<N> thus breaking RC
204 # status in versions. nb: rc in context should be same as $variant
205 # in local context.
206
207 #my $stamp = $context->{minor} ? '_'.$context->{minor} : '';
208 #$stamp .= $context->{rc} if $context->{rc};
209 my $variant = $context->{v}{rc} // '';
210
211 print "[$cwd] create version.sh\n" if $verbose;
212 open(my $v, '>', 'version.sh') or die "Can't open version.sh for writing: $!\n";
213 print {$v} <<__;
214 # initial version automatically generated by $0
215 EXIM_RELEASE_VERSION=$context->{v}{target_release}
216 EXIM_VARIANT_VERSION=$variant
217 EXIM_COMPILE_NUMBER=0
218 # echo "[[[ \$EXIM_RELEASE_VERSION | \$EXIM_VARIANT_VERSION | \$EXIM_COMPILE_NUMBER ]]]"
219 __
220 close $v or die "$0: Can not close $source_tree/version.h: $!\n";
221 unlink 'version.h' or die "$ME: Can not unlink $source_tree/version.h: $!\n"
222 if -f 'version.h';
223
224 # Later, if we get the reversion script fixed, we can call it again.
225 # For now (25. Feb. 2016) we'll leave it unused.
226 #my @cmd = ('../scripts/reversion', 'release', $context->{git_tag});
227
228 my @cmd = ('../scripts/reversion', 'release');
229 print "[$cwd] Running: @cmd\n" if $verbose;
230 system(@cmd) and croak "reversion failed";
231
232 die "$ME: failed to create version.sh"
233 unless -f 'version.sh';
234
235 exit 0;
236 }
237 else {
238 $pid == waitpid($pid, 0) or die "$0: waidpid: $!\n";
239 exit $? >> 8 if $?;
240 }
241 }
242
243 sub build_documentation {
244 my $context = shift;
245 my $docdir = catdir $context->{d}{vcs_export}, 'doc', 'doc-docbook';
246
247 # documentation building does a chdir, so we'll do it in a
248 # subprocess
249 if (not my $pid = fork // die "$ME: Can't fork: $!\n") {
250 chdir $docdir or die "$ME: Can't chdir to $docdir: $!\n";
251 system('./OS-Fixups') == 0 or exit $?;
252 exec $context->{make_cmd},
253 "EXIM_VER=$context->{v}{target_release}" .
254 ($context->{v}{rc} ? "-$context->{v}{rc}" : ''),
255 'everything';
256 die "$ME: [$cwd] Cannot exec $context->{make_cmd}: $!\n";
257 }
258 else {
259 waitpid($pid, 0);
260 exit $? >> 8 if $?;
261 }
262
263 $context->copy_docbook_files;
264 $context->build_html_documentation if $context->{web};
265 }
266
267 sub copy_docbook_files {
268 my $context = shift;
269
270 # where the generated docbook files can be found
271 my $docdir = catdir $context->{d}{vcs_export}, 'doc', 'doc-docbook';
272
273 foreach ('spec.xml', 'filter.xml') {
274 my $from = catfile $docdir, $_;
275 my $to = catdir $context->{d}{tmp}; # {dookbook}
276 copy $from => $to or die $@;
277 }
278 }
279
280 sub build_html_documentation {
281 my $context = shift;
282
283 # where the website docbook source dir is - push the generated
284 # files there
285 {
286 my $webdir = catdir $context->{website_base}, 'docbook', $context->{v}{target_release};
287 make_path $webdir, { verbose => $verbose + $debug };
288 copy catfile($context->{d}{vcs_export}, 'doc', 'doc-docbook', $_)
289 => $webdir or die $@
290 for 'spec.xml', 'filter.xml';
291 }
292
293 my $gen = catfile $context->{website_base}, 'script/gen';
294 my $outdir = catdir $context->{d}{pkg_trees}, "exim-html-$context->{v}{release}";
295
296 make_path $outdir, { verbose => $verbose || $debug };
297
298 my @cmd = (
299 $gen,
300 '--spec' => catfile($context->{d}{tmp}, 'spec.xml'), # {dookbook}
301 '--filter' => catfile($context->{d}{tmp}, 'filter.xml'), # {dookbok}
302 '--latest' => $context->{v}{target_release},
303 '--docroot' => $outdir,
304 '--localstatic',
305 ($verbose || $debug ? '--verbose' : ()),
306 );
307
308 print "[$cwd] Executing @cmd\n";
309 0 == system @cmd or exit $? >> 8;
310
311 }
312
313 sub move_to_outdir {
314 my $context = shift;
315 make_path $context->{OUTDIR}, { verbose => $verbose || $debug };
316 move $_ => $context->{OUTDIR} or die $@
317 for glob "$context->{d}{pkg_tars}/*";
318 }
319
320 sub build_src_package_directory {
321 my $context = shift;
322
323 # build the exim package directory path
324 $context->{d}{src} = catdir $context->{d}{pkg_trees}, "exim-$context->{v}{release}";
325
326 # initially we move the exim-src directory to the new directory name
327 move
328 catdir( $context->{d}{vcs_export}, 'src')
329 => $context->{d}{src}
330 or croak "Move of src dir failed - $!";
331
332 # add Local subdirectory
333 make_path( catdir( $context->{d}{src}, 'Local' ), { verbose => $verbose || $debug } );
334
335 # now add the text docs
336 $context->move_text_docs_into_pkg;
337 }
338
339 sub build_doc_packages_directory {
340 my $context = shift;
341
342 ##foreach my $format (qw/pdf postscript texinfo info/) {
343 foreach my $format (qw/pdf postscript/) {
344 my $target = catdir $context->{d}{pkg_trees}, "exim-$format-$context->{v}{release}", 'doc';
345 make_path( $target, { verbose => $verbose || $debug } );
346
347 # move documents across
348 foreach my $file (
349 glob(
350 catfile(
351 $context->{d}{vcs_export},
352 'doc',
353 'doc-docbook',
354 (
355 ( $format eq 'postscript' )
356 ? '*.ps'
357 : ( '*.' . $format )
358 )
359 )
360 )
361 )
362 {
363 move( $file, catfile( $target, ( splitpath($file) )[2] ) );
364 }
365 }
366 }
367
368 sub move_text_docs_into_pkg {
369 my $context = shift;
370
371 my $old_docdir = catdir( $context->{d}{vcs_export}, 'doc', 'doc-docbook' );
372 my $old_txtdir = catdir( $context->{d}{vcs_export}, 'doc', 'doc-txt' );
373 my $new_docdir = catdir( $context->{d}{src}, 'doc' );
374 make_path( $new_docdir, { verbose => $verbose || $debug } );
375
376 # move generated documents from docbook stuff
377 foreach my $file (qw/exim.8 spec.txt filter.txt/) {
378 die "Empty file \"$file\"\n" if -z catfile( $old_docdir, $file );
379 move( catfile( $old_docdir, $file ), catfile( $new_docdir, $file ) );
380 }
381
382 # move text documents across
383 foreach my $file ( glob( catfile( $old_txtdir, '*' ) ) ) {
384
385 # skip a few we dont want
386 my $fn = ( splitpath($file) )[2];
387 next
388 if ( ( $fn eq 'ABOUT' )
389 || ( $fn eq 'ChangeLog.0' )
390 || ( $fn eq 'test-harness.txt' )
391 # Debian issue re licensing of RFCs
392 || ( $fn =~ /^draft-ietf-.*/ )
393 || ( $fn =~ /^rfc.*/ )
394 );
395 move( $file, catfile( $new_docdir, $fn ) );
396 }
397 }
398
399 sub create_tar_files {
400 my $context = shift;
401
402 my $pkg_tars = $context->{d}{pkg_tars};
403 my $pkg_trees = $context->{d}{pkg_trees};
404 my $tar = $context->{tar_cmd};
405 if ($verbose) {
406 foreach my $c (keys %{ $context->{compressors} }) {
407 print "Compression: $c\t$context->{compressors}{$c}\n";
408 }
409 }
410
411 # We ideally do not want local system user information in release tarballs;
412 # those are artifacts of use of tar for backups and have no place in
413 # software release packaging; if someone extracts as root, then they should
414 # get sane file ownerships.
415 my $ownership = "";
416 if (`tar --help 2>&1` =~ /^\s*--owner=/m) {
417 $ownership .= " --owner=$context->{tar_perms}{user} --group=$context->{tar_perms}{group}";
418 # on this GNU tar, --numeric-owner works during creation too
419 $ownership .= " --numeric-owner";
420 }
421
422 # See also environment variables set in main, tuning compression levels
423 my @COMPRESSIONS = (
424 # compressors-dict-key, file-extension, flags-as-string
425 [ "gzip", "gz", "--gzip" ],
426 [ "bzip2", "bz2", "--bzip2" ],
427 [ "lzip", "lz", "--lzip" ],
428 [ "xz", "xz", "--xz" ],
429 );
430
431 foreach my $dir ( glob( catdir( $pkg_trees, ( 'exim*-' . $context->{v}{release} ) ) ) ) {
432 my $dirname = ( splitdir($dir) )[-1];
433 foreach my $comp (@COMPRESSIONS) {
434 my ($compkey, $extension, $flags) = @{$comp};
435 next unless $context->{compressors}{$compkey};
436 print "Creating: ${pkg_tars}/${dirname}.tar.${extension}\n" if $verbose || $debug;
437 0 == system("$tar cf ${pkg_tars}/${dirname}.tar.${extension} ${flags} ${ownership} -C ${pkg_trees} ${dirname}")
438 or exit $? >> 8;
439 }
440 }
441 }
442
443 sub do_cleanup {
444 my $context = shift;
445
446 print "Cleaning up\n" if $verbose;
447 remove_tree $context->{d}{tmp}, { verbose => $debug + $verbose };
448 }
449
450 }
451
452 MAIN: {
453
454 # some of these settings are useful only if we're in the
455 # exim-projekt-root, but the check, if we're, is deferred
456 my $context = Context->new(
457 pkgname => 'exim',
458 website_base => abs_path('../exim-website'),
459 tar_cmd => 'tar',
460 tar_perms => {
461 user => '0',
462 group => '0',
463 },
464 make_cmd => 'make',
465 compressors => {
466 gzip => 1,
467 bzip2 => 1,
468 xz => 1,
469 lzip => 0,
470 },
471 docs => 1,
472 web => 1,
473 delete => 0,
474 cleanup => 1,
475 );
476
477 ##$ENV{'PATH'} = '/opt/local/bin:' . $ENV{'PATH'};
478 # We are creating files for mass distribution, so work harder to make smaller files.
479 $ENV{GZIP} = -9;
480 $ENV{BZIP2} = -9;
481 # xz documents minimum file sizes for levels higher than -6 to be useful and each
482 # requires more RAM on the decompressing system. Exim tarball currently 24MiB so
483 # using -8.
484 $ENV{XZ_DEFAULTS} = -8;
485
486 GetOptions(
487 $context,
488 qw(workspace|tmp=s outdir=s website_base|webgen_base=s tar_cmd=s make_cmd=s docs|build-docs! web|build-web!
489 delete! cleanup!),
490 'lzip!' => \$context->{compressors}{lzip},
491 'verbose!' => \$verbose,
492 'debug!' => \$debug,
493 'help|?' => sub { pod2usage(-verbose => 1, -exit => 0) },
494 'man!' => sub { pod2usage(-verbose => 2, -exit => 0, -noperldoc => system('perldoc -V >/dev/null 2>&1')) },
495 ) and @ARGV == 2 or pod2usage;
496
497
498 $context->{OUTDIR} = pop @ARGV;
499
500 -f '.exim-project-root'
501 or die "$ME: please call this script from the root of the Exim project sources\n";
502
503 warn "$ME: changed umask to 022\n" if umask(022) != 022;
504
505 $context->check_version(shift);
506 $context->override_tar_cmd;
507 $context->prepare_working_directory;
508 $context->export_git_tree;
509 $context->unpack_tree;
510 $context->make_version_script;
511 $context->build_documentation if $context->{docs};
512
513 $context->build_src_package_directory;
514 $context->build_doc_packages_directory if $context->{docs};
515
516 $context->create_tar_files;
517 $context->move_to_outdir;
518 $context->do_cleanup if $context->{cleanup};
519
520 ### $context
521 }
522
523 1;
524
525 __END__
526
527 =head1 NAME
528
529 mk_exim_release - Build an exim release
530
531 =head1 SYNOPSIS
532
533 mk_exim_release [options] version PKG-DIRECTORY
534
535 =head1 DESCRIPTION
536
537 B<mk_exim_release> builds an exim release.
538
539 Starting in a populated git repo that has already been tagged for
540 release it builds docs, packages etc. Parameter is the version number
541 to build as - ie 4.72 4.72-RC1, 4.86.1, etc, without any prefix.
542
543 This scripts expects to find a tag "exim-<version>".
544
545 After creating the release files, they should be signed. There is another
546 helper for creating the signatures:
547 F<release-process/scripts/sign_exim_packages>.
548
549 Call B<mk_exim_release> about like this:
550
551 release-process/scripts/mk_exim_release 4.99 OUT-DIR
552
553
554 =head1 OPTIONS
555
556 =over 4
557
558 =item B<--[no]cleanup>
559
560 Do (or do not) cleanup the tmp directory at exit (default: do cleanup)
561
562 =item B<--[no]debug>
563
564 Forces debug mode. (default: no debug info)
565
566 =item B<--[no]delete>
567
568 Delete a pre-existing tmp- and package-directory at start. (default: don't delete)
569
570 =item B<--tmpdir> I<dir>
571
572 Change the name of the tmp directory (default: temporary directory)
573
574 =item B<--[no]help>
575
576 Display short help and exit cleanly. (default: don't do that)
577
578 =item B<--[no]lzip>
579
580 Control the creation of B<lzip> tarballs. (default: do not use lzip)
581
582 =item B<--make> I<cmd>
583
584 Force the use of a specific C<make> command. This may be necessary if C<make> is not
585 C<gmake> (default: C<make>)
586
587 =item B<--[no]man>
588
589 Display man page and exit cleanly. (default: don't do that)
590
591 =item B<--tar> I<cmd>
592
593 Use to override the path to the C<tar> command. Need GNU tar in case
594 I<lzip> is selected. (default: C<gtar>, if not found, use C<tar>)
595
596 =item B<--[no]web>
597
598 Control the creation of the website. For creation of the website, the F<../exim-website>
599 (but see the B<website-base> option) directory must exist. (default: create the website)
600
601 =item B<--website-base> I<dir>
602
603 Base directory for the web site generation (default: F<../exim-website>)
604
605 =item B<--verbose>
606
607 Force verbose mode. (default: no verbosity)
608
609 =back
610
611 =head1 AUTHOR
612
613 Nigel Metheringham <Nigel.Metheringham@dev.intechnology.co.uk>,
614 some changes by Heiko Schlittermann <hs@schlittermann.de>
615
616 =head1 COPYRIGHT
617
618 Copyright 2010-2016 Exim Maintainers. All rights reserved.
619
620 =cut
621 # vim: set sw=4 et :