--- embedaddon/rsync/packaging/release-rsync 2016/11/01 09:54:32 1.1.1.3 +++ embedaddon/rsync/packaging/release-rsync 2021/03/17 00:32:36 1.1.1.4 @@ -1,421 +1,381 @@ -#!/usr/bin/perl +#!/usr/bin/env -S python3 -B + # This script expects the directory ~/samba-rsync-ftp to exist and to be a # copy of the /home/ftp/pub/rsync dir on samba.org. When the script is done, # the git repository in the current directory will be updated, and the local # ~/samba-rsync-ftp dir will be ready to be rsynced to samba.org. -use strict; -use warnings; -use Cwd; -use Getopt::Long; -use Term::ReadKey; -use Date::Format; +import os, sys, re, argparse, glob, shutil, signal +from datetime import datetime +from getpass import getpass -my $dest = $ENV{HOME} . '/samba-rsync-ftp'; -my $passfile = $ENV{HOME} . '/.rsyncpass'; -my $path = $ENV{PATH}; -my $make_gen_cmd = 'make -f prepare-source.mak conf && ./config.status && make gen'; +sys.path = ['packaging'] + sys.path -&Getopt::Long::Configure('bundling'); -&usage if !&GetOptions( - 'branch|b=s' => \( my $master_branch = 'master' ), - 'help|h' => \( my $help_opt ), -); -&usage if $help_opt; +from pkglib import * -my $now = time; -my $cl_today = time2str('* %a %b %d %Y', $now); -my $year = time2str('%Y', $now); -my $ztoday = time2str('%d %b %Y', $now); -(my $today = $ztoday) =~ s/^0//; +os.environ['LESS'] = 'mqeiXR'; # Make sure that -F is turned off and -R is turned on. +dest = os.environ['HOME'] + '/samba-rsync-ftp' +ORIGINAL_PATH = os.environ['PATH'] -my $curdir = Cwd::cwd; +def main(): + now = datetime.now() + cl_today = now.strftime('* %a %b %d %Y') + year = now.strftime('%Y') + ztoday = now.strftime('%d %b %Y') + today = ztoday.lstrip('0') -END { - unlink($passfile); -} + mandate_gensend_hook() -my @extra_files; -open(IN, '<', 'Makefile.in') or die "Couldn't open Makefile.in: $!\n"; -while () { - if (s/^GENFILES=//) { - while (s/\\$//) { - $_ .= ; - } - @extra_files = split(' ', $_); - last; - } -} -close IN; + curdir = os.getcwd() -my $break = <) { - if (/^AC_INIT\(\[rsync\],\s*\[(\d.+?)\]/) { - $confversion = $1; - last; - } -} -close IN; -die "Unable to find AC_INIT with version in configure.ac\n" unless defined $confversion; + curversion = get_rsync_version() -open(IN, '<', 'OLDNEWS') or die $!; -$_ = ; -my($lastversion) = /(\d+\.\d+\.\d+)/; -my($last_protocol_version, %pdate); -while () { - if (my($ver,$pdate,$pver) = /^\s+\S\S\s\S\S\S\s\d\d\d\d\s+(\d+\.\d+\.\d+)\s+(\d\d \w\w\w \d\d\d\d\s+)?(\d+)$/) { - $pdate{$ver} = $pdate if defined $pdate; - $last_protocol_version = $pver if $ver eq $lastversion; - } -} -close IN; -die "Unable to determine protocol_version for $lastversion.\n" unless defined $last_protocol_version; + # All version values are strings! + lastversion, last_protocol_version, pdate = get_NEWS_version_info() + protocol_version, subprotocol_version = get_protocol_versions() -my $protocol_version; -open(IN, '<', 'rsync.h') or die $!; -while () { - if (/^#define\s+PROTOCOL_VERSION\s+(\d+)/) { - $protocol_version = $1; - last; - } -} -close IN; -die "Unable to determine the current PROTOCOL_VERSION.\n" unless defined $protocol_version; + version = curversion + m = re.search(r'pre(\d+)', version) + if m: + version = re.sub(r'pre\d+', 'pre' + str(int(m[1]) + 1), version) + else: + version = version.replace('dev', 'pre1') -my $version = $confversion; -$version =~ s/dev/pre1/ || $version =~ s/pre(\d+)/ 'pre' . ($1 + 1) /e; + ans = input(f"Please enter the version number of this release: [{version}] ") + if ans == '.': + version = re.sub(r'pre\d+', '', version) + elif ans != '': + version = ans + if not re.match(r'^[\d.]+(pre\d+)?$', version): + die(f'Invalid version: "{version}"') -print "Please enter the version number of this release: [$version] "; -chomp($_ = ); -if ($_ eq '.') { - $version =~ s/pre\d+//; -} elsif ($_ ne '') { - $version = $_; -} -die "Invalid version: `$version'\n" unless $version =~ /^[\d.]+(pre\d+)?$/; + v_ver = 'v' + version + rsync_ver = 'rsync-' + version -if (`git tag -l v$version` ne '') { - print "Tag v$version already exists.\n\nDelete tag or quit? [q/del] "; - $_ = ; - exit 1 unless /^del/i; - system "git tag -d v$version"; -} + if os.path.lexists(rsync_ver): + die(f'"{rsync_ver}" must not exist in the current directory.') -if ($version =~ s/[-.]*pre[-.]*/pre/ && $confversion !~ /dev$/) { - $lastversion = $confversion; -} + out = cmd_txt_chk(['git', 'tag', '-l', v_ver]) + if out != '': + print(f"Tag {v_ver} already exists.") + ans = input("\nDelete tag or quit? [Q/del] ") + if not re.match(r'^del', ans, flags=re.I): + die("Aborted") + cmd_chk(['git', 'tag', '-d', v_ver]) -print "Enter the previous version to produce a patch against: [$lastversion] "; -chomp($_ = ); -$lastversion = $_ if $_ ne ''; -$lastversion =~ s/[-.]*pre[-.]*/pre/; + version = re.sub(r'[-.]*pre[-.]*', 'pre', version) + if 'pre' in version and not curversion.endswith('dev'): + lastversion = curversion -my $pre = $version =~ /(pre\d+)/ ? $1 : ''; + ans = input(f"Enter the previous version to produce a patch against: [{lastversion}] ") + if ans != '': + lastversion = ans + lastversion = re.sub(r'[-.]*pre[-.]*', 'pre', lastversion) -my $release = $pre ? '0.1' : '1'; -print "Please enter the RPM release number of this release: [$release] "; -chomp($_ = ); -$release = $_ if $_ ne ''; -$release .= ".$pre" if $pre; + rsync_lastver = 'rsync-' + lastversion + if os.path.lexists(rsync_lastver): + die(f'"{rsync_lastver}" must not exist in the current directory.') -(my $finalversion = $version) =~ s/pre\d+//; -my($proto_changed,$proto_change_date); -if ($protocol_version eq $last_protocol_version) { - $proto_changed = 'unchanged'; - $proto_change_date = "\t\t"; -} else { - $proto_changed = 'changed'; - if (!defined($proto_change_date = $pdate{$finalversion})) { - while (1) { - print "On what date did the protocol change to $protocol_version get checked in? (dd Mmm yyyy) "; - chomp($_ = ); - last if /^\d\d \w\w\w \d\d\d\d$/; - } - $proto_change_date = "$_\t"; - } -} + m = re.search(r'(pre\d+)', version) + pre = m[1] if m else '' -my($srcdir,$srcdiffdir,$lastsrcdir,$skipping); -if ($lastversion =~ /pre/) { - if (!$pre) { - die "You should not diff a release version against a pre-release version.\n"; - } - $srcdir = $srcdiffdir = $lastsrcdir = 'src-previews'; - $skipping = ' ** SKIPPING **'; -} elsif ($pre) { - $srcdir = $srcdiffdir = 'src-previews'; - $lastsrcdir = 'src'; - $skipping = ' ** SKIPPING **'; -} else { - $srcdir = $lastsrcdir = 'src'; - $srcdiffdir = 'src-diffs'; - $skipping = ''; -} + release = '0.1' if pre else '1' + ans = input(f"Please enter the RPM release number of this release: [{release}] ") + if ans != '': + release = ans + if pre: + release += '.' + pre -print "\n", $break, < ") -EOT -print " "; -$_ = ; + specvars = { + 'Version:': finalversion, + 'Release:': release, + '%define fullversion': f'%{{version}}{pre}', + 'Released': version + '.', + '%define srcdir': srcdir, + } -my %specvars = ( 'Version:' => $finalversion, 'Release:' => $release, - '%define fullversion' => "\%{version}$pre", 'Released' => "$version.", - '%define srcdir' => $srcdir ); -my @tweak_files = ( glob('packaging/*.spec'), glob('packaging/*/*.spec'), glob('*.yo'), - qw( configure.ac rsync.h NEWS OLDNEWS options.c ) ); + tweak_files = 'version.h rsync.h NEWS.md'.split() + tweak_files += glob.glob('packaging/*.spec') + tweak_files += glob.glob('packaging/*/*.spec') -foreach my $fn (@tweak_files) { - open(IN, '<', $fn) or die $!; - undef $/; $_ = ; $/ = "\n"; - close IN; - if ($fn =~ /configure/) { - s/^(AC_INIT\(\[rsync\],\s*\[)\d.+?(\])/$1$version$2/m - or die "Unable to update AC_INIT with version in $fn\n"; - } elsif ($fn =~ /\.spec/) { - while (my($str, $val) = each %specvars) { - s/^\Q$str\E .*/$str $val/m - or die "Unable to update $str in $fn\n"; - } - s/^\* \w\w\w \w\w\w \d\d \d\d\d\d (.*)/$cl_today $1/m - or die "Unable to update ChangeLog header in $fn\n"; - } elsif ($fn =~ /\.yo/) { - s/^(manpage\([^)]+\)\(\d+\)\()[^)]+(\).*)/$1$today$2/m - or die "Unable to update date in manpage() header in $fn\n"; - s/^(This man ?page is current for version) \S+ (of rsync)/$1 $version $2/m - or die "Unable to update current version info in $fn\n"; - } elsif ($fn eq 'rsync.h') { - s{(#define\s+SUBPROTOCOL_VERSION)\s+(\d+)} - { $1 . ' ' . get_subprotocol_version($2) }e - or die "Unable to find SUBPROTOCOL_VERSION define in $fn\n"; - } elsif ($fn eq 'NEWS') { - s{^(NEWS for rsync \Q$finalversion\E )(\(UNRELEASED\))\s*(\nProtocol: )(\d+) (\([^)]+\))\n} - { $1 . ($pre ? $2 : "($today)") . "$3$protocol_version ($proto_changed)\n" }ei - or die "The first 2 lines of $fn are not in the right format. They must be:\n" - . "NEWS for rsync $finalversion (UNRELEASED)\n" - . "Protocol: $protocol_version ($proto_changed)\n"; - } elsif ($fn eq 'OLDNEWS') { - s{^(\t\S\S\s\S\S\S\s\d\d\d\d)(\t\Q$finalversion\E\t).*} - { ($pre ? $1 : "\t$ztoday") . $2 . $proto_change_date . $protocol_version }em - or die "Unable to find \"?? ??? $year\t$finalversion\" line in $fn\n"; - } elsif ($fn eq 'options.c') { - if (s/(Copyright \(C\) 2002-)(\d+)( Wayne Davison)/$1$year$3/ - && $2 ne $year) { - die "Copyright comments need to be updated to $year in all files!\n"; - } - # Adjust the year in the --version output. - s/(rprintf\(f, "Copyright \(C\) 1996-)(\d+)/$1$year/ - or die "Unable to find Copyright string in --version output of $fn\n"; - next if $2 eq $year; - } else { - die "Unrecognized file in \@tweak_files: $fn\n"; - } - open(OUT, '>', $fn) or die $!; - print OUT $_; - close OUT; -} + for fn in tweak_files: + with open(fn, 'r', encoding='utf-8') as fh: + old_txt = txt = fh.read() + if fn == 'version.h': + txt = f'#define RSYNC_VERSION "{version}"\n' + elif '.spec' in fn: + for var, val in specvars.items(): + x_re = re.compile(r'^%s .*' % re.escape(var), re.M) + txt = replace_or_die(x_re, var + ' ' + val, txt, f"Unable to update {var} in {fn}") + x_re = re.compile(r'^\* \w\w\w \w\w\w \d\d \d\d\d\d (.*)', re.M) + txt = replace_or_die(x_re, r'%s \1' % cl_today, txt, f"Unable to update ChangeLog header in {fn}") + elif fn == 'rsync.h': + x_re = re.compile('(#define\s+SUBPROTOCOL_VERSION)\s+(\d+)') + repl = lambda m: m[1] + ' ' + ('0' if not pre or not proto_changed else '1' if m[2] == '0' else m[2]) + txt = replace_or_die(x_re, repl, txt, f"Unable to find SUBPROTOCOL_VERSION define in {fn}") + elif fn == 'NEWS.md': + efv = re.escape(finalversion) + x_re = re.compile(r'^<.+>\s+# NEWS for rsync %s \(UNRELEASED\)\s+## Changes in this version:\n' % efv + + r'(\n### PROTOCOL NUMBER:\s+- The protocol number was changed to \d+\.\n)?') + rel_day = 'UNRELEASED' if pre else today + repl = (f'\n\n# NEWS for rsync {finalversion} ({rel_day})\n\n' + + '## Changes in this version:\n') + if proto_changed: + repl += f'\n### PROTOCOL NUMBER:\n\n - The protocol number was changed to {protocol_version}.\n' + good_top = re.sub(r'\(.*?\)', '(UNRELEASED)', repl, 1) + msg = f"The top lines of {fn} are not in the right format. It should be:\n" + good_top + txt = replace_or_die(x_re, repl, txt, msg) + x_re = re.compile(r'^(\| )(\S{2} \S{3} \d{4})(\s+\|\s+%s\s+\| ).{11}(\s+\| )\S{2}(\s+\|+)$' % efv, re.M) + repl = lambda m: m[1] + (m[2] if pre else ztoday) + m[3] + proto_change_date + m[4] + protocol_version + m[5] + txt = replace_or_die(x_re, repl, txt, f'Unable to find "| ?? ??? {year} | {finalversion} | ... |" line in {fn}') + else: + die(f"Unrecognized file in tweak_files: {fn}") -print $break; -system "git diff --color | less -p '^diff .*'"; + if txt != old_txt: + print(f"Updating {fn}") + with open(fn, 'w', encoding='utf-8') as fh: + fh.write(txt) -my $srctar_name = "rsync-$version.tar.gz"; -my $pattar_name = "rsync-patches-$version.tar.gz"; -my $diff_name = "rsync-$lastversion-$version.diffs.gz"; -my $srctar_file = "$dest/$srcdir/$srctar_name"; -my $pattar_file = "$dest/$srcdir/$pattar_name"; -my $diff_file = "$dest/$srcdiffdir/$diff_name"; -my $news_file = "$dest/$srcdir/rsync-$version-NEWS"; -my $lasttar_file = "$dest/$lastsrcdir/rsync-$lastversion.tar.gz"; + cmd_chk(['packaging/year-tweak']) -print $break, < ") -EOT -print " "; -my $ans = ; + s = cmd_run(['git', 'commit', '-a', '-m', f'Preparing for release of {version}']) + if s.returncode: + die('Aborting') -system "git commit -a -m 'Preparing for release of $version'" and exit 1; + cmd_chk('make gen') -print "Updating files in \"patches\" dir ...\n"; -system "packaging/patch-update --branch=$master_branch"; + print(f'Creating any missing patch branches.') + s = cmd_run(f'packaging/branch-from-patch --branch={args.master_branch} --add-missing') + if s.returncode: + die('Aborting') -if ($ans =~ /^y/i) { - print "\nVisiting all \"patch/$master_branch/*\" branches ...\n"; - system "packaging/patch-update --branch=$master_branch --skip-check --shell"; -} + print('Updating files in "patches" dir ...') + s = cmd_run(f'packaging/patch-update --branch={args.master_branch}') + if s.returncode: + die('Aborting') -if (-d 'patches/.git') { - system "cd patches && git commit -a -m 'The patches for $version.'" and exit 1; -} + if re.match(r'^y', ans, re.I): + print(f'\nRunning smart-make on all "patch/{args.master_branch}/*" branches ...') + cmd_run(f"packaging/patch-update --branch={args.master_branch} --skip-check --make") -print $break, < ") -EOT -print " "; -$_ = ; + # TODO: is there a better way to ensure that our passphrase is in the agent? + cmd_run("touch TeMp; gpg --sign TeMp; rm TeMp*") -# We want to use our passphrase-providing "gpg" script, so modify the PATH. -$ENV{PATH} = "$curdir/packaging/bin:$path"; + out = cmd_txt(f"git tag -s -m 'Version {version}.' {v_ver}", capture='combined') + print(out, end='') + if 'bad passphrase' in out or 'failed' in out: + die('Aborting') -my $passphrase; -while (1) { - ReadMode('noecho'); - print "\nEnter your GPG pass-phrase: "; - chomp($passphrase = ); - ReadMode(0); - print "\n"; + if os.path.isdir('patches/.git'): + out = cmd_txt(f"cd patches && git tag -s -m 'Version {version}.' {v_ver}", capture='combined') + print(out, end='') + if 'bad passphrase' in out or 'failed' in out: + die('Aborting') - # Briefly create a temp file with the passphrase for git's tagging use. - my $oldmask = umask 077; - unlink($passfile); - open(OUT, '>', $passfile) or die $!; - print OUT $passphrase, "\n"; - close OUT; - umask $oldmask; - $ENV{'GPG_PASSFILE'} = $passfile; + os.environ['PATH'] = ORIGINAL_PATH - $_ = `git tag -s -m 'Version $version.' v$version 2>&1`; - print $_; - next if /bad passphrase/; - exit 1 if /failed/; + # Extract the generated files from the old tar. + tweaked_gen_files = [ os.path.join(rsync_lastver, fn) for fn in gen_files ] + cmd_run(['tar', 'xzf', lasttar_file, *tweaked_gen_files]) + os.rename(rsync_lastver, 'a') - if (-d 'patches/.git') { - $_ = `cd patches && git tag -s -m 'Version $version.' v$version 2>&1`; - print $_; - exit 1 if /bad passphrase|failed/; - } + print(f"Creating {diff_file} ...") + cmd_chk(['rsync', '-a', *gen_pathnames, 'b/']) - unlink($passfile); - last; -} + sed_script = r's:^((---|\+\+\+) [ab]/[^\t]+)\t.*:\1:' # CAUTION: must not contain any single quotes! + cmd_chk(f"(git diff v{lastversion} {v_ver} -- ':!.github'; diff -upN a b | sed -r '{sed_script}') | gzip -9 >{diff_file}") + shutil.rmtree('a') + os.rename('b', rsync_ver) -$ENV{PATH} = $path; + print(f"Creating {srctar_file} ...") + cmd_chk(f"git archive --format=tar --prefix={rsync_ver}/ {v_ver} | tar xf -") + cmd_chk(f"support/git-set-file-times --quiet --prefix={rsync_ver}/") + cmd_chk(['fakeroot', 'tar', 'czf', srctar_file, '--exclude=.github', rsync_ver]) + shutil.rmtree(rsync_ver) -# Extract the generated files from the old tar. -@_ = @extra_files; -map { s#^#rsync-$lastversion/# } @_; -system "tar xzf $lasttar_file @_"; -rename("rsync-$lastversion", 'a'); + print(f'Updating files in "{rsync_ver}/patches" dir ...') + os.mkdir(rsync_ver, 0o755) + os.mkdir(f"{rsync_ver}/patches", 0o755) + cmd_chk(f"packaging/patch-update --skip-check --branch={args.master_branch} --gen={rsync_ver}/patches".split()) -print "Creating $diff_file ...\n"; -system "$make_gen_cmd && rsync -a @extra_files b/" and exit 1; -my $sed_script = 's:^((---|\+\+\+) [ab]/[^\t]+)\t.*:\1:'; -system "(git diff v$lastversion v$version; diff -upN a b | sed -r '$sed_script') | gzip -9 >$diff_file"; -system "rm -rf a"; -rename('b', "rsync-$version"); + print(f"Creating {pattar_file} ...") + cmd_chk(['fakeroot', 'tar', 'chzf', pattar_file, rsync_ver + '/patches']) + shutil.rmtree(rsync_ver) -print "Creating $srctar_file ...\n"; -system "git archive --format=tar --prefix=rsync-$version/ v$version | tar xf -"; -system "support/git-set-file-times --prefix=rsync-$version/"; -system "fakeroot tar czf $srctar_file rsync-$version; rm -rf rsync-$version"; + print(f"Updating the other files in {dest} ...") + md_files = 'README.md NEWS.md INSTALL.md'.split() + html_files = [ fn for fn in gen_pathnames if fn.endswith('.html') ] + cmd_chk(['rsync', '-a', *md_files, *html_files, dest]) + cmd_chk(["packaging/md2html"] + [ dest +'/'+ fn for fn in md_files ]) -print "Updating files in \"rsync-$version/patches\" dir ...\n"; -mkdir("rsync-$version", 0755); -mkdir("rsync-$version/patches", 0755); -system "packaging/patch-update --skip-check --branch=$master_branch --gen=rsync-$version/patches"; + cmd_chk(f"git log --name-status | gzip -9 >{dest}/ChangeLog.gz") -print "Creating $pattar_file ...\n"; -system "fakeroot tar chzf $pattar_file rsync-$version/patches; rm -rf rsync-$version"; + for fn in (srctar_file, pattar_file, diff_file): + asc_fn = fn + '.asc' + if os.path.lexists(asc_fn): + os.unlink(asc_fn) + res = cmd_run(['gpg', '--batch', '-ba', fn]) + if res.returncode != 0 and res.returncode != 2: + die("gpg signing failed") -print "Updating the other files in $dest ...\n"; -system "rsync -a README NEWS OLDNEWS TODO $dest"; -unlink($news_file); -link("$dest/NEWS", $news_file); -system "git log --name-status | gzip -9 >$dest/ChangeLog.gz"; + if not pre: + for find in f'{dest}/rsync-*.gz {dest}/rsync-*.asc {dest}/src-previews/rsync-*diffs.gz*'.split(): + for fn in glob.glob(find): + os.unlink(fn) + top_link = [ + srctar_file, f"{srctar_file}.asc", + pattar_file, f"{pattar_file}.asc", + diff_file, f"{diff_file}.asc", + ] + for fn in top_link: + os.link(fn, re.sub(r'/src(-\w+)?/', '/', fn)) -system "yodl2html -o $dest/rsync.html rsync.yo"; -system "yodl2html -o $dest/rsyncd.conf.html rsyncd.conf.yo"; + print(f"""\ +{dash_line} -foreach my $fn ($srctar_file, $pattar_file, $diff_file) { - unlink("$fn.asc"); - open(GPG, '|-', "gpg --batch --passphrase-fd=0 -ba $fn") or die $!; - print GPG $passphrase, "\n"; - close GPG; -} - -if (!$pre) { - system "rm $dest/rsync-*.gz $dest/rsync-*.asc $dest/rsync-*-NEWS $dest/src-previews/rsync-*diffs.gz*"; - - foreach my $fn ($srctar_file, "$srctar_file.asc", - $pattar_file, "$pattar_file.asc", - $diff_file, "$diff_file.asc", $news_file) { - (my $top_fn = $fn) =~ s#/src(-\w+)?/#/#; - link($fn, $top_fn); - } -} - -print $break, <<'EOT'; - Local changes are done. When you're satisfied, push the git repository and rsync the release files. Remember to announce the release on *BOTH* rsync-announce@lists.samba.org and rsync@lists.samba.org (and the web)! -EOT +""") -exit; -sub get_subprotocol_version -{ - my($subver) = @_; - if ($pre && $proto_changed eq 'changed') { - return $subver == 0 ? 1 : $subver; - } - 0; -} +def replace_or_die(regex, repl, txt, die_msg): + m = regex.search(txt) + if not m: + die(die_msg) + return regex.sub(repl, txt, 1) -sub usage -{ - die <