automation: implement "publish-windows-artifacts" command
authorGregory Szorc <gregory.szorc@gmail.com>
Thu, 05 Sep 2019 21:09:58 -0700
changeset 42907 92593d72e10b
parent 42906 6bf88befa027
child 42908 f78f7448a969
automation: implement "publish-windows-artifacts" command The new command and associated functionality can be used to automate the publishing of Windows release artifacts. It supports uploading wheels to PyPI (using twine) and copying the artifacts to mercurial-scm.org and updating the latest.dat file to advertise them via the website. I ran `automation.py publish-windows-artifacts 5.1.1` and it appeared to "just work." But the real test will be to do this on the next release... Differential Revision: https://phab.mercurial-scm.org/D6786
contrib/automation/README.rst
contrib/automation/hgautomation/cli.py
contrib/automation/hgautomation/pypi.py
contrib/automation/hgautomation/windows.py
contrib/automation/requirements.txt
contrib/automation/requirements.txt.in
tests/test-check-code.t
--- a/contrib/automation/README.rst	Thu Sep 05 21:08:35 2019 -0700
+++ b/contrib/automation/README.rst	Thu Sep 05 21:09:58 2019 -0700
@@ -181,3 +181,25 @@
 Documenting them is beyond the scope of this document. Various tests
 also require other optional dependencies and missing dependencies will
 be printed by the test runner when a test is skipped.
+
+Releasing Windows Artifacts
+===========================
+
+The `automation.py` script can be used to automate the release of Windows
+artifacts::
+
+   $ ./automation.py build-all-windows-packages --revision 5.1.1
+   $ ./automation.py publish-windows-artifacts 5.1.1
+
+The first command will launch an EC2 instance to build all Windows packages
+and copy them into the `dist` directory relative to the repository root. The
+second command will then attempt to upload these files to PyPI (via `twine`)
+and to `mercurial-scm.org` (via SSH).
+
+Uploading to PyPI requires a PyPI account with write access to the `Mercurial`
+package. You can skip PyPI uploading by passing `--no-pypi`.
+
+Uploading to `mercurial-scm.org` requires an SSH account on that server
+with `windows` group membership and for the SSH key for that account to be the
+default SSH key (e.g. `~/.ssh/id_rsa`) or in a running SSH agent. You can
+skip `mercurial-scm.org` uploading by passing `--no-mercurial-scm-org`.
--- a/contrib/automation/hgautomation/cli.py	Thu Sep 05 21:08:35 2019 -0700
+++ b/contrib/automation/hgautomation/cli.py	Thu Sep 05 21:09:58 2019 -0700
@@ -185,6 +185,14 @@
                           test_flags)
 
 
+def publish_windows_artifacts(hg: HGAutomation, aws_region, version: str,
+                              pypi: bool, mercurial_scm_org: bool,
+                              ssh_username: str):
+    windows.publish_artifacts(DIST_PATH, version,
+                              pypi=pypi, mercurial_scm_org=mercurial_scm_org,
+                              ssh_username=ssh_username)
+
+
 def get_parser():
     parser = argparse.ArgumentParser()
 
@@ -403,6 +411,34 @@
     )
     sp.set_defaults(func=run_tests_windows)
 
+    sp = subparsers.add_parser(
+        'publish-windows-artifacts',
+        help='Publish built Windows artifacts (wheels, installers, etc)'
+    )
+    sp.add_argument(
+        '--no-pypi',
+        dest='pypi',
+        action='store_false',
+        default=True,
+        help='Skip uploading to PyPI',
+    )
+    sp.add_argument(
+        '--no-mercurial-scm-org',
+        dest='mercurial_scm_org',
+        action='store_false',
+        default=True,
+        help='Skip uploading to www.mercurial-scm.org',
+    )
+    sp.add_argument(
+        '--ssh-username',
+        help='SSH username for mercurial-scm.org',
+    )
+    sp.add_argument(
+        'version',
+        help='Mercurial version string to locate local packages',
+    )
+    sp.set_defaults(func=publish_windows_artifacts)
+
     return parser
 
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/contrib/automation/hgautomation/pypi.py	Thu Sep 05 21:09:58 2019 -0700
@@ -0,0 +1,25 @@
+# pypi.py - Automation around PyPI
+#
+# Copyright 2019 Gregory Szorc <gregory.szorc@gmail.com>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+# no-check-code because Python 3 native.
+
+from twine.commands.upload import (
+    upload as twine_upload,
+)
+from twine.settings import (
+    Settings,
+)
+
+
+def upload(paths):
+    """Upload files to PyPI.
+
+    `paths` is an iterable of `pathlib.Path`.
+    """
+    settings = Settings()
+
+    twine_upload(settings, [str(p) for p in paths])
--- a/contrib/automation/hgautomation/windows.py	Thu Sep 05 21:08:35 2019 -0700
+++ b/contrib/automation/hgautomation/windows.py	Thu Sep 05 21:09:58 2019 -0700
@@ -7,12 +7,17 @@
 
 # no-check-code because Python 3 native.
 
+import datetime
 import os
+import paramiko
 import pathlib
 import re
 import subprocess
 import tempfile
 
+from .pypi import (
+    upload as pypi_upload,
+)
 from .winrm import (
     run_powershell,
 )
@@ -100,6 +105,26 @@
 }}
 '''
 
+X86_WHEEL_FILENAME = 'mercurial-{version}-cp27-cp27m-win32.whl'
+X64_WHEEL_FILENAME = 'mercurial-{version}-cp27-cp27m-win_amd64.whl'
+X86_EXE_FILENAME = 'Mercurial-{version}.exe'
+X64_EXE_FILENAME = 'Mercurial-{version}-x64.exe'
+X86_MSI_FILENAME = 'mercurial-{version}-x86.msi'
+X64_MSI_FILENAME = 'mercurial-{version}-x64.msi'
+
+MERCURIAL_SCM_BASE_URL = 'https://mercurial-scm.org/release/windows'
+
+X86_USER_AGENT_PATTERN = '.*Windows.*'
+X64_USER_AGENT_PATTERN = '.*Windows.*(WOW|x)64.*'
+
+X86_EXE_DESCRIPTION = ('Mercurial {version} Inno Setup installer - x86 Windows '
+                       '- does not require admin rights')
+X64_EXE_DESCRIPTION = ('Mercurial {version} Inno Setup installer - x64 Windows '
+                       '- does not require admin rights')
+X86_MSI_DESCRIPTION = ('Mercurial {version} MSI installer - x86 Windows '
+                       '- requires admin rights')
+X64_MSI_DESCRIPTION = ('Mercurial {version} MSI installer - x64 Windows '
+                       '- requires admin rights')
 
 def get_vc_prefix(arch):
     if arch == 'x86':
@@ -296,3 +321,152 @@
     )
 
     run_powershell(winrm_client, ps)
+
+
+def resolve_wheel_artifacts(dist_path: pathlib.Path, version: str):
+    return (
+        dist_path / X86_WHEEL_FILENAME.format(version=version),
+        dist_path / X64_WHEEL_FILENAME.format(version=version),
+    )
+
+
+def resolve_all_artifacts(dist_path: pathlib.Path, version: str):
+    return (
+        dist_path / X86_WHEEL_FILENAME.format(version=version),
+        dist_path / X64_WHEEL_FILENAME.format(version=version),
+        dist_path / X86_EXE_FILENAME.format(version=version),
+        dist_path / X64_EXE_FILENAME.format(version=version),
+        dist_path / X86_MSI_FILENAME.format(version=version),
+        dist_path / X64_MSI_FILENAME.format(version=version),
+    )
+
+
+def generate_latest_dat(version: str):
+    x86_exe_filename = X86_EXE_FILENAME.format(version=version)
+    x64_exe_filename = X64_EXE_FILENAME.format(version=version)
+    x86_msi_filename = X86_MSI_FILENAME.format(version=version)
+    x64_msi_filename = X64_MSI_FILENAME.format(version=version)
+
+    entries = (
+        (
+            '10',
+            version,
+            X86_USER_AGENT_PATTERN,
+            '%s/%s' % (MERCURIAL_SCM_BASE_URL, x86_exe_filename),
+            X86_EXE_DESCRIPTION.format(version=version),
+        ),
+        (
+            '10',
+            version,
+            X64_USER_AGENT_PATTERN,
+            '%s/%s' % (MERCURIAL_SCM_BASE_URL, x64_exe_filename),
+            X64_EXE_DESCRIPTION.format(version=version),
+        ),
+        (
+            '10',
+            version,
+            X86_USER_AGENT_PATTERN,
+            '%s/%s' % (MERCURIAL_SCM_BASE_URL, x86_msi_filename),
+            X86_MSI_DESCRIPTION.format(version=version),
+        ),
+        (
+            '10',
+            version,
+            X64_USER_AGENT_PATTERN,
+            '%s/%s' % (MERCURIAL_SCM_BASE_URL, x64_msi_filename),
+            X64_MSI_DESCRIPTION.format(version=version)
+        )
+    )
+
+    lines = ['\t'.join(e) for e in entries]
+
+    return '\n'.join(lines) + '\n'
+
+
+def publish_artifacts_pypi(dist_path: pathlib.Path, version: str):
+    """Publish Windows release artifacts to PyPI."""
+
+    wheel_paths = resolve_wheel_artifacts(dist_path, version)
+
+    for p in wheel_paths:
+        if not p.exists():
+            raise Exception('%s not found' % p)
+
+    print('uploading wheels to PyPI (you may be prompted for credentials)')
+    pypi_upload(wheel_paths)
+
+
+def publish_artifacts_mercurial_scm_org(dist_path: pathlib.Path, version: str,
+                                        ssh_username=None):
+    """Publish Windows release artifacts to mercurial-scm.org."""
+    all_paths = resolve_all_artifacts(dist_path, version)
+
+    for p in all_paths:
+        if not p.exists():
+            raise Exception('%s not found' % p)
+
+    client = paramiko.SSHClient()
+    client.load_system_host_keys()
+    # We assume the system SSH configuration knows how to connect.
+    print('connecting to mercurial-scm.org via ssh...')
+    try:
+        client.connect('mercurial-scm.org', username=ssh_username)
+    except paramiko.AuthenticationException:
+        print('error authenticating; is an SSH key available in an SSH agent?')
+        raise
+
+    print('SSH connection established')
+
+    print('opening SFTP client...')
+    sftp = client.open_sftp()
+    print('SFTP client obtained')
+
+    for p in all_paths:
+        dest_path = '/var/www/release/windows/%s' % p.name
+        print('uploading %s to %s' % (p, dest_path))
+
+        with p.open('rb') as fh:
+            data = fh.read()
+
+        with sftp.open(dest_path, 'wb') as fh:
+            fh.write(data)
+            fh.chmod(0o0664)
+
+    latest_dat_path = '/var/www/release/windows/latest.dat'
+
+    now = datetime.datetime.utcnow()
+    backup_path = dist_path / (
+        'latest-windows-%s.dat' % now.strftime('%Y%m%dT%H%M%S'))
+    print('backing up %s to %s' % (latest_dat_path, backup_path))
+
+    with sftp.open(latest_dat_path, 'rb') as fh:
+        latest_dat_old = fh.read()
+
+    with backup_path.open('wb') as fh:
+        fh.write(latest_dat_old)
+
+    print('writing %s with content:' % latest_dat_path)
+    latest_dat_content = generate_latest_dat(version)
+    print(latest_dat_content)
+
+    with sftp.open(latest_dat_path, 'wb') as fh:
+        fh.write(latest_dat_content.encode('ascii'))
+
+
+def publish_artifacts(dist_path: pathlib.Path, version: str,
+                      pypi=True, mercurial_scm_org=True,
+                      ssh_username=None):
+    """Publish Windows release artifacts.
+
+    Files are found in `dist_path`. We will look for files with version string
+    `version`.
+
+    `pypi` controls whether we upload to PyPI.
+    `mercurial_scm_org` controls whether we upload to mercurial-scm.org.
+    """
+    if pypi:
+        publish_artifacts_pypi(dist_path, version)
+
+    if mercurial_scm_org:
+        publish_artifacts_mercurial_scm_org(dist_path, version,
+                                            ssh_username=ssh_username)
--- a/contrib/automation/requirements.txt	Thu Sep 05 21:08:35 2019 -0700
+++ b/contrib/automation/requirements.txt	Thu Sep 05 21:09:58 2019 -0700
@@ -26,6 +26,10 @@
     --hash=sha256:d7bdc26475679dd073ba0ed2766445bb5b20ca4793ca0db32b399dccc6bc84b7 \
     --hash=sha256:ff032765bb8716d9387fd5376d987a937254b0619eff0972779515b5c98820bc \
     # via paramiko
+bleach==3.1.0 \
+    --hash=sha256:213336e49e102af26d9cde77dd2d0397afabc5a6bf2fed985dc35b5d1e285a16 \
+    --hash=sha256:3fdf7f77adcf649c9911387df51254b813185e32b2c6619f690b593a617e19fa \
+    # via readme-renderer
 boto3==1.9.223 \
     --hash=sha256:12ceb047c3cfbd2363b35e1c24b082808a1bb9b90f4f0b7375e83d21015bf47b \
     --hash=sha256:6e833a9068309c24d7752e280b2925cf5968a88111bc95fcebc451a09f8b424e
@@ -93,7 +97,7 @@
     --hash=sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0 \
     --hash=sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827 \
     --hash=sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99 \
-    # via botocore
+    # via botocore, readme-renderer
 idna==2.8 \
     --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \
     --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \
@@ -109,9 +113,17 @@
 paramiko==2.6.0 \
     --hash=sha256:99f0179bdc176281d21961a003ffdb2ec369daac1a1007241f53374e376576cf \
     --hash=sha256:f4b2edfa0d226b70bd4ca31ea7e389325990283da23465d572ed1f70a7583041
+pkginfo==1.5.0.1 \
+    --hash=sha256:7424f2c8511c186cd5424bbf31045b77435b37a8d604990b79d4e70d741148bb \
+    --hash=sha256:a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32 \
+    # via twine
 pycparser==2.19 \
     --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 \
     # via cffi
+pygments==2.4.2 \
+    --hash=sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127 \
+    --hash=sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297 \
+    # via readme-renderer
 pynacl==1.3.0 \
     --hash=sha256:05c26f93964373fc0abe332676cb6735f0ecad27711035b9472751faa8521255 \
     --hash=sha256:0c6100edd16fefd1557da078c7a31e7b7d7a52ce39fdca2bec29d4f7b6e7600c \
@@ -140,10 +152,18 @@
     --hash=sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb \
     --hash=sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e \
     # via botocore
+readme-renderer==24.0 \
+    --hash=sha256:bb16f55b259f27f75f640acf5e00cf897845a8b3e4731b5c1a436e4b8529202f \
+    --hash=sha256:c8532b79afc0375a85f10433eca157d6b50f7d6990f337fa498c96cd4bfc203d \
+    # via twine
+requests-toolbelt==0.9.1 \
+    --hash=sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f \
+    --hash=sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0 \
+    # via twine
 requests==2.22.0 \
     --hash=sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4 \
     --hash=sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31 \
-    # via pypsrp
+    # via pypsrp, requests-toolbelt, twine
 s3transfer==0.2.1 \
     --hash=sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d \
     --hash=sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba \
@@ -151,8 +171,23 @@
 six==1.12.0 \
     --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \
     --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \
-    # via bcrypt, cryptography, pynacl, pypsrp, python-dateutil
+    # via bcrypt, bleach, cryptography, pynacl, pypsrp, python-dateutil, readme-renderer
+tqdm==4.35.0 \
+    --hash=sha256:1be3e4e3198f2d0e47b928e9d9a8ec1b63525db29095cec1467f4c5a4ea8ebf9 \
+    --hash=sha256:7e39a30e3d34a7a6539378e39d7490326253b7ee354878a92255656dc4284457 \
+    # via twine
+twine==1.13.0 \
+    --hash=sha256:0fb0bfa3df4f62076cab5def36b1a71a2e4acb4d1fa5c97475b048117b1a6446 \
+    --hash=sha256:d6c29c933ecfc74e9b1d9fa13aa1f87c5d5770e119f5a4ce032092f0ff5b14dc
 urllib3==1.25.3 \
     --hash=sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1 \
     --hash=sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232 \
     # via botocore, requests
+webencodings==0.5.1 \
+    --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \
+    --hash=sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923 \
+    # via bleach
+
+# WARNING: The following packages were not pinned, but pip requires them to be
+# pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag.
+# setuptools==41.2.0        # via twine
--- a/contrib/automation/requirements.txt.in	Thu Sep 05 21:08:35 2019 -0700
+++ b/contrib/automation/requirements.txt.in	Thu Sep 05 21:09:58 2019 -0700
@@ -1,3 +1,4 @@
 boto3
 paramiko
 pypsrp
+twine
--- a/tests/test-check-code.t	Thu Sep 05 21:08:35 2019 -0700
+++ b/tests/test-check-code.t	Thu Sep 05 21:09:58 2019 -0700
@@ -16,6 +16,7 @@
   Skipping contrib/automation/hgautomation/aws.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/cli.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/linux.py it has no-che?k-code (glob)
+  Skipping contrib/automation/hgautomation/pypi.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/ssh.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/windows.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/winrm.py it has no-che?k-code (glob)