D6786: automation: implement "publish-windows-artifacts" command

indygreg (Gregory Szorc) phabricator at mercurial-scm.org
Mon Sep 9 12:09:41 EDT 2019


Closed by commit rHG92593d72e10b: automation: implement "publish-windows-artifacts" command (authored by indygreg).
This revision was automatically updated to reflect the committed changes.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D6786?vs=16389&id=16458

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D6786/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D6786

AFFECTED FILES
  contrib/automation/README.rst
  contrib/automation/hgautomation/cli.py
  contrib/automation/hgautomation/pypi.py
  contrib/automation/hgautomation/windows.py
  contrib/automation/requirements.txt
  contrib/automation/requirements.txt.in
  tests/test-check-code.t

CHANGE DETAILS

diff --git a/tests/test-check-code.t b/tests/test-check-code.t
--- a/tests/test-check-code.t
+++ b/tests/test-check-code.t
@@ -16,6 +16,7 @@
   Skipping contrib/automation/hgautomation/aws.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/cli.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/linux.py it has no-che?k-code (glob)
+  Skipping contrib/automation/hgautomation/pypi.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/ssh.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/windows.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/winrm.py it has no-che?k-code (glob)
diff --git a/contrib/automation/requirements.txt.in b/contrib/automation/requirements.txt.in
--- a/contrib/automation/requirements.txt.in
+++ b/contrib/automation/requirements.txt.in
@@ -1,3 +1,4 @@
 boto3
 paramiko
 pypsrp
+twine
diff --git a/contrib/automation/requirements.txt b/contrib/automation/requirements.txt
--- a/contrib/automation/requirements.txt
+++ b/contrib/automation/requirements.txt
@@ -26,6 +26,10 @@
     --hash=sha256:d7bdc26475679dd073ba0ed2766445bb5b20ca4793ca0db32b399dccc6bc84b7 \
     --hash=sha256:ff032765bb8716d9387fd5376d987a937254b0619eff0972779515b5c98820bc \
     # via paramiko
+bleach==3.1.0 \
+    --hash=sha256:213336e49e102af26d9cde77dd2d0397afabc5a6bf2fed985dc35b5d1e285a16 \
+    --hash=sha256:3fdf7f77adcf649c9911387df51254b813185e32b2c6619f690b593a617e19fa \
+    # via readme-renderer
 boto3==1.9.223 \
     --hash=sha256:12ceb047c3cfbd2363b35e1c24b082808a1bb9b90f4f0b7375e83d21015bf47b \
     --hash=sha256:6e833a9068309c24d7752e280b2925cf5968a88111bc95fcebc451a09f8b424e
@@ -93,7 +97,7 @@
     --hash=sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0 \
     --hash=sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827 \
     --hash=sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99 \
-    # via botocore
+    # via botocore, readme-renderer
 idna==2.8 \
     --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \
     --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c \
@@ -109,9 +113,17 @@
 paramiko==2.6.0 \
     --hash=sha256:99f0179bdc176281d21961a003ffdb2ec369daac1a1007241f53374e376576cf \
     --hash=sha256:f4b2edfa0d226b70bd4ca31ea7e389325990283da23465d572ed1f70a7583041
+pkginfo==1.5.0.1 \
+    --hash=sha256:7424f2c8511c186cd5424bbf31045b77435b37a8d604990b79d4e70d741148bb \
+    --hash=sha256:a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32 \
+    # via twine
 pycparser==2.19 \
     --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 \
     # via cffi
+pygments==2.4.2 \
+    --hash=sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127 \
+    --hash=sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297 \
+    # via readme-renderer
 pynacl==1.3.0 \
     --hash=sha256:05c26f93964373fc0abe332676cb6735f0ecad27711035b9472751faa8521255 \
     --hash=sha256:0c6100edd16fefd1557da078c7a31e7b7d7a52ce39fdca2bec29d4f7b6e7600c \
@@ -140,10 +152,18 @@
     --hash=sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb \
     --hash=sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e \
     # via botocore
+readme-renderer==24.0 \
+    --hash=sha256:bb16f55b259f27f75f640acf5e00cf897845a8b3e4731b5c1a436e4b8529202f \
+    --hash=sha256:c8532b79afc0375a85f10433eca157d6b50f7d6990f337fa498c96cd4bfc203d \
+    # via twine
+requests-toolbelt==0.9.1 \
+    --hash=sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f \
+    --hash=sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0 \
+    # via twine
 requests==2.22.0 \
     --hash=sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4 \
     --hash=sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31 \
-    # via pypsrp
+    # via pypsrp, requests-toolbelt, twine
 s3transfer==0.2.1 \
     --hash=sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d \
     --hash=sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba \
@@ -151,8 +171,23 @@
 six==1.12.0 \
     --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \
     --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \
-    # via bcrypt, cryptography, pynacl, pypsrp, python-dateutil
+    # via bcrypt, bleach, cryptography, pynacl, pypsrp, python-dateutil, readme-renderer
+tqdm==4.35.0 \
+    --hash=sha256:1be3e4e3198f2d0e47b928e9d9a8ec1b63525db29095cec1467f4c5a4ea8ebf9 \
+    --hash=sha256:7e39a30e3d34a7a6539378e39d7490326253b7ee354878a92255656dc4284457 \
+    # via twine
+twine==1.13.0 \
+    --hash=sha256:0fb0bfa3df4f62076cab5def36b1a71a2e4acb4d1fa5c97475b048117b1a6446 \
+    --hash=sha256:d6c29c933ecfc74e9b1d9fa13aa1f87c5d5770e119f5a4ce032092f0ff5b14dc
 urllib3==1.25.3 \
     --hash=sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1 \
     --hash=sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232 \
     # via botocore, requests
+webencodings==0.5.1 \
+    --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \
+    --hash=sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923 \
+    # via bleach
+
+# WARNING: The following packages were not pinned, but pip requires them to be
+# pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag.
+# setuptools==41.2.0        # via twine
diff --git a/contrib/automation/hgautomation/windows.py b/contrib/automation/hgautomation/windows.py
--- a/contrib/automation/hgautomation/windows.py
+++ b/contrib/automation/hgautomation/windows.py
@@ -7,12 +7,17 @@
 
 # no-check-code because Python 3 native.
 
+import datetime
 import os
+import paramiko
 import pathlib
 import re
 import subprocess
 import tempfile
 
+from .pypi import (
+    upload as pypi_upload,
+)
 from .winrm import (
     run_powershell,
 )
@@ -100,6 +105,26 @@
 }}
 '''
 
+X86_WHEEL_FILENAME = 'mercurial-{version}-cp27-cp27m-win32.whl'
+X64_WHEEL_FILENAME = 'mercurial-{version}-cp27-cp27m-win_amd64.whl'
+X86_EXE_FILENAME = 'Mercurial-{version}.exe'
+X64_EXE_FILENAME = 'Mercurial-{version}-x64.exe'
+X86_MSI_FILENAME = 'mercurial-{version}-x86.msi'
+X64_MSI_FILENAME = 'mercurial-{version}-x64.msi'
+
+MERCURIAL_SCM_BASE_URL = 'https://mercurial-scm.org/release/windows'
+
+X86_USER_AGENT_PATTERN = '.*Windows.*'
+X64_USER_AGENT_PATTERN = '.*Windows.*(WOW|x)64.*'
+
+X86_EXE_DESCRIPTION = ('Mercurial {version} Inno Setup installer - x86 Windows '
+                       '- does not require admin rights')
+X64_EXE_DESCRIPTION = ('Mercurial {version} Inno Setup installer - x64 Windows '
+                       '- does not require admin rights')
+X86_MSI_DESCRIPTION = ('Mercurial {version} MSI installer - x86 Windows '
+                       '- requires admin rights')
+X64_MSI_DESCRIPTION = ('Mercurial {version} MSI installer - x64 Windows '
+                       '- requires admin rights')
 
 def get_vc_prefix(arch):
     if arch == 'x86':
@@ -296,3 +321,152 @@
     )
 
     run_powershell(winrm_client, ps)
+
+
+def resolve_wheel_artifacts(dist_path: pathlib.Path, version: str):
+    return (
+        dist_path / X86_WHEEL_FILENAME.format(version=version),
+        dist_path / X64_WHEEL_FILENAME.format(version=version),
+    )
+
+
+def resolve_all_artifacts(dist_path: pathlib.Path, version: str):
+    return (
+        dist_path / X86_WHEEL_FILENAME.format(version=version),
+        dist_path / X64_WHEEL_FILENAME.format(version=version),
+        dist_path / X86_EXE_FILENAME.format(version=version),
+        dist_path / X64_EXE_FILENAME.format(version=version),
+        dist_path / X86_MSI_FILENAME.format(version=version),
+        dist_path / X64_MSI_FILENAME.format(version=version),
+    )
+
+
+def generate_latest_dat(version: str):
+    x86_exe_filename = X86_EXE_FILENAME.format(version=version)
+    x64_exe_filename = X64_EXE_FILENAME.format(version=version)
+    x86_msi_filename = X86_MSI_FILENAME.format(version=version)
+    x64_msi_filename = X64_MSI_FILENAME.format(version=version)
+
+    entries = (
+        (
+            '10',
+            version,
+            X86_USER_AGENT_PATTERN,
+            '%s/%s' % (MERCURIAL_SCM_BASE_URL, x86_exe_filename),
+            X86_EXE_DESCRIPTION.format(version=version),
+        ),
+        (
+            '10',
+            version,
+            X64_USER_AGENT_PATTERN,
+            '%s/%s' % (MERCURIAL_SCM_BASE_URL, x64_exe_filename),
+            X64_EXE_DESCRIPTION.format(version=version),
+        ),
+        (
+            '10',
+            version,
+            X86_USER_AGENT_PATTERN,
+            '%s/%s' % (MERCURIAL_SCM_BASE_URL, x86_msi_filename),
+            X86_MSI_DESCRIPTION.format(version=version),
+        ),
+        (
+            '10',
+            version,
+            X64_USER_AGENT_PATTERN,
+            '%s/%s' % (MERCURIAL_SCM_BASE_URL, x64_msi_filename),
+            X64_MSI_DESCRIPTION.format(version=version)
+        )
+    )
+
+    lines = ['\t'.join(e) for e in entries]
+
+    return '\n'.join(lines) + '\n'
+
+
+def publish_artifacts_pypi(dist_path: pathlib.Path, version: str):
+    """Publish Windows release artifacts to PyPI."""
+
+    wheel_paths = resolve_wheel_artifacts(dist_path, version)
+
+    for p in wheel_paths:
+        if not p.exists():
+            raise Exception('%s not found' % p)
+
+    print('uploading wheels to PyPI (you may be prompted for credentials)')
+    pypi_upload(wheel_paths)
+
+
+def publish_artifacts_mercurial_scm_org(dist_path: pathlib.Path, version: str,
+                                        ssh_username=None):
+    """Publish Windows release artifacts to mercurial-scm.org."""
+    all_paths = resolve_all_artifacts(dist_path, version)
+
+    for p in all_paths:
+        if not p.exists():
+            raise Exception('%s not found' % p)
+
+    client = paramiko.SSHClient()
+    client.load_system_host_keys()
+    # We assume the system SSH configuration knows how to connect.
+    print('connecting to mercurial-scm.org via ssh...')
+    try:
+        client.connect('mercurial-scm.org', username=ssh_username)
+    except paramiko.AuthenticationException:
+        print('error authenticating; is an SSH key available in an SSH agent?')
+        raise
+
+    print('SSH connection established')
+
+    print('opening SFTP client...')
+    sftp = client.open_sftp()
+    print('SFTP client obtained')
+
+    for p in all_paths:
+        dest_path = '/var/www/release/windows/%s' % p.name
+        print('uploading %s to %s' % (p, dest_path))
+
+        with p.open('rb') as fh:
+            data = fh.read()
+
+        with sftp.open(dest_path, 'wb') as fh:
+            fh.write(data)
+            fh.chmod(0o0664)
+
+    latest_dat_path = '/var/www/release/windows/latest.dat'
+
+    now = datetime.datetime.utcnow()
+    backup_path = dist_path / (
+        'latest-windows-%s.dat' % now.strftime('%Y%m%dT%H%M%S'))
+    print('backing up %s to %s' % (latest_dat_path, backup_path))
+
+    with sftp.open(latest_dat_path, 'rb') as fh:
+        latest_dat_old = fh.read()
+
+    with backup_path.open('wb') as fh:
+        fh.write(latest_dat_old)
+
+    print('writing %s with content:' % latest_dat_path)
+    latest_dat_content = generate_latest_dat(version)
+    print(latest_dat_content)
+
+    with sftp.open(latest_dat_path, 'wb') as fh:
+        fh.write(latest_dat_content.encode('ascii'))
+
+
+def publish_artifacts(dist_path: pathlib.Path, version: str,
+                      pypi=True, mercurial_scm_org=True,
+                      ssh_username=None):
+    """Publish Windows release artifacts.
+
+    Files are found in `dist_path`. We will look for files with version string
+    `version`.
+
+    `pypi` controls whether we upload to PyPI.
+    `mercurial_scm_org` controls whether we upload to mercurial-scm.org.
+    """
+    if pypi:
+        publish_artifacts_pypi(dist_path, version)
+
+    if mercurial_scm_org:
+        publish_artifacts_mercurial_scm_org(dist_path, version,
+                                            ssh_username=ssh_username)
diff --git a/contrib/automation/hgautomation/pypi.py b/contrib/automation/hgautomation/pypi.py
new file mode 100644
--- /dev/null
+++ b/contrib/automation/hgautomation/pypi.py
@@ -0,0 +1,25 @@
+# pypi.py - Automation around PyPI
+#
+# Copyright 2019 Gregory Szorc <gregory.szorc at gmail.com>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+# no-check-code because Python 3 native.
+
+from twine.commands.upload import (
+    upload as twine_upload,
+)
+from twine.settings import (
+    Settings,
+)
+
+
+def upload(paths):
+    """Upload files to PyPI.
+
+    `paths` is an iterable of `pathlib.Path`.
+    """
+    settings = Settings()
+
+    twine_upload(settings, [str(p) for p in paths])
diff --git a/contrib/automation/hgautomation/cli.py b/contrib/automation/hgautomation/cli.py
--- a/contrib/automation/hgautomation/cli.py
+++ b/contrib/automation/hgautomation/cli.py
@@ -185,6 +185,14 @@
                           test_flags)
 
 
+def publish_windows_artifacts(hg: HGAutomation, aws_region, version: str,
+                              pypi: bool, mercurial_scm_org: bool,
+                              ssh_username: str):
+    windows.publish_artifacts(DIST_PATH, version,
+                              pypi=pypi, mercurial_scm_org=mercurial_scm_org,
+                              ssh_username=ssh_username)
+
+
 def get_parser():
     parser = argparse.ArgumentParser()
 
@@ -403,6 +411,34 @@
     )
     sp.set_defaults(func=run_tests_windows)
 
+    sp = subparsers.add_parser(
+        'publish-windows-artifacts',
+        help='Publish built Windows artifacts (wheels, installers, etc)'
+    )
+    sp.add_argument(
+        '--no-pypi',
+        dest='pypi',
+        action='store_false',
+        default=True,
+        help='Skip uploading to PyPI',
+    )
+    sp.add_argument(
+        '--no-mercurial-scm-org',
+        dest='mercurial_scm_org',
+        action='store_false',
+        default=True,
+        help='Skip uploading to www.mercurial-scm.org',
+    )
+    sp.add_argument(
+        '--ssh-username',
+        help='SSH username for mercurial-scm.org',
+    )
+    sp.add_argument(
+        'version',
+        help='Mercurial version string to locate local packages',
+    )
+    sp.set_defaults(func=publish_windows_artifacts)
+
     return parser
 
 
diff --git a/contrib/automation/README.rst b/contrib/automation/README.rst
--- a/contrib/automation/README.rst
+++ b/contrib/automation/README.rst
@@ -181,3 +181,25 @@
 Documenting them is beyond the scope of this document. Various tests
 also require other optional dependencies and missing dependencies will
 be printed by the test runner when a test is skipped.
+
+Releasing Windows Artifacts
+===========================
+
+The `automation.py` script can be used to automate the release of Windows
+artifacts::
+
+   $ ./automation.py build-all-windows-packages --revision 5.1.1
+   $ ./automation.py publish-windows-artifacts 5.1.1
+
+The first command will launch an EC2 instance to build all Windows packages
+and copy them into the `dist` directory relative to the repository root. The
+second command will then attempt to upload these files to PyPI (via `twine`)
+and to `mercurial-scm.org` (via SSH).
+
+Uploading to PyPI requires a PyPI account with write access to the `Mercurial`
+package. You can skip PyPI uploading by passing `--no-pypi`.
+
+Uploading to `mercurial-scm.org` requires an SSH account on that server
+with `windows` group membership and for the SSH key for that account to be the
+default SSH key (e.g. `~/.ssh/id_rsa`) or in a running SSH agent. You can
+skip `mercurial-scm.org` uploading by passing `--no-mercurial-scm-org`.



To: indygreg, #hg-reviewers, pulkit
Cc: mercurial-devel


More information about the Mercurial-devel mailing list