[PATCH audit RFC] audit: add core audit module, extension using it, and a minimal test
Mike Edgar
adgar at google.com
Wed Dec 2 17:38:42 UTC 2015
# HG changeset patch
# User Mike Edgar <adgar at google.com>
# Date 1448701927 18000
# Sat Nov 28 04:12:07 2015 -0500
# Node ID 2a3144c7158e2f80a9ea7ce7da026c1492ffa9e6
# Parent 389b9907470c61bc502a4e78724aa9b336d81cf6
audit: add core audit module, extension using it, and a minimal test
This is the first piece of implementing audit trails/chain-of-custody tracking
in core Mercurial, providing minimal core functionality and and an extension
for accessing that core functionality.
For more details on the audit trail design, see:
https://www.mercurial-scm.org/wiki/AuditTrailPlan
diff -r 389b9907470c -r 2a3144c7158e hgext/audit.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/hgext/audit.py Sat Nov 28 04:12:07 2015 -0500
@@ -0,0 +1,138 @@
+# Copyright (C) 2015 - Mike Edgar <adgar at google.com>
+#
+# This extension enables direct manipulation of mercurial's built-in audit trail
+# metadata.
+
+"""view and edit changeset audit trail"""
+
+from mercurial import audit as auditmod
+from mercurial import lock as lockmod
+from mercurial import cmdutil, context, error, filelog, obsolete, revlog
+from mercurial import scmutil, util
+from mercurial.i18n import _
+
+cmdtable = {}
+command = cmdutil.command(cmdtable)
+# Note for extension authors: ONLY specify testedwith = 'internal' for
+# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
+# be specifying the version(s) of Mercurial they are tested with, or
+# leave the attribute unspecified.
+testedwith = 'internal'
+
+ at command('audit',
+ [('r', 'rev', '', _('revision to audit'), _('REV'))],
+ _('-r REV'))
+def audit(ui, repo, rev='', **opts):
+ """directly read a revision's audit trail"""
+ rev, ctx = _getrev(repo, rev)
+ sigs = auditmod.extractsigs(ctx.extra())
+ for (num, sigkey, sigvalue) in sigs:
+ atts = auditmod.decodesig(sigvalue)
+ ui.status(_('signature %s\n%s\n%s\n') % (
+ num,
+ '-' * len('signature %s' % num),
+ ''.join('%s=%s\n' % (k, atts[k]) for k in sorted(atts))))
+
+ at command('debugaudit',
+ [('r', 'rev', '', _('revision to audit'), _('REV')),
+ ('u', 'user', '', _('recorded user in audit trail'), _('USER')),
+ ('d', 'date', '', _('recorded date in audit trail'), _('DATE')),
+ ('a', 'attest', [], _('attestation(s) to add (use \'key=value\')'),
+ _('ATTESTATION')),
+ ('', 'delete', [], _('attestation key(s) to delete'), _('KEY'))],
+ _('-r REV [-u USER] [-d DATE] [-a ATTESTATION]... [--delete KEY]...'))
+def debugaudit(ui, repo, rev='', user='', date='', attest=None, delete=None,
+ **opts):
+ """directly add or delete attestations in a changeset's audit trail"""
+ rev, ctx = _getrev(repo, rev)
+ attest = _checkattest(attest or [])
+ delete = _checkdelete(delete or [])
+ user = user or ui.username()
+ date = util.parsedate(date) if date else util.makedate()
+
+ # Compute the new extra dict for the changeset.
+ extra = ctx.extra().copy()
+ sigs = auditmod.extractsigs(extra)
+ atts = {}
+ if sigs:
+ atts = auditmod.decodesig(sigs[-1][2])
+
+ # Now, apply input options to atts
+ for a in attest:
+ k, eq, v = a.partition('=')
+ atts[k] = v
+ for d in delete:
+ del atts[d]
+ extra.update(auditmod.audit('debugaudit', ctx, user, date, atts))
+ if extra == ctx.extra():
+ ui.status(_('no changes\n'))
+ return 1
+
+ # Having computed the new extra dict, do an amend with the new extra.
+ def filectxfn(repo, memctx, path):
+ fctx = ctx[path]
+ copied = fctx.renamed()
+ if copied:
+ copied = copied[0]
+ return context.memfilectx(repo, path, fctx.data(),
+ islink=fctx.islink(), isexec=fctx.isexec(),
+ copied=copied, memctx=memctx)
+ new = context.memctx(repo,
+ parents=[ctx.p1().node(), ctx.p2().node()],
+ text=ctx.description(),
+ files=ctx.files(),
+ filectxfn=filectxfn,
+ user=user,
+ date=date,
+ extra=extra)
+
+ wlock = lock = newid = None
+ try:
+ wlock = repo.wlock()
+ lock = repo.lock()
+ tr = repo.transaction('debugaudit')
+ try:
+ oldid = ctx.node()
+ newid = repo.commitctx(new)
+ if newid != oldid:
+ ps = [c.node() for c in repo.parents()]
+ if oldid in ps:
+ repo.setparents(*[newid if p == oldid else p for p in ps])
+ if obsolete.isenabled(repo, obsolete.createmarkersopt):
+ # mark the new changeset as successor of the rewritten one
+ new = repo[newid]
+ obs = [(ctx, (new,))]
+ obsolete.createmarkers(repo, obs)
+ tr.close()
+ finally:
+ tr.release()
+ finally:
+ lockmod.release(lock, wlock)
+
+def _getrev(repo, rev):
+ if not rev:
+ raise error.Abort(_('must specify revision to censor'))
+
+ rev = scmutil.revsingle(repo, rev, rev).rev()
+ try:
+ return rev, repo[rev]
+ except KeyError:
+ raise error.Abort(_('invalid revision identifier %s') % rev)
+
+def _checkattest(attest):
+ for a in attest:
+ if a.find('=') < 0:
+ raise error.Abort(_('attestation missing equals sign: %s') % a)
+ elif a.find('=') == 0:
+ raise error.Abort(_('attestation missing key: %s') % a)
+ elif a[:a.find('=')] in auditmod.required_atts:
+ raise error.Abort(_('cannot set builtin attestation: %s') % a)
+ return attest
+
+def _checkdelete(delete):
+ for d in delete:
+ if '=' in d:
+ raise error.Abort(_('invalid key contains equals sign: %s') % d)
+ elif d in auditmod.required_atts:
+ raise error.Abort(_('cannot delete builtin attestation: %s') % d)
+ return delete
diff -r 389b9907470c -r 2a3144c7158e mercurial/audit.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/mercurial/audit.py Sat Nov 28 04:12:07 2015 -0500
@@ -0,0 +1,98 @@
+# audit.py - encoding changeset chain of custody in extra
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+"""
+This module serves to implement the data formatting machinery employed by other
+modules to produce and consume audit log data.
+"""
+
+import re
+
+from mercurial.node import hex
+from mercurial import error, ui, util
+from mercurial.i18n import _
+
+# hg.custodian: required attestation. The creator of an audit entry. RFC822.
+att_custodian = 'hg.custodian'
+# hg.date: required attestation. The date of the last change to the audit entry.
+# Same format as in changelogs (see changelog.py).
+att_date = 'hg.date'
+# hg.link: required attestation. For the Nth signature, link is the SHA of a
+# changeset with N-1 matching signatures. For sig0, hg.link is always nullid.
+# The linked changeset is only needed for full audit verification.
+att_source = 'hg.link'
+# Required builtin attestations, used for validation. Extensions might augment
+# this list to enforce local policy.
+required_atts = [att_custodian, att_date, att_source]
+
+# Valid characters in an attestation name.
+att_chars = set(
+ 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789.-')
+
+sigkey_pat = re.compile('^sig\d+$')
+
+def enabled(ui, source):
+ """Check if a given audit log source has auditing enabled in config."""
+ return (ui.configbool('experimental', 'audit-all', False) or
+ ui.configbool('experimental', 'audit-' + source, False))
+
+def audit(source, ctx, user, date, atts):
+ if not enabled(ctx.repo().ui, source):
+ return {} # not enabled
+ if not atts:
+ atts = {}
+ atts = atts.copy() # copy because we will add our own attestations later
+
+ # Signatures are linearly ordered from 1 (maybe 0)
+ sigs = extractsigs(ctx.extra())
+ if sigs:
+ oldsig = sigs[-1][1]
+ newsig = 'sig%d' % (sigs[-1][0]+1,)
+ lastuser = decodesig(sigs[-1][2])[att_custodian]
+ else:
+ oldsig = 'sig0'
+ newsig = 'sig1'
+ lastuser = ctx.user()
+
+ if user == lastuser and not atts:
+ return {} # custodian didn't change, no attestations. don't update.
+
+ if user == lastuser:
+ sig = oldsig
+ else:
+ sig = newsig
+ if user != lastuser or not sigs:
+ atts[att_source] = hex(ctx.node())
+
+ # if custodian user changed add new signature, else update old signature.
+ atts[att_custodian] = user
+ # unconditionally add hg.date attestation to timestamp the audit entry.
+ atts[att_date] = '%d %d' % date
+
+ verifyatts(atts)
+ return {sig: encodesig(atts)}
+
+def extractsigs(extra):
+ return sorted((int(k[3:]), k, v)
+ for (k, v) in extra.iteritems() if sigkey_pat.match(k))
+
+def decodesig(b):
+ atts = {}
+ for attline in b.splitlines():
+ k, colon, v = attline.partition(':')
+ atts[k] = v
+ return atts
+
+def encodesig(atts):
+ return ''.join('%s:%s\n' % (k, v) for (k, v) in sorted(atts.iteritems()))
+
+def verifyatts(atts):
+ for att in required_atts:
+ if att not in atts:
+ raise error.Abort(_('audit attestations with no %s key') % att)
+ for att in atts.iterkeys():
+ if any(c not in att_chars for c in att):
+ badchars = ''.join(sorted(c for c in att if c not in att_chars))
+ raise error.Abort(_('attestation %s has invalid characters: %s') % (
+ att, '[%s]' % badchars))
diff -r 389b9907470c -r 2a3144c7158e tests/test-audit.t
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-audit.t Sat Nov 28 04:12:07 2015 -0500
@@ -0,0 +1,244 @@
+ $ cat >> $HGRCPATH <<EOF
+ > [extensions]
+ > audit=
+ > rebase=
+ > [alias]
+ > debugaudit=debugaudit -d '0 0'
+ > [experimental]
+ > evolution=createmarkers
+ > audit-debugaudit=True
+ > EOF
+ $ USER1='Test User <test at example.net>'
+ $ USER2='Foo User <foo at example.org>'
+ $ HGUSER="$USER1"
+ $ export HGUSER
+
+Create repo and perform a rebase with a different user
+
+ $ hg init r
+ $ cd r
+ $ echo init > f
+ $ hg add f
+ $ hg ci -m init
+
+ $ echo v2 > f2
+ $ hg add f2
+ $ hg ci -m two
+ $ V2NODE=`hg id -q`
+ $ hg update .~1
+ 0 files updated, 0 files merged, 1 files removed, 0 files unresolved
+ $ echo v2b > f2b
+ $ hg add f2b
+ $ hg ci -m tobe
+ created new head
+ $ hg rebase -s $V2NODE -d .
+ rebasing 1:6e7218dcac9d "two"
+ $ hg id -r tip
+ 8940c25c8bf0 tip
+
+Use debugaudit to add arbitrary attestations for testing
+
+ $ hg debugaudit -a tested=yes -u "$USER2" -r tip
+
+Look at new audit log in modified changeset
+
+ $ hg id -r tip
+ b56c8275b93c tip
+ $ hg audit -r tip
+ signature 1
+ -----------
+ hg.custodian=Foo User <foo at example.org>
+ hg.date=0 0
+ hg.link=8940c25c8bf083973a903d690b2d4a27495c2c65
+ tested=yes
+
+
+Adding more attestations works with -a flag
+
+ $ hg debugaudit -a linted=yes -a tested.manual=yes -u "$USER2" -r tip
+ $ hg audit -r tip
+ signature 1
+ -----------
+ hg.custodian=Foo User <foo at example.org>
+ hg.date=0 0
+ hg.link=8940c25c8bf083973a903d690b2d4a27495c2c65
+ linted=yes
+ tested=yes
+ tested.manual=yes
+
+
+Adding more attestations, updating old attestations works with -a flag
+
+ $ hg debugaudit -a linted=twice -u "$USER2" -r tip
+ $ hg audit -r tip
+ signature 1
+ -----------
+ hg.custodian=Foo User <foo at example.org>
+ hg.date=0 0
+ hg.link=8940c25c8bf083973a903d690b2d4a27495c2c65
+ linted=twice
+ tested=yes
+ tested.manual=yes
+
+
+Removing attestation works with --delete flag
+
+ $ hg debugaudit --delete linted --delete tested -u "$USER2" -r tip
+ $ hg audit -r tip
+ signature 1
+ -----------
+ hg.custodian=Foo User <foo at example.org>
+ hg.date=0 0
+ hg.link=8940c25c8bf083973a903d690b2d4a27495c2c65
+ tested.manual=yes
+
+
+Test changing attestations for the working copy parent
+
+ $ hg update -q tip
+ $ hg debugaudit -a tested=2 -u "$USER2" -r tip
+ $ hg audit -r tip
+ signature 1
+ -----------
+ hg.custodian=Foo User <foo at example.org>
+ hg.date=0 0
+ hg.link=8940c25c8bf083973a903d690b2d4a27495c2c65
+ tested=2
+ tested.manual=yes
+
+
+Changing attestation with a different user starts a new audit entry
+
+ $ hg debugaudit -a linted=thrice -u "$USER1" -r tip
+ $ hg audit -r tip
+ signature 1
+ -----------
+ hg.custodian=Foo User <foo at example.org>
+ hg.date=0 0
+ hg.link=8940c25c8bf083973a903d690b2d4a27495c2c65
+ tested=2
+ tested.manual=yes
+
+ signature 2
+ -----------
+ hg.custodian=Test User <test at example.net>
+ hg.date=0 0
+ hg.link=6a32f1897b5a9727722b7b7140b9912f29780fac
+ linted=thrice
+ tested=2
+ tested.manual=yes
+
+
+Date flag works when updating an audit entry or creating a new one
+
+ $ hg debugaudit -u "$USER1" -d '1 0' -r tip
+ $ hg audit -r tip
+ signature 1
+ -----------
+ hg.custodian=Foo User <foo at example.org>
+ hg.date=0 0
+ hg.link=8940c25c8bf083973a903d690b2d4a27495c2c65
+ tested=2
+ tested.manual=yes
+
+ signature 2
+ -----------
+ hg.custodian=Test User <test at example.net>
+ hg.date=1 0
+ hg.link=6a32f1897b5a9727722b7b7140b9912f29780fac
+ linted=thrice
+ tested=2
+ tested.manual=yes
+
+ $ hg debugaudit -u "$USER2" -d '2 0' -r tip
+ $ hg audit -r tip
+ signature 1
+ -----------
+ hg.custodian=Foo User <foo at example.org>
+ hg.date=0 0
+ hg.link=8940c25c8bf083973a903d690b2d4a27495c2c65
+ tested=2
+ tested.manual=yes
+
+ signature 2
+ -----------
+ hg.custodian=Test User <test at example.net>
+ hg.date=1 0
+ hg.link=6a32f1897b5a9727722b7b7140b9912f29780fac
+ linted=thrice
+ tested=2
+ tested.manual=yes
+
+ signature 3
+ -----------
+ hg.custodian=Foo User <foo at example.org>
+ hg.date=2 0
+ hg.link=1eb1212029d6f2644deebc363b794402b3ea8570
+ linted=thrice
+ tested=2
+ tested.manual=yes
+
+
+Test creating custom attestations by original changeset author, in audit entry 0
+
+ $ cd ..
+ $ hg init r2
+ $ cd r2
+ $ echo f > f
+ $ hg add f
+ $ hg ci -m 'init'
+ $ hg audit -r tip
+ $ hg debugaudit -a 'tested=yes' -r tip
+ $ hg audit -r tip
+ signature 0
+ -----------
+ hg.custodian=Test User <test at example.net>
+ hg.date=0 0
+ hg.link=8f4d710dd79cd833f8b9d3d4d4509135737abdea
+ tested=yes
+
+
+Test disabled by config
+
+ $ cat >> $HGRCPATH <<EOF
+ > [experimental]
+ > audit-debugaudit=False
+ > EOF
+ $ hg debugaudit -a 'tested=repeatedly' -r tip
+ no changes
+ [1]
+
+ $ cat >> $HGRCPATH <<EOF
+ > [experimental]
+ > audit-all=True
+ > EOF
+ $ hg debugaudit -a 'tested=repeatedly' -r tip
+ $ hg audit -r tip
+ signature 0
+ -----------
+ hg.custodian=Test User <test at example.net>
+ hg.date=0 0
+ hg.link=8f4d710dd79cd833f8b9d3d4d4509135737abdea
+ tested=repeatedly
+
+
+Test invalid attestation format, no changes scenario, etc.
+
+ $ hg debugaudit -a '=yes' -u "$USER2" -r tip
+ abort: attestation missing key: =yes
+ [255]
+ $ hg debugaudit -a 'testedisyes' -u "$USER2" -r tip
+ abort: attestation missing equals sign: testedisyes
+ [255]
+ $ hg debugaudit -a 'hg.custodian=lol' -u "$USER2" -r tip
+ abort: cannot set builtin attestation: hg.custodian=lol
+ [255]
+ $ hg debugaudit --delete 'tested=' -u "$USER2" -r tip
+ abort: invalid key contains equals sign: tested=
+ [255]
+ $ hg debugaudit --delete 'hg.date' -u "$USER2" -r tip
+ abort: cannot delete builtin attestation: hg.date
+ [255]
+ $ hg debugaudit -r tip -u "$USER1"
+ no changes
+ [1]
diff -r 389b9907470c -r 2a3144c7158e tests/test-help.t
--- a/tests/test-help.t Sat Nov 28 04:11:57 2015 -0500
+++ b/tests/test-help.t Sat Nov 28 04:12:07 2015 -0500
@@ -245,6 +245,7 @@
disabled extensions:
acl hooks for controlling repository access
+ audit view and edit changeset audit trail
blackbox log repository events to a blackbox for debugging
bugzilla hooks for integrating with the Bugzilla bug tracker
censor erase file content at a given revision
More information about the Mercurial-devel
mailing list