[PATCH 3 of 3 v4] releasenotes: add similarity check function to compare incoming notes
Rishabh Madan
rishabhmadan96 at gmail.com
Tue Jul 18 08:40:20 EDT 2017
# HG changeset patch
# User Rishabh Madan <rishabhmadan96 at gmail.com>
# Date 1500381505 -19800
# Tue Jul 18 18:08:25 2017 +0530
# Node ID 24e52f71afd2f311466e9d54f45a95e5da3a51d1
# Parent 6bfef60b45ba91e170d21ea31af60bb5062007db
releasenotes: add similarity check function to compare incoming notes
It is possible that the incoming note fragments might have some similar content
as the existing release notes. In case of a bug fix, we match for issueNNNN in $
existing notes. For other general cases, it makes use of fuzzywuzzy library to
get a similarity score. If the score is above a certain threshold, we ignore
the fragment otherwise add it. But the score might be misleading for small comm$
messages. So, it uses similarity function if only the length of string (in word$
is above a certain number. The patch also adds tests related to its usage.
But it needs improvement in the sense of combining the incoming notes. We can
use interactive mode for adding the notes. Maybe we can do this if similarity
score is under a certain range.
diff -r 6bfef60b45ba -r 24e52f71afd2 hgext/releasenotes.py
--- a/hgext/releasenotes.py Tue Jul 18 15:57:08 2017 +0530
+++ b/hgext/releasenotes.py Tue Jul 18 18:08:25 2017 +0530
@@ -14,6 +14,7 @@
from __future__ import absolute_import
import errno
+import fuzzywuzzy.fuzz as fuzz
import re
import sys
import textwrap
@@ -45,6 +46,7 @@
]
RE_DIRECTIVE = re.compile('^\.\. ([a-zA-Z0-9_]+)::\s*([^$]+)?$')
+RE_ISSUE = r'\bissue [0-9]{4,6}(?![0-9])\b|\bissue[0-9]{4,6}(?![0-9])\b'
BULLET_SECTION = _('Other Changes')
@@ -91,6 +93,9 @@
This is used to combine multiple sources of release notes together.
"""
for section in other:
+ titlediter = iter(self.titledforsection(section))
+ nontitlediter = iter(self.nontitledforsection(section))
+ merge = mergereleasenotes(section, titlediter, nontitlediter)
for title, paragraphs in other.titledforsection(section):
if self.hastitledinsection(section, title):
# TODO prompt for resolution if different and running in
@@ -99,17 +104,15 @@
(title, section))
continue
- # TODO perform similarity comparison and try to match against
- # existing.
- self.addtitleditem(section, title, paragraphs)
+ if merge.checkmerge(ui, section, paragraphs, title=True):
+ self.addtitleditem(section, title, paragraphs)
for paragraphs in other.nontitledforsection(section):
if paragraphs in self.nontitledforsection(section):
continue
- # TODO perform similarily comparison and try to match against
- # existing.
- self.addnontitleditem(section, paragraphs)
+ if merge.checkmerge(ui, section, paragraphs):
+ self.addnontitleditem(section, paragraphs)
class releasenotessections(object):
def __init__(self, ui, repo=None):
@@ -135,6 +138,81 @@
return None
+class mergereleasenotes(object):
+ def __init__(self, section, titlediter, nontitlediter):
+ self.incoming_points = []
+ self.existing_points = self.gatherexistingnotes(section, titlediter,
+ nontitlediter)
+
+ def gatherexistingnotes(self, section, titlediter, nontitlediter):
+ existing_points = []
+ titled_existing_points = converttostring(titlediter, title=True)
+ nontitled_existing_points = converttostring(nontitlediter)
+ existing_points = titled_existing_points + nontitled_existing_points
+ return existing_points
+
+ def checkmerge(self, ui, section, paragraphs, title=False):
+ incoming_str = converttostring(iter(paragraphs), title)[0]
+ if section == 'fix':
+ issues = re.search(RE_ISSUE, incoming_str, re.IGNORECASE)
+ if issues:
+ issuenumber = issues.group()
+ issuenumber = "".join(issuenumber.split())
+ if any(issuenumber in s for s in self.existing_points):
+ ui.write(_("\"%s\" already exists in notes; "
+ "ignoring\n") % issuenumber)
+ return False
+ else:
+ return True
+ if len(incoming_str.split()) > 10:
+ merge = similaritycheck(incoming_str, self.existing_points)
+ if not merge:
+ ui.write(_("\"%s\" already exists in notes file; "
+ "ignoring\n") % incoming_str)
+ return False
+ else:
+ return True
+ else:
+ return True
+
+def converttostring(iterable, title=False):
+ """
+ Converts paragraph and bullet data to individual strings.
+ """
+ string_list = []
+ if title:
+ for titledparagraphs in iterable:
+ str = ""
+ for paragraphs in titledparagraphs:
+ if isinstance(paragraphs, basestring):
+ continue
+ else:
+ for para in paragraphs:
+ str += ' '.join(para) + ' '
+ string_list.append(str)
+ else:
+ for paragraphs in iterable:
+ str = ""
+ if isinstance(paragraphs[0], basestring):
+ str += ' '.join(paragraphs) + ' '
+ else:
+ for para in paragraphs:
+ str += ' '.join(para) + ' '
+ string_list.append(str)
+ return string_list
+
+def similaritycheck(incoming_str, existingnotes):
+ """
+ Returns true when note fragment can be merged to existing notes.
+ """
+ merge = True
+ for bullet in existingnotes:
+ score = fuzz.token_set_ratio(incoming_str, bullet)
+ if score > 75:
+ merge = False
+ break
+ return merge
+
def getcustomadmonitions(repo):
custom_sections = list()
ctx = repo['.']
@@ -196,7 +274,6 @@
paragraphs.append(pblock['lines'])
else:
break
-
# TODO consider using title as paragraph for more concise notes.
if not paragraphs:
raise error.Abort(_('could not find content for release note '
diff -r 6bfef60b45ba -r 24e52f71afd2 tests/test-releasenotes-formatting.t
--- a/tests/test-releasenotes-formatting.t Tue Jul 18 15:57:08 2017 +0530
+++ b/tests/test-releasenotes-formatting.t Tue Jul 18 18:08:25 2017 +0530
@@ -1,3 +1,5 @@
+#require fuzzywuzzy
+
$ cat >> $HGRCPATH << EOF
> [extensions]
> releasenotes=
diff -r 6bfef60b45ba -r 24e52f71afd2 tests/test-releasenotes-merging.t
--- a/tests/test-releasenotes-merging.t Tue Jul 18 15:57:08 2017 +0530
+++ b/tests/test-releasenotes-merging.t Tue Jul 18 18:08:25 2017 +0530
@@ -1,3 +1,5 @@
+#require fuzzywuzzy
+
$ cat >> $HGRCPATH << EOF
> [extensions]
> releasenotes=
@@ -158,3 +160,122 @@
* this is fix3.
+ $ cd ..
+
+Ignores commit messages containing issueNNNN based on issue number.
+
+ $ hg init simple-fuzzrepo
+ $ cd simple-fuzzrepo
+ $ touch fix1
+ $ hg -q commit -A -l - << EOF
+ > commit 1
+ >
+ > .. fix::
+ >
+ > Resolved issue4567.
+ > EOF
+
+ $ cat >> $TESTTMP/issue-number-notes << EOF
+ > Bug Fixes
+ > =========
+ >
+ > * Fixed issue1234 related to XYZ.
+ >
+ > * Fixed issue4567 related to ABC.
+ >
+ > * Fixed issue3986 related to PQR.
+ > EOF
+
+ $ hg releasenotes -r . $TESTTMP/issue-number-notes
+ "issue4567" already exists in notes; ignoring
+
+ $ cat $TESTTMP/issue-number-notes
+ Bug Fixes
+ =========
+
+ * Fixed issue1234 related to XYZ.
+
+ * Fixed issue4567 related to ABC.
+
+ * Fixed issue3986 related to PQR.
+
+ $ cd ..
+
+Adds short commit messages (words < 10) without
+comparison unless there is an exact match.
+
+ $ hg init tempdir
+ $ cd tempdir
+ $ touch feature1
+ $ hg -q commit -A -l - << EOF
+ > commit 1
+ >
+ > .. feature::
+ >
+ > Adds a new feature 1.
+ > EOF
+
+ $ hg releasenotes -r . $TESTTMP/short-sentence-notes
+
+ $ touch feature2
+ $ hg -q commit -A -l - << EOF
+ > commit 2
+ >
+ > .. feature::
+ >
+ > Adds a new feature 2.
+ > EOF
+
+ $ hg releasenotes -r . $TESTTMP/short-sentence-notes
+ $ cat $TESTTMP/short-sentence-notes
+ New Features
+ ============
+
+ * Adds a new feature 1.
+
+ * Adds a new feature 2.
+
+ $ cd ..
+
+Ignores commit messages based on fuzzy comparison.
+
+ $ hg init fuzznotes
+ $ cd fuzznotes
+ $ touch fix1
+ $ hg -q commit -A -l - << EOF
+ > commit 1
+ >
+ > .. fix::
+ >
+ > This is a fix with another line.
+ > And it is a big one.
+ > EOF
+
+ $ cat >> $TESTTMP/fuzz-ignore-notes << EOF
+ > Bug Fixes
+ > =========
+ >
+ > * Fixed issue4567 by improving X.
+ >
+ > * This is the first line. This is next line with one newline.
+ >
+ > This is another line written after two newlines. This is going to be a big one.
+ >
+ > * This fixes another problem.
+ > EOF
+
+ $ hg releasenotes -r . $TESTTMP/fuzz-ignore-notes
+ "This is a fix with another line. And it is a big one. " already exists in notes file; ignoring
+
+ $ cat $TESTTMP/fuzz-ignore-notes
+ Bug Fixes
+ =========
+
+ * Fixed issue4567 by improving X.
+
+ * This is the first line. This is next line with one newline.
+
+ This is another line written after two newlines. This is going to be a big
+ one.
+
+ * This fixes another problem.
diff -r 6bfef60b45ba -r 24e52f71afd2 tests/test-releasenotes-parsing.t
--- a/tests/test-releasenotes-parsing.t Tue Jul 18 15:57:08 2017 +0530
+++ b/tests/test-releasenotes-parsing.t Tue Jul 18 18:08:25 2017 +0530
@@ -1,3 +1,5 @@
+#require fuzzywuzzy
+
$ cat >> $HGRCPATH << EOF
> [extensions]
> releasenotes=
More information about the Mercurial-devel
mailing list