No subject


Tue Jan 4 04:09:52 UTC 2011


reason for this is that many tools on Windows don't handle long file
names gracefully. Time has passed though and more programs now work,
including all Java programs.

  This extension transparently uses so-called Universal Naming Convention
(UNC) paths which allow 32768 character filenames in Windows.

rev2:
  Addressed some code review
  Added some test cases
  Use pywin32 to implement os.listdir and os.mkdir
  Hide inconsistencies in os.path with UNC at the root of a drive
  Add hg lfn --list and hg lfn --clean to make it easier to deal with
    a working copy if you don't have UNC tools. In particular,
    if a directory tree gets too deep, it's annoying to delete it
    without hg lfn --clean

rev3:
  Correct an issue accessing network shares (\\server\host)
  Update the documentation about hg root naming conventions
  Change hg lfn --list and --clean to hg lfn list and clean and
    print usage if no command is given
  Clean up the tests (combine into one file, separate into functions)
  Wrap os.chdir and os.getcwd, this is the most controversial part
    of the patch in my opinion. We no longer use Windows' notion
    of current directory and instead emulate it internally

diff -r 9f707b297b0f -r 6e72a5a75afc hgext/win32lfn.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/hgext/win32lfn.py	Mon Jan 17 02:24:02 2011 -0500
@@ -0,0 +1,273 @@
+'''Allow manipulating long file names
+
+''Author: Aaron Cohen <aaron at assonance.org>''
+
+=== Overview ===
+
+Allows creating working copy files whose path is longer than 260 characters on
+ Windows (up to ~32768 characters).
+
+Caveats:
+
+ - Some filesystems may have their own pathname restrictions, such as some FAT
+    filesystems. Use NTFS or a newer FAT.
+
+ - cmd.exe has trouble manipulating long pathnames (del, move, rename will all
+    fail). Use powershell.
+
+ - Many legacy Windows programs will have difficulty opening files with long
+    pathnames, though most java and 64-bit programs will work fine.
+
+ - explorer.exe may have trouble manipulating directories with long paths,
+    with dialogs like, "The source file name(s) are larger than is supported
+    by the file system. Try moving to a location which has a shorter path name."
+    To address this, use a tool other than explorer.exe or delete the affected
+    files using :hg:`lfn clean`.
+
+ - Things get more complicated if the root of your repository is more than 244
+    characters long, including directory separators.
+
+    - Firstly, there is no way in Windows to "cd" into a directory that
+    long. As a result, to use hg with the repo, you will have to use
+    :hg:`-R` or :hg:`--repository`.
+
+     - When Mercurial first starts up, it will not be able to find the
+     ".hg" directory in such a repository until this extension is loaded.
+     This implies that this extension must be configured in either the
+     system-wide or user hgrc or mercurial.ini, not the per-repository
+     ".hg/hgrc".
+
+=== Configuration ===
+
+Enable the extension in the configuration file (mercurial.ini)::
+
+    [extensions]
+    win32lfn=
+'''
+
+import __builtin__, os, errno
+
+_errmap = None
+
+from mercurial import util, osutil, error
+from mercurial.i18n import _
+
+_win32 = False
+try:
+    import win32api, win32file, winerror, pywintypes
+
+    _win32 = True
+
+    _errmap = {
+        winerror.ERROR_ALREADY_EXISTS: errno.EEXIST,
+        winerror.ERROR_PATH_NOT_FOUND: errno.ENOENT
+    }
+except ImportError:
+    pass
+
+_uncprefix = "\\\\?\\"
+
+_suppressunc = 0
+
+# UNC filenames require different normalization than mercurial and python want
+def unc(path):
+    global _suppressunc
+    if not _suppressunc:
+        _suppressunc += 1
+        if not path.startswith(_uncprefix) and not path.startswith("\\\\.\\"):
+            path = os.path.abspath(path)
+            # path may now be UNC after abspath
+            if not path.startswith(_uncprefix):
+                if path.startswith("\\\\"):
+                    path = _uncprefix + "UNC\\" + path[2:]
+                else:
+                    path = _uncprefix + path
+        _suppressunc -= 1
+    return path
+
+def wrap(method):
+    def fn(*args, **kwargs):
+        path = unc(args[0])
+        return method(path, *args[1:], **kwargs)
+
+    return fn
+
+def wrap2(method):
+    def fn(*args, **kwargs):
+        src = unc(args[0])
+        dst = unc(args[1])
+        return method(src, dst, *args[2:], **kwargs)
+
+    return fn
+
+# vanilla os.listdir handles UNC ok, but breaks if they're longer than MAX_PATH
+def lfnlistdir(path):
+    path = unc(path)
+    if not os.path.exists(path) or not os.path.isdir(path):
+        return []
+    files = win32file.FindFilesW(os.path.join(path, "*.*"))
+    result = []
+    for f in files:
+        file = f[8]
+        if not file == u".." and not file == u".":
+            result.append(file)
+    return result
+
+# vanilla handles UNC pathes but not if longer than MAX_PATH
+def lfnmkdir(path, mode=None):
+    path = unc(path)
+    try:
+        # second parameter is a security descriptor, mapping it up to our
+        # "mode" parameter is non-trivial and hopefully unnecessary
+        win32file.CreateDirectoryW(path, None)
+    except pywintypes.error, err:
+        if err.winerror in _errmap:
+            pyerrno = _errmap[err.winerror]
+            raise OSError(pyerrno, err.strerror)
+        raise
+
+# vanilla returns a relative path for filenames longer than MAX_PATH
+# os.path.abspath(30 * "123456789\\") -> 30 * "123456789\\"
+def wrapabspath(abspath):
+    def lfnabspath(path):
+        result = path
+        if not os.path.isabs(result):
+            result = os.path.join(os.getcwd(), result)
+        result = os.path.normpath(result)
+        return result
+
+    return lfnabspath
+
+def _addmissingbackslash(path):
+    if path.endswith(":"):
+        path += "\\"
+    return path
+
+# vanilla loses a trailing backslash:
+# os.path.split('\\\\?\\C:\\') -> ('\\\\?\\C:', '')
+def wrapsplit(split):
+    def lfnsplit(path):
+        result = split(path)
+        result = (_addmissingbackslash(result[0]), result[1])
+        return result
+
+    return lfnsplit
+
+# vanilla loses a trailing backslash:
+# os.path.dirname('\\\\?\\C:\\') -> '\\\\?\\C:'
+def wrapdirname(dirname):
+    def lfndirname(path):
+        result = dirname(path)
+        return _addmissingbackslash(result)
+
+    return lfndirname
+
+# Windows API has no SetCurrentDirectory for long paths,
+# so we implement it internally
+# http://social.msdn.microsoft.com/Forums/en/windowsgeneraldevelopmentissues/thread/7998d7ec-cf5a-4b5e-a554-13fa855e4a3d
+def wrapchdir(chdir):
+    def lfnchdir(path):
+        if len(os.path.abspath(path)) >= 248:
+            path = unc(path)
+        if os.path.exists(path):
+            # Use an environment variable so subprocesses get the correct cwd
+            os.environ["CD"] = path
+        else:
+            raise OSError(errno.ENOENT, _("Directory doesn't exist: %s") % path)
+
+    return lfnchdir
+
+def wrapgetcwd(getcwd):
+    def lfngetcwd():
+        if "CD" in os.environ:
+            result = os.environ["CD"]
+        else:
+            result = getcwd()
+            # Should I un-UNC long directories here?
+        return result
+
+    return lfngetcwd
+
+def uisetup(ui):
+    if not _win32:
+        ui.warn(_("This extension requires the pywin32 extensions\n"))
+        return
+    os.listdir = lfnlistdir
+    os.mkdir = lfnmkdir
+    os.path.abspath = wrapabspath(os.path.abspath)
+    os.path.split = wrapsplit(os.path.split)
+    os.path.dirname = wrapdirname(os.path.dirname)
+
+    # No wrapping needed for os.makedirs
+
+    os.chdir = wrapchdir(os.chdir)
+    os.getcwd = wrapgetcwd(os.getcwd)
+
+    os.stat = wrap(os.stat)
+    os.lstat = wrap(os.lstat)
+    os.open = wrap(os.open)
+    os.chmod = wrap(os.chmod)
+    os.remove = wrap(os.remove)
+    os.unlink = wrap(os.unlink)
+    os.rmdir = wrap(os.rmdir)
+    os.removedirs = wrap(os.removedirs)
+    os.rename = wrap2(os.rename)
+    os.renames = wrap2(os.renames)
+    __builtin__.open = wrap(__builtin__.open)
+
+    osutil.listdir = wrap(osutil.listdir)
+    osutil.posixfile = wrap(osutil.posixfile)
+
+    util.posixfile = wrap(util.posixfile)
+    util.makedirs = wrap(util.makedirs)
+    util.rename = wrap2(util.rename)
+    util.copyfile = wrap2(util.copyfile)
+    util.copyfiles = wrap2(util.copyfiles)
+    if hasattr(util, "unlinkpath"):
+        util.unlinkpath = wrap(util.unlinkpath)
+    if hasattr(util, "unlink"):
+        util.unlink = wrap(util.unlink)
+
+def list(ui, repo):
+    for root, _ignored, files in os.walk(repo.root):
+        for file in files:
+            if len(root + file) >= 259:
+                ui.write(os.path.join(root, file) + "\n")
+
+def clean(ui, repo, force=False):
+    for root, _ignored, files in os.walk(repo.root):
+        for file in files:
+            if len(root + file) >= 259:
+                path = os.path.join(root, file)
+                c = ui.promptchoice(_("Delete %s? [N/y]") % path,
+                                    (_("&No"), _("&Yes")), 0)
+                if c or force:
+                    if hasattr(util, "unlink"):
+                        util.unlink(path)
+                    else:
+                        util.unlinkpath(path)
+
+_commands = {
+    'list': list,
+    'clean': clean
+}
+
+def lfn(ui, repo, command):
+    '''Search for or delete files in the working copy that are longer than \
+MAX_PATH (260) characters.
+
+ :hg lfn list: List all files in the repository longer than MAX_PATH
+
+ :hg lfn clean: Prompt to delete all files in the repository longer than
+    MAX_PATH. This may make it easier to deal with such files, since many
+    Windows programs are unable to.'''
+    if command in _commands:
+        _commands[command](ui, repo)
+    else:
+        raise error.SignatureError
+
+cmdtable = {
+    "lfn": (lfn,
+            [],
+            _('list | clean')),
+}
diff -r 9f707b297b0f -r 6e72a5a75afc tests/test-win32lfn.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-win32lfn.py	Mon Jan 17 02:24:02 2011 -0500
@@ -0,0 +1,237 @@
+import os, errno, re
+
+from hgext import win32lfn
+
+win32lfn.uisetup(None)
+
+cwd = os.getcwd()
+
+# 299 chars, mixed slashes
+name = "123456789\\" + 28 * "123456789/" + "123456789"
+
+uncname = win32lfn.unc(name)
+
+convolutedpath = "C:\\d/d\\d/././.\\..\\\\//."
+
+shortpath = "C:\\"
+
+def testCanonization():
+    expectedpath = "\\\\?\\" + cwd + 30 * "\\123456789"
+    assert uncname == expectedpath
+
+    canonpath = win32lfn.unc(convolutedpath)
+    expected = "\\\\?\\C:\\d\\d"
+    assert canonpath == expected
+
+    normpath = os.path.normpath(convolutedpath)
+    expected = "C:\\d\\d"
+    assert normpath == expected
+
+    shortunc = win32lfn.unc(shortpath)
+    expected = "\\\\?\\C:\\"
+    assert shortunc == expected
+
+    assert os.path.dirname(shortunc) == shortunc
+
+    head, tail = os.path.split(shortunc)
+    expected = "\\\\?\\C:\\"
+    assert head == expected, tail == ""
+
+    # Tempting as it is, make sure we don't touch paths that were already UNC
+    assert win32lfn.unc("\\\\?\\" + convolutedpath) == \
+           "\\\\?\\" + convolutedpath
+
+def computetestpaths():
+    parent = os.path.normpath(os.path.join(name, ".."))
+
+    thrown = False
+    try:
+        os.chdir("Blargh")
+    except OSError, err:
+        assert err.errno == errno.ENOENT
+        thrown = True
+    assert thrown
+
+    testpaths = [(parent, "123456789"), (cwd, name), (cwd, uncname)]
+
+    # Verify "\\servername\share\repo" works, using \\localhost\c$
+    # This only works if the test is being run from a local drive, not a
+    # network share
+    index = uncname.find(":")
+    if index != -1:
+        drive, tail = re.findall("\\\\?\\\(.):(.*)", uncname)[0]
+        sharename = r"\\localhost\%s$%s" % (drive, tail)
+        testpaths.append((cwd, sharename))
+    else:
+        print "Skipping \\localhost\c$ tests, cwd is not on a local drive"
+
+    return testpaths
+
+def cleanup(d):
+    if not os.path.exists(d):
+        return
+    for root, dirs, files in os.walk(d):
+        for file in files:
+            f = os.path.join(root, file)
+            if os.path.isfile(f):
+                os.unlink(f)
+        if not dirs:
+            os.removedirs(root)
+
+#for testpath in testpaths:
+def testos(root, d):
+    print "Running os tests for %s, %s" % (root, d)
+    f1 = os.path.join(d, "f1")
+    f2 = os.path.join(d, "f2")
+
+    assert os.path.split(f1) == (d, "f1")
+    assert os.path.dirname(f2) == d
+    assert os.path.basename(f2) == "f2"
+
+    # Make the root if it doesn't exist and chdir into it, to test chdir
+    if not os.path.exists(root):
+        os.makedirs(root)
+    os.chdir(root)
+
+    os.makedirs(d)
+
+    assert os.path.exists(d)
+    assert os.path.isdir(d)
+
+    os.rmdir(d)
+    assert not os.path.exists(d)
+
+    # os.mkdir must raise EEXIST if the directory exists
+    os.mkdir(d)
+    thrown = False
+    try:
+        os.mkdir(d)
+    except OSError, err:
+        thrown = True
+        assert err.errno == errno.EEXIST
+    assert thrown
+
+    f = open(f1, 'w')
+    f.write("Test")
+    f.close()
+
+    f = open(f1, 'r')
+    assert f.readline() == "Test"
+    f.close()
+
+    os.stat(f1)
+    os.lstat(f1)
+    os.chmod(f1, 660)
+    assert os.path.isfile(f1)
+
+    files = os.listdir(d)
+    assert len(files) == 1
+    assert os.path.basename(f1) in files
+
+    os.rename(f1, f2)
+    os.stat(f2)
+    assert not os.path.exists(f1)
+
+    fd = os.open(f1, os.O_CREAT | os.O_BINARY | os.O_RDWR)
+    os.write(fd, "Test2")
+    os.close(fd)
+
+    os.remove(f1)
+    os.unlink(f2)
+    os.removedirs(d)
+    assert not os.path.exists(os.path.join(cwd, "123456789"))
+
+    os.chdir(cwd)
+
+def testutil(root, d):
+    from mercurial import util, osutil
+
+    def _util_unlink(file):
+        if hasattr(util, "unlinkpath"):
+            util.unlinkpath(file)
+        else:
+            util.unlink(file)
+
+    print "Running util tests for %s, %s" % (root, d)
+
+    if not os.path.exists(root):
+        util.makedirs(root)
+    os.chdir(root)
+
+    util.makedirs(d)
+    assert os.path.exists(d)
+
+    f1 = os.path.join(d, "f1")
+    f2 = os.path.join(d, "f2")
+    f3 = os.path.join(d, "f3")
+
+    f = util.posixfile(f1, 'w')
+    f.write("Test")
+    f.close()
+
+    util.copyfile(f1, f2)
+
+    f = osutil.posixfile(f3, 'w')
+    f.write("Test")
+    f.close()
+
+    files = os.listdir(d)
+    assert len(files) == 3
+    assert os.path.basename(f1) in files and\
+           os.path.basename(f2) in files and\
+           os.path.basename(f3) in files
+
+    files = osutil.listdir(d)
+    assert len(files) == 3
+    j = 1
+    for file in files:
+        assert file[0] == "f" + str(j)
+        j += 1
+
+    util.copyfiles("123456789", "d123456789")
+
+    os.remove(f1)
+    os.unlink(f2)
+    _util_unlink(f3)
+    assert not os.path.exists("123456789")
+
+    # All the files still there after the copy?
+    os.rename("d123456789", "123456789")
+    files = os.listdir(d)
+    assert len(files) == 3
+    assert os.path.basename(f1) in files and\
+           os.path.basename(f2) in files and\
+           os.path.basename(f3) in files
+
+    os.remove(f1)
+    os.unlink(f2)
+    _util_unlink(f3)
+    assert not os.path.exists("123456789")
+
+def testclean():
+    print 'Testing "hg lfn clean"'
+    from mercurial import ui, hg, util
+
+    util.makedirs(uncname)
+    f = util.posixfile(os.path.join(uncname, "f"), 'w')
+    f.write("Test")
+    f.close()
+
+    u = ui.ui()
+    r = hg.repository(u, "123456789", create=1)
+
+    win32lfn.clean(u, r, force=True)
+
+    # The project root will still exist but everything else should be gone
+    assert not os.path.exists("123456789/123456789")
+    cleanup("123456789")
+
+
+testCanonization()
+# If the tests get interrupted or fail, this will cleanup
+#cleanup("123456789")
+#cleanup("d123456789")
+for test in computetestpaths():
+    testos(*test)
+    testutil(*test)
+testclean()


More information about the Mercurial-devel mailing list