[PATCH 3 of 3 V2] reimplement posixfile for Windows in win32.py by using ctypes

Adrian Buehlmann adrian at cadifra.com
Mon May 16 07:23:36 CDT 2011


# HG changeset patch
# User Adrian Buehlmann <adrian at cadifra.com>
# Date 1305488041 -7200
# Node ID 6c6cd3b4e9c61adb7dabf240b4b46c7e64a671d3
# Parent  f9e6b4151691b7459a67aef98910a13c423e6c76
reimplement posixfile for Windows in win32.py by using ctypes

- obsoletes the C implementation in osutil.c
- eliminates dependency on CPython
- makes posixfile available for pure Python (e.g. pypy)

Why is posixfile now a class?

Because the implementation needs to use the Python library call os.fdopen [1],
which sets the 'name' attribute on the Python file object it creates to the
mostly meaningless string '<fdopen>', since file descriptors don't have a name.

But users of posixfile depend on the name attribute [2] being set to a proper
value, like Python's built-in 'open' function sets it on file objects.

Python file's name attribute is read-only, so we can't just assign to it after
the file object has alrady been created.

To solve this problem, we save the name of the file on a wrapper object,
and delegate the file function calls to the wrapped (private) file object
using __getattr__.

[1] http://docs.python.org/library/os.html#os.fdopen
[2] http://docs.python.org/library/stdtypes.html#file.name

diff --git a/mercurial/osutil.c b/mercurial/osutil.c
--- a/mercurial/osutil.c
+++ b/mercurial/osutil.c
@@ -401,119 +401,6 @@
 	return _listdir(path, plen, wantstat, skip);
 }
 
-#ifdef _WIN32
-static PyObject *posixfile(PyObject *self, PyObject *args, PyObject *kwds)
-{
-	static char *kwlist[] = {"name", "mode", "buffering", NULL};
-	PyObject *file_obj = NULL;
-	char *name = NULL;
-	char *mode = "rb";
-	DWORD access = 0;
-	DWORD creation;
-	HANDLE handle;
-	int fd, flags = 0;
-	int bufsize = -1;
-	char m0, m1, m2;
-	char fpmode[4];
-	int fppos = 0;
-	int plus;
-	FILE *fp;
-
-	if (!PyArg_ParseTupleAndKeywords(args, kwds, "et|si:posixfile", kwlist,
-					 Py_FileSystemDefaultEncoding,
-					 &name, &mode, &bufsize))
-		return NULL;
-
-	m0 = mode[0];
-	m1 = m0 ? mode[1] : '\0';
-	m2 = m1 ? mode[2] : '\0';
-	plus = m1 == '+' || m2 == '+';
-
-	fpmode[fppos++] = m0;
-	if (m1 == 'b' || m2 == 'b') {
-		flags = _O_BINARY;
-		fpmode[fppos++] = 'b';
-	}
-	else
-		flags = _O_TEXT;
-	if (m0 == 'r' && !plus) {
-		flags |= _O_RDONLY;
-		access = GENERIC_READ;
-	} else {
-		/*
-		work around http://support.microsoft.com/kb/899149 and
-		set _O_RDWR for 'w' and 'a', even if mode has no '+'
-		*/
-		flags |= _O_RDWR;
-		access = GENERIC_READ | GENERIC_WRITE;
-		fpmode[fppos++] = '+';
-	}
-	fpmode[fppos++] = '\0';
-
-	switch (m0) {
-	case 'r':
-		creation = OPEN_EXISTING;
-		break;
-	case 'w':
-		creation = CREATE_ALWAYS;
-		break;
-	case 'a':
-		creation = OPEN_ALWAYS;
-		flags |= _O_APPEND;
-		break;
-	default:
-		PyErr_Format(PyExc_ValueError,
-			     "mode string must begin with one of 'r', 'w', "
-			     "or 'a', not '%c'", m0);
-		goto bail;
-	}
-
-	handle = CreateFile(name, access,
-			    FILE_SHARE_READ | FILE_SHARE_WRITE |
-			    FILE_SHARE_DELETE,
-			    NULL,
-			    creation,
-			    FILE_ATTRIBUTE_NORMAL,
-			    0);
-
-	if (handle == INVALID_HANDLE_VALUE) {
-		PyErr_SetFromWindowsErrWithFilename(GetLastError(), name);
-		goto bail;
-	}
-
-	fd = _open_osfhandle((intptr_t)handle, flags);
-
-	if (fd == -1) {
-		CloseHandle(handle);
-		PyErr_SetFromErrnoWithFilename(PyExc_IOError, name);
-		goto bail;
-	}
-#ifndef IS_PY3K
-	fp = _fdopen(fd, fpmode);
-	if (fp == NULL) {
-		_close(fd);
-		PyErr_SetFromErrnoWithFilename(PyExc_IOError, name);
-		goto bail;
-	}
-
-	file_obj = PyFile_FromFile(fp, name, mode, fclose);
-	if (file_obj == NULL) {
-		fclose(fp);
-		goto bail;
-	}
-
-	PyFile_SetBufSize(file_obj, bufsize);
-#else
-	file_obj = PyFile_FromFd(fd, name, mode, bufsize, NULL, NULL, NULL, 1);
-	if (file_obj == NULL)
-		goto bail;
-#endif
-bail:
-	PyMem_Free(name);
-	return file_obj;
-}
-#endif
-
 #ifdef __APPLE__
 #include <ApplicationServices/ApplicationServices.h>
 
@@ -535,11 +422,6 @@
 static PyMethodDef methods[] = {
 	{"listdir", (PyCFunction)listdir, METH_VARARGS | METH_KEYWORDS,
 	 "list a directory\n"},
-#ifdef _WIN32
-	{"posixfile", (PyCFunction)posixfile, METH_VARARGS | METH_KEYWORDS,
-	 "Open a file with POSIX-like semantics.\n"
-"On error, this function may raise either a WindowsError or an IOError."},
-#endif
 #ifdef __APPLE__
 	{
 		"isgui", (PyCFunction)isgui, METH_NOARGS,
diff --git a/mercurial/pure/osutil.py b/mercurial/pure/osutil.py
--- a/mercurial/pure/osutil.py
+++ b/mercurial/pure/osutil.py
@@ -8,8 +8,6 @@
 import os
 import stat as statmod
 
-posixfile = open
-
 def _mode_to_kind(mode):
     if statmod.S_ISREG(mode):
         return statmod.S_IFREG
diff --git a/mercurial/win32.py b/mercurial/win32.py
--- a/mercurial/win32.py
+++ b/mercurial/win32.py
@@ -6,12 +6,21 @@
 # GNU General Public License version 2 or any later version.
 
 import encoding
-import ctypes, errno, os, struct, subprocess, random
+import ctypes, ctypes.util, errno, os, struct, subprocess, random
 
 _kernel32 = ctypes.windll.kernel32
 _advapi32 = ctypes.windll.advapi32
 _user32 = ctypes.windll.user32
 
+def _crtname():
+    try:
+        # find_msvcrt was introduced in Python 2.6
+        return ctypes.util.find_msvcrt()
+    except AttributeError:
+        return 'msvcr80.dll' # CPython 2.5
+
+_crt = ctypes.PyDLL(_crtname())
+
 _BOOL = ctypes.c_long
 _WORD = ctypes.c_ushort
 _DWORD = ctypes.c_ulong
@@ -58,7 +67,20 @@
 _FILE_SHARE_WRITE = 0x00000002
 _FILE_SHARE_DELETE = 0x00000004
 
+_CREATE_ALWAYS = 2
 _OPEN_EXISTING = 3
+_OPEN_ALWAYS = 4
+
+_GENERIC_READ = 0x80000000
+_GENERIC_WRITE = 0x40000000
+
+# _open_osfhandle
+_O_RDONLY = 0x0000
+_O_RDWR = 0x0002
+_O_APPEND = 0x0008
+
+_O_TEXT = 0x4000
+_O_BINARY = 0x8000
 
 # SetFileAttributes
 _FILE_ATTRIBUTE_NORMAL = 0x80
@@ -201,10 +223,17 @@
 _user32.EnumWindows.argtypes = [_WNDENUMPROC, _LPARAM]
 _user32.EnumWindows.restype = _BOOL
 
+_crt._open_osfhandle.argtypes = [_HANDLE, ctypes.c_int]
+_crt._open_osfhandle.restype = ctypes.c_int
+
 def _raiseoserror(name):
     err = ctypes.WinError()
     raise OSError(err.errno, '%s: %s' % (name, err.strerror))
 
+def _raiseioerror(name):
+    err = ctypes.WinError()
+    raise IOError(err.errno, '%s: %s' % (name, err.strerror))
+
 def _getfileinfo(name):
     fh = _kernel32.CreateFileA(name, 0,
             _FILE_SHARE_READ | _FILE_SHARE_WRITE | _FILE_SHARE_DELETE,
@@ -447,3 +476,75 @@
     os.mkdir(path)
     if notindexed:
         _kernel32.SetFileAttributesA(path, _FILE_ATTRIBUTE_NOT_CONTENT_INDEXED)
+
+class posixfile(object):
+    '''a file object aiming for POSIX-like semantics
+
+    CPython's open() returns a file that was opened *without* setting the
+    _FILE_SHARE_DELETE flag, which causes rename and unlink to abort.
+    This even happens if any hardlinked copy of the file is in open state.
+    We set _FILE_SHARE_DELETE here, so files opened with posixfile can be
+    renamed and deleted while they are held open.
+    Note that if a file opened with posixfile is unlinked, the file
+    remains but cannot be opened again or be recreated under the same name,
+    until all reading processes have closed the file.'''
+
+    def __init__(self, name, mode='r', bufsize=-1):
+        if 'b' in mode:
+            flags = _O_BINARY
+        else:
+            flags = _O_TEXT
+
+        m0 = mode[0]
+        if m0 == 'r' and not '+' in mode:
+            flags |= _O_RDONLY
+            access = _GENERIC_READ
+        else:
+            # work around http://support.microsoft.com/kb/899149 and
+            # set _O_RDWR for 'w' and 'a', even if mode has no '+'
+            flags |= _O_RDWR
+            access = _GENERIC_READ | _GENERIC_WRITE
+
+        if m0 == 'r':
+            creation = _OPEN_EXISTING
+        elif m0 == 'w':
+            creation = _CREATE_ALWAYS
+        elif m0 == 'a':
+            creation = _OPEN_ALWAYS
+            flags |= _O_APPEND
+        else:
+            raise ValueError("invalid mode: %s" % mode)
+
+        fh = _kernel32.CreateFileA(name, access,
+                _FILE_SHARE_READ | _FILE_SHARE_WRITE | _FILE_SHARE_DELETE,
+                None, creation, _FILE_ATTRIBUTE_NORMAL, None)
+        if fh == _INVALID_HANDLE_VALUE:
+            _raiseioerror(name)
+
+        # for CPython we must use the same CRT as Python uses,
+        # or the os.fdopen call below will abort with
+        #   "OSError: [Errno 9] Bad file descriptor"
+        fd = _crt._open_osfhandle(fh, flags)
+        if fd == -1:
+            _kernel32.CloseHandle(fh)
+            _raiseioerror(name)
+
+        f = os.fdopen(fd, mode, bufsize)
+        # unfortunately, f.name is '<fdopen>' at this point -- so we store
+        # the name on this wrapper. We cannot just assign to f.name,
+        # because that attribute is read-only.
+        object.__setattr__(self, 'name', name)
+        object.__setattr__(self, '_file', f)
+
+    def __iter__(self):
+        return self._file
+
+    def __getattr__(self, name):
+        return getattr(self._file, name)
+
+    def __setattr__(self, name, value):
+        '''mimics the read-only attributes of Python file objects
+        by raising 'TypeError: readonly attribute' if someone tries:
+          f = posixfile('foo.txt')
+          f.name = 'bla'  '''
+        return self._file.__setattr__(name, value)
diff --git a/mercurial/windows.py b/mercurial/windows.py
--- a/mercurial/windows.py
+++ b/mercurial/windows.py
@@ -12,14 +12,6 @@
 nulldev = 'NUL:'
 umask = 002
 
-# wrap osutil.posixfile to provide friendlier exceptions
-def posixfile(name, mode='r', buffering=-1):
-    try:
-        return osutil.posixfile(name, mode, buffering)
-    except WindowsError, err:
-        raise IOError(err.errno, '%s: %s' % (name, err.strerror))
-posixfile.__doc__ = osutil.posixfile.__doc__
-
 class winstdout(object):
     '''stdout on windows misbehaves if sent through a pipe'''
 


More information about the Mercurial-devel mailing list