sync to pytest 2.5.2 and pylib 1.4.20

17 files changed, 659 insertions, 353 deletions

diff --git a/py/__init__.py b/py/__init__.py
index bf6c2128fa..c94f0699c5 100644
--- a/py/__init__.py
+++ b/py/__init__.py
@@ -6,9 +6,9 @@ and classes.  The initpkg-dictionary  below specifies
 name->value mappings where value can be another namespace
 dictionary or an import path.
 
-(c) Holger Krekel and others, 2004-2010
+(c) Holger Krekel and others, 2004-2013
 """
-__version__ = '1.4.7'
+__version__ = '1.4.20'
 
 from py import _apipkg
 
@@ -104,6 +104,8 @@ _apipkg.initpkg(__name__, attr={'_apipkg': _apipkg}, exportdefs={
         'builtins'       : '._builtin:builtins',
         'execfile'       : '._builtin:execfile',
         'callable'       : '._builtin:callable',
+        'bytes'       : '._builtin:bytes',
+        'text'       : '._builtin:text',
     },
 
     # input-output helping
diff --git a/py/_apipkg.py b/py/_apipkg.py
index afd1e6d274..4907f6bfca 100644
--- a/py/_apipkg.py
+++ b/py/_apipkg.py
@@ -9,7 +9,17 @@ import os
 import sys
 from types import ModuleType
 
-__version__ = '1.2.dev6'
+__version__ = '1.3.dev'
+
+def _py_abspath(path):
+    """
+    special version of abspath
+    that will leave paths from jython jars alone
+    """
+    if path.startswith('__pyclasspath__'):
+        return path
+    else:
+        return os.path.abspath(path)
 
 def initpkg(pkgname, exportdefs, attr=dict()):
     """ initialize given package from the export definitions. """
@@ -17,14 +27,14 @@ def initpkg(pkgname, exportdefs, attr=dict()):
     d = {}
     f = getattr(oldmod, '__file__', None)
     if f:
-        f = os.path.abspath(f)
+        f = _py_abspath(f)
     d['__file__'] = f
     if hasattr(oldmod, '__version__'):
         d['__version__'] = oldmod.__version__
     if hasattr(oldmod, '__loader__'):
         d['__loader__'] = oldmod.__loader__
     if hasattr(oldmod, '__path__'):
-        d['__path__'] = [os.path.abspath(p) for p in oldmod.__path__]
+        d['__path__'] = [_py_abspath(p) for p in oldmod.__path__]
     if '__doc__' not in exportdefs and getattr(oldmod, '__doc__', None):
         d['__doc__'] = oldmod.__doc__
     d.update(attr)
@@ -164,4 +174,4 @@ def AliasModule(modname, modpath, attrname=None):
         def __delattr__(self, name):
             delattr(getmod(), name)
 
-    return AliasModule(modname)
+    return AliasModule(str(modname))
diff --git a/py/_builtin.py b/py/_builtin.py
index 67971f0afc..fa3b797c44 100644
--- a/py/_builtin.py
+++ b/py/_builtin.py
@@ -107,6 +107,12 @@ except NameError:
 
 _sysex = (KeyboardInterrupt, SystemExit, MemoryError, GeneratorExit)
 
+try:
+    callable = callable
+except NameError:
+    def callable(obj):
+        return hasattr(obj, "__call__")
+
 if sys.version_info >= (3, 0):
     exec ("print_ = print ; exec_=exec")
     import builtins
@@ -128,6 +134,10 @@ if sys.version_info >= (3, 0):
     def _istext(x):
         return isinstance(x, str)
 
+    text = str
+    bytes = bytes
+
+
     def _getimself(function):
         return getattr(function, '__self__', None)
 
@@ -153,13 +163,12 @@ if sys.version_info >= (3, 0):
         co = compile(source, fn, "exec", dont_inherit=True)
         exec_(co, globs, locs)
 
-    def callable(obj):
-        return hasattr(obj, "__call__")
-
 else:
     import __builtin__ as builtins
     _totext = unicode
     _basestring = basestring
+    text = unicode
+    bytes = str
     execfile = execfile
     callable = callable
     def _isbytes(x):
@@ -231,7 +240,9 @@ def _tryimport(*names):
     assert names
     for name in names:
         try:
-            return __import__(name, None, None, '__doc__')
+            __import__(name)
         except ImportError:
             excinfo = sys.exc_info()
+        else:
+            return sys.modules[name]
     _reraise(*excinfo)
diff --git a/py/_code/_py2traceback.py b/py/_code/_py2traceback.py
new file mode 100644
index 0000000000..d65e27cb73
--- /dev/null
+++ b/py/_code/_py2traceback.py
@@ -0,0 +1,79 @@
+# copied from python-2.7.3's traceback.py
+# CHANGES:
+# - some_str is replaced, trying to create unicode strings
+#
+import types
+
+def format_exception_only(etype, value):
+    """Format the exception part of a traceback.
+
+    The arguments are the exception type and value such as given by
+    sys.last_type and sys.last_value. The return value is a list of
+    strings, each ending in a newline.
+
+    Normally, the list contains a single string; however, for
+    SyntaxError exceptions, it contains several lines that (when
+    printed) display detailed information about where the syntax
+    error occurred.
+
+    The message indicating which exception occurred is always the last
+    string in the list.
+
+    """
+
+    # An instance should not have a meaningful value parameter, but
+    # sometimes does, particularly for string exceptions, such as
+    # >>> raise string1, string2  # deprecated
+    #
+    # Clear these out first because issubtype(string1, SyntaxError)
+    # would throw another exception and mask the original problem.
+    if (isinstance(etype, BaseException) or
+        isinstance(etype, types.InstanceType) or
+        etype is None or type(etype) is str):
+        return [_format_final_exc_line(etype, value)]
+
+    stype = etype.__name__
+
+    if not issubclass(etype, SyntaxError):
+        return [_format_final_exc_line(stype, value)]
+
+    # It was a syntax error; show exactly where the problem was found.
+    lines = []
+    try:
+        msg, (filename, lineno, offset, badline) = value.args
+    except Exception:
+        pass
+    else:
+        filename = filename or "<string>"
+        lines.append('  File "%s", line %d\n' % (filename, lineno))
+        if badline is not None:
+            lines.append('    %s\n' % badline.strip())
+            if offset is not None:
+                caretspace = badline.rstrip('\n')[:offset].lstrip()
+                # non-space whitespace (likes tabs) must be kept for alignment
+                caretspace = ((c.isspace() and c or ' ') for c in caretspace)
+                # only three spaces to account for offset1 == pos 0
+                lines.append('   %s^\n' % ''.join(caretspace))
+        value = msg
+
+    lines.append(_format_final_exc_line(stype, value))
+    return lines
+
+def _format_final_exc_line(etype, value):
+    """Return a list of a single line -- normal case for format_exception_only"""
+    valuestr = _some_str(value)
+    if value is None or not valuestr:
+        line = "%s\n" % etype
+    else:
+        line = "%s: %s\n" % (etype, valuestr)
+    return line
+
+def _some_str(value):
+    try:
+        return unicode(value)
+    except Exception:
+        try:
+            return str(value)
+        except Exception:
+            pass
+    return '<unprintable %s object>' % type(value).__name__
diff --git a/py/_code/code.py b/py/_code/code.py
index 8b65de73ba..aa60da8017 100644
--- a/py/_code/code.py
+++ b/py/_code/code.py
@@ -1,21 +1,28 @@
 import py
-import sys, os.path
+import sys
+from inspect import CO_VARARGS, CO_VARKEYWORDS
 
 builtin_repr = repr
 
 reprlib = py.builtin._tryimport('repr', 'reprlib')
 
+if sys.version_info[0] >= 3:
+    from traceback import format_exception_only
+else:
+    from py._code._py2traceback import format_exception_only
+
 class Code(object):
     """ wrapper around Python code objects """
     def __init__(self, rawcode):
-        rawcode = py.code.getrawcode(rawcode)
-        self.raw = rawcode
+        if not hasattr(rawcode, "co_filename"):
+            rawcode = py.code.getrawcode(rawcode)
         try:
             self.filename = rawcode.co_filename
             self.firstlineno = rawcode.co_firstlineno - 1
             self.name = rawcode.co_name
         except AttributeError:
             raise TypeError("not a code object: %r" %(rawcode,))
+        self.raw = rawcode
 
     def __eq__(self, other):
         return self.raw == other.raw
@@ -23,25 +30,25 @@ class Code(object):
     def __ne__(self, other):
         return not self == other
 
+    @property
     def path(self):
-        """ return a path object pointing to source code"""
+        """ return a path object pointing to source code (note that it
+        might not point to an actually existing file). """
         p = py.path.local(self.raw.co_filename)
+        # maybe don't try this checking
         if not p.check():
             # XXX maybe try harder like the weird logic
             # in the standard lib [linecache.updatecache] does?
             p = self.raw.co_filename
         return p
 
-    path = property(path, None, None, "path of this code object")
-
+    @property
     def fullsource(self):
         """ return a py.code.Source object for the full source file of the code
         """
         from py._code import source
         full, _ = source.findsource(self.raw)
         return full
-    fullsource = property(fullsource, None, None,
-                          "full source containing this code object")
 
     def source(self):
         """ return a py.code.Source object for the code object's source only
@@ -49,30 +56,37 @@ class Code(object):
         # return source only for that part of code
         return py.code.Source(self.raw)
 
-    def getargs(self):
+    def getargs(self, var=False):
         """ return a tuple with the argument names for the code object
+
+            if 'var' is set True also return the names of the variable and
+            keyword arguments when present
         """
         # handfull shortcut for getting args
         raw = self.raw
-        return raw.co_varnames[:raw.co_argcount]
+        argcount = raw.co_argcount
+        if var:
+            argcount += raw.co_flags & CO_VARARGS
+            argcount += raw.co_flags & CO_VARKEYWORDS
+        return raw.co_varnames[:argcount]
 
 class Frame(object):
     """Wrapper around a Python frame holding f_locals and f_globals
     in which expressions can be evaluated."""
 
     def __init__(self, frame):
-        self.code = py.code.Code(frame.f_code)
         self.lineno = frame.f_lineno - 1
         self.f_globals = frame.f_globals
         self.f_locals = frame.f_locals
         self.raw = frame
+        self.code = py.code.Code(frame.f_code)
 
+    @property
     def statement(self):
+        """ statement this frame is at """
         if self.code.fullsource is None:
             return py.code.Source("")
         return self.code.fullsource.getstatement(self.lineno)
-    statement = property(statement, None, None,
-                         "statement this frame is at")
 
     def eval(self, code, **vars):
         """ evaluate 'code' in the frame
@@ -102,11 +116,14 @@ class Frame(object):
     def is_true(self, object):
         return object
 
-    def getargs(self):
+    def getargs(self, var=False):
         """ return a list of tuples (name, value) for all arguments
+
+            if 'var' is set True also include the variable and keyword
+            arguments when present
         """
         retval = []
-        for arg in self.code.getargs():
+        for arg in self.code.getargs(var):
             try:
                 retval.append((arg, self.f_locals[arg]))
             except KeyError:
@@ -120,26 +137,29 @@ class TracebackEntry(object):
 
     def __init__(self, rawentry):
         self._rawentry = rawentry
-        self.frame = py.code.Frame(rawentry.tb_frame)
-        # Ugh. 2.4 and 2.5 differs here when encountering
-        # multi-line statements. Not sure about the solution, but
-        # should be portable
         self.lineno = rawentry.tb_lineno - 1
-        self.relline = self.lineno - self.frame.code.firstlineno
+
+    @property
+    def frame(self):
+        return py.code.Frame(self._rawentry.tb_frame)
+
+    @property
+    def relline(self):
+        return self.lineno - self.frame.code.firstlineno
 
     def __repr__(self):
         return "<TracebackEntry %s:%d>" %(self.frame.code.path, self.lineno+1)
 
+    @property
     def statement(self):
-        """ return a py.code.Source object for the current statement """
+        """ py.code.Source object for the current statement """
         source = self.frame.code.fullsource
         return source.getstatement(self.lineno)
-    statement = property(statement, None, None,
-                         "statement of this traceback entry.")
 
+    @property
     def path(self):
+        """ path to the source code """
         return self.frame.code.path
-    path = property(path, None, None, "path to the full source code")
 
     def getlocals(self):
         return self.frame.f_locals
@@ -160,26 +180,30 @@ class TracebackEntry(object):
         # on Jython this firstlineno can be -1 apparently
         return max(self.frame.code.firstlineno, 0)
 
-    def getsource(self):
+    def getsource(self, astcache=None):
         """ return failing source code. """
+        # we use the passed in astcache to not reparse asttrees
+        # within exception info printing
+        from py._code.source import getstatementrange_ast
         source = self.frame.code.fullsource
         if source is None:
             return None
+        key = astnode = None
+        if astcache is not None:
+            key = self.frame.code.path
+            if key is not None:
+                astnode = astcache.get(key, None)
         start = self.getfirstlinesource()
-        end = self.lineno
         try:
-            _, end = source.getstatementrange(end)
-        except (IndexError, ValueError):
+            astnode, _, end = getstatementrange_ast(self.lineno, source,
+                                                    astnode=astnode)
+        except SyntaxError:
             end = self.lineno + 1
-        # heuristic to stop displaying source on e.g.
-        #   if something:  # assume this causes a NameError
-        #      # _this_ lines and the one
-               #        below we don't want from entry.getsource()
-        for i in range(self.lineno, end):
-            if source[i].rstrip().endswith(':'):
-                end = i + 1
-                break
+        else:
+            if key is not None:
+                astcache[key] = astnode
         return source[start:end]
+
     source = property(getsource)
 
     def ishidden(self):
@@ -189,11 +213,12 @@ class TracebackEntry(object):
             mostly for internal use
         """
         try:
-            return self.frame.eval("__tracebackhide__")
-        except py.builtin._sysex:
-            raise
-        except:
-            return False
+            return self.frame.f_locals['__tracebackhide__']
+        except KeyError:
+            try:
+                return self.frame.f_globals['__tracebackhide__']
+            except KeyError:
+                return False
 
     def __str__(self):
         try:
@@ -272,10 +297,11 @@ class Traceback(list):
         """ return last non-hidden traceback entry that lead
         to the exception of a traceback.
         """
-        tb = self.filter()
-        if not tb:
-            tb = self
-        return tb[-1]
+        for i in range(-1, -len(self)-1, -1):
+            entry = self[i]
+            if not entry.ishidden():
+                return entry
+        return self[-1]
 
     def recursionindex(self):
         """ return the index of the frame/TracebackItem where recursion
@@ -310,8 +336,6 @@ class ExceptionInfo(object):
     """
     _striptext = ''
     def __init__(self, tup=None, exprinfo=None):
-        # NB. all attributes are private!  Subclasses or other
-        #     ExceptionInfo-like classes may have different attributes.
         if tup is None:
             tup = sys.exc_info()
             if exprinfo is None and isinstance(tup[1], AssertionError):
@@ -321,9 +345,16 @@ class ExceptionInfo(object):
                 if exprinfo and exprinfo.startswith('assert '):
                     self._striptext = 'AssertionError: '
         self._excinfo = tup
-        self.type, self.value, tb = self._excinfo
+        #: the exception class
+        self.type = tup[0]
+        #: the exception instance
+        self.value = tup[1]
+        #: the exception raw traceback
+        self.tb = tup[2]
+        #: the exception type name
         self.typename = self.type.__name__
-        self.traceback = py.code.Traceback(tb)
+        #: the exception traceback (py.code.Traceback instance)
+        self.traceback = py.code.Traceback(self.tb)
 
     def __repr__(self):
         return "<ExceptionInfo %s tblen=%d>" % (self.typename, len(self.traceback))
@@ -336,7 +367,7 @@ class ExceptionInfo(object):
             the exception representation is returned (so 'AssertionError: ' is
             removed from the beginning)
         """
-        lines = py.std.traceback.format_exception_only(self.type, self.value)
+        lines = format_exception_only(self.type, self.value)
         text = ''.join(lines)
         text = text.rstrip()
         if tryshort:
@@ -351,9 +382,8 @@ class ExceptionInfo(object):
     def _getreprcrash(self):
         exconly = self.exconly(tryshort=True)
         entry = self.traceback.getcrashentry()
-        path, lineno = entry.path, entry.lineno
-        reprcrash = ReprFileLocation(path, lineno+1, exconly)
-        return reprcrash
+        path, lineno = entry.frame.code.raw.co_filename, entry.lineno
+        return ReprFileLocation(path, lineno+1, exconly)
 
     def getrepr(self, showlocals=False, style="long",
             abspath=False, tbfilter=True, funcargs=False):
@@ -399,6 +429,7 @@ class FormattedExcinfo(object):
         self.tbfilter = tbfilter
         self.funcargs = funcargs
         self.abspath = abspath
+        self.astcache = {}
 
     def _getindent(self, source):
         # figure out indent for given source
@@ -416,7 +447,7 @@ class FormattedExcinfo(object):
         return 4 + (len(s) - len(s.lstrip()))
 
     def _getentrysource(self, entry):
-        source = entry.getsource()
+        source = entry.getsource(self.astcache)
         if source is not None:
             source = source.deindent()
         return source
@@ -427,7 +458,7 @@ class FormattedExcinfo(object):
     def repr_args(self, entry):
         if self.funcargs:
             args = []
-            for argname, argvalue in entry.frame.getargs():
+            for argname, argvalue in entry.frame.getargs(var=True):
                 args.append((argname, self._saferepr(argvalue)))
             return ReprFuncArgs(args)
 
@@ -562,22 +593,16 @@ class TerminalRepr:
         return s
 
     def __unicode__(self):
-        l = []
-        tw = py.io.TerminalWriter(l.append)
+        # FYI this is called from pytest-xdist's serialization of exception
+        # information.
+        io = py.io.TextIO()
+        tw = py.io.TerminalWriter(file=io)
         self.toterminal(tw)
-        l = map(unicode_or_repr, l)
-        return "".join(l).strip()
+        return io.getvalue().strip()
 
     def __repr__(self):
         return "<%s instance at %0x>" %(self.__class__, id(self))
 
-def unicode_or_repr(obj):
-    try:
-        return py.builtin._totext(obj)
-    except KeyboardInterrupt:
-        raise
-    except Exception:
-        return "<print-error: %r>" % py.io.saferepr(obj)
 
 class ReprExceptionInfo(TerminalRepr):
     def __init__(self, reprtraceback, reprcrash):
diff --git a/py/_code/source.py b/py/_code/source.py
index 94c5da7f66..e17bc1cd35 100644
--- a/py/_code/source.py
+++ b/py/_code/source.py
@@ -108,52 +108,12 @@ class Source(object):
     def getstatementrange(self, lineno, assertion=False):
         """ return (start, end) tuple which spans the minimal
             statement region which containing the given lineno.
-            raise an IndexError if no such statementrange can be found.
         """
-        # XXX there must be a better than these heuristic ways ...
-        # XXX there may even be better heuristics :-)
         if not (0 <= lineno < len(self)):
             raise IndexError("lineno out of range")
-
-        # 1. find the start of the statement
-        from codeop import compile_command
-        end = None
-        for start in range(lineno, -1, -1):
-            if assertion:
-                line = self.lines[start]
-                # the following lines are not fully tested, change with care
-                if 'super' in line and 'self' in line and '__init__' in line:
-                    raise IndexError("likely a subclass")
-                if "assert" not in line and "raise" not in line:
-                    continue
-            trylines = self.lines[start:lineno+1]
-            # quick hack to indent the source and get it as a string in one go
-            trylines.insert(0, 'if xxx:')
-            trysource = '\n '.join(trylines)
-            #              ^ space here
-            try:
-                compile_command(trysource)
-            except (SyntaxError, OverflowError, ValueError):
-                continue
-
-            # 2. find the end of the statement
-            for end in range(lineno+1, len(self)+1):
-                trysource = self[start:end]
-                if trysource.isparseable():
-                    return start, end
-                if end == start + 100:   # XXX otherwise, it takes forever
-                    break                # XXX
-        if end is None:
-            raise IndexError("no valid source range around line %d " % (lineno,))
+        ast, start, end = getstatementrange_ast(lineno, self)
         return start, end
 
-    def getblockend(self, lineno):
-        # XXX
-        lines = [x + '\n' for x in self.lines[lineno:]]
-        blocklines = inspect.getblock(lines)
-        #print blocklines
-        return lineno + len(blocklines) - 1
-
     def deindent(self, offset=None):
         """ return a new source object deindented by offset.
             If offset is None then guess an indentation offset from
@@ -356,3 +316,106 @@ def deindent(lines, offset=None):
     # Add any lines we didn't see. E.g. if an exception was raised.
     newlines.extend(lines[len(newlines):])
     return newlines
+
+def get_statement_startend(lineno, nodelist):
+    from bisect import bisect_right
+    # lineno starts at 0
+    nextlineno = None
+    while 1:
+        lineno_list = [x.lineno-1 for x in nodelist] # ast indexes start at 1
+        insert_index = bisect_right(lineno_list, lineno)
+        if insert_index >= len(nodelist):
+            insert_index -= 1
+        elif lineno < (nodelist[insert_index].lineno - 1) and insert_index > 0:
+            insert_index -= 1
+            assert lineno >= (nodelist[insert_index].lineno - 1)
+        nextnode = nodelist[insert_index]
+
+        try:
+            nextlineno = nodelist[insert_index+1].lineno - 1
+        except IndexError:
+            pass
+        lastnodelist = nodelist
+        nodelist = getnodelist(nextnode)
+        if not nodelist:
+            start, end = nextnode.lineno-1, nextlineno
+            start = min(lineno, start)
+            assert start <= lineno  and (end is None or lineno < end)
+            #print "returning", start, end
+            return start, end
+
+def getnodelist(node):
+    import _ast
+    l = []
+    #print "node", node, "fields", node._fields, "lineno", getattr(node, "lineno", 0) - 1
+    for subname in "test", "type", "body", "handlers", "orelse", "finalbody":
+        attr = getattr(node, subname, None)
+        if attr is not None:
+            if isinstance(attr, list):
+                l.extend(attr)
+            elif hasattr(attr, "lineno"):
+                l.append(attr)
+    #print "returning nodelist", l
+    return l
+
+def getstatementrange_ast(lineno, source, assertion=False, astnode=None):
+    if astnode is None:
+        content = str(source)
+        if sys.version_info < (2,7):
+            content += "\n"
+        try:
+            astnode = compile(content, "source", "exec", 1024)  # 1024 for AST
+        except ValueError:
+            start, end = getstatementrange_old(lineno, source, assertion)
+            return None, start, end
+    start, end = get_statement_startend(lineno, getnodelist(astnode))
+    # we need to correct the end:
+    # - ast-parsing strips comments
+    # - else statements do not have a separate lineno
+    # - there might be empty lines
+    if end is None:
+        end = len(source.lines)
+    while end:
+        line = source.lines[end-1].lstrip()
+        if (not line or line.startswith("#") or line.startswith("else:") or
+            line.startswith("finally:")):
+            end -= 1
+        else:
+            break
+    return astnode, start, end
+
+def getstatementrange_old(lineno, source, assertion=False):
+    """ return (start, end) tuple which spans the minimal
+        statement region which containing the given lineno.
+        raise an IndexError if no such statementrange can be found.
+    """
+    # XXX this logic is only used on python2.4 and below
+    # 1. find the start of the statement
+    from codeop import compile_command
+    for start in range(lineno, -1, -1):
+        if assertion:
+            line = source.lines[start]
+            # the following lines are not fully tested, change with care
+            if 'super' in line and 'self' in line and '__init__' in line:
+                raise IndexError("likely a subclass")
+            if "assert" not in line and "raise" not in line:
+                continue
+        trylines = source.lines[start:lineno+1]
+        # quick hack to prepare parsing an indented line with
+        # compile_command() (which errors on "return" outside defs)
+        trylines.insert(0, 'def xxx():')
+        trysource = '\n '.join(trylines)
+        #              ^ space here
+        try:
+            compile_command(trysource)
+        except (SyntaxError, OverflowError, ValueError):
+            continue
+
+        # 2. find the end of the statement
+        for end in range(lineno+1, len(source)+1):
+            trysource = source[start:end]
+            if trysource.isparseable():
+                return start, end
+    raise SyntaxError("no valid source range around line %d " % (lineno,))
+
+
diff --git a/py/_error.py b/py/_error.py
index f70b671c22..550fb521a0 100644
--- a/py/_error.py
+++ b/py/_error.py
@@ -25,6 +25,7 @@ _winerrnomap = {
     17: errno.EEXIST,
     13: errno.EBUSY, # empty cd drive, but ENOMEDIUM seems unavailiable
     22: errno.ENOTDIR,
+    20: errno.ENOTDIR,
     267: errno.ENOTDIR,
     5: errno.EACCES,  # anything better?
 }
@@ -63,7 +64,7 @@ class ErrorMaker(object):
             return func(*args, **kwargs)
         except self.Error:
             raise
-        except EnvironmentError:
+        except (OSError, EnvironmentError):
             cls, value, tb = sys.exc_info()
             if not hasattr(value, 'errno'):
                 raise
@@ -82,5 +83,6 @@ class ErrorMaker(object):
                     raise value
             raise cls("%s%r" % (func.__name__, args))
             __tracebackhide__ = True
+            
 
 error = ErrorMaker()
diff --git a/py/_iniconfig.py b/py/_iniconfig.py
index c216340785..92b50bd853 100644
--- a/py/_iniconfig.py
+++ b/py/_iniconfig.py
@@ -5,6 +5,8 @@ __version__ = "0.2.dev2"
 
 __all__ = ['IniConfig', 'ParseError']
 
+COMMENTCHARS = "#;"
+
 class ParseError(Exception):
     def __init__(self, path, lineno, msg):
         Exception.__init__(self, path, lineno, msg)
@@ -101,24 +103,30 @@ class IniConfig(object):
         return result
 
     def _parseline(self, line, lineno):
-        # comments
-        line = line.split('#')[0].rstrip()
-        line = line.split(';')[0].rstrip()
         # blank lines
+        if iscommentline(line):
+            line = ""
+        else:
+            line = line.rstrip()
         if not line:
             return None, None
         # section
-        if line[0] == '[' and line[-1] == ']':
-            return line[1:-1], None
+        if line[0] == '[':
+            realline = line
+            for c in COMMENTCHARS:
+                line = line.split(c)[0].rstrip()
+            if line[-1] == "]":
+                return line[1:-1], None
+            return None, realline.strip()
         # value
         elif not line[0].isspace():
             try:
                 name, value = line.split('=', 1)
-                if ": " in name:
+                if ":" in name:
                     raise ValueError()
             except ValueError:
                 try:
-                    name, value = line.split(": ", 1)
+                    name, value = line.split(":", 1)
                 except ValueError:
                     self._raise(lineno, 'unexpected line: %r' % line)
             return name.strip(), value.strip()
@@ -148,3 +156,7 @@ class IniConfig(object):
 
     def __contains__(self, arg):
         return arg in self.sections
+
+def iscommentline(line):
+    c = line.lstrip()[:1]
+    return c in COMMENTCHARS
diff --git a/py/_io/capture.py b/py/_io/capture.py
index d6e7c4833a..bc157ed978 100644
--- a/py/_io/capture.py
+++ b/py/_io/capture.py
@@ -99,12 +99,12 @@ def dupfile(f, mode=None, buffering=0, raising=False, encoding=None):
     """
     try:
         fd = f.fileno()
+        mode = mode or f.mode
     except AttributeError:
         if raising:
             raise
         return f
     newfd = os.dup(fd)
-    mode = mode and mode or f.mode
     if sys.version_info >= (3,0):
         if encoding is not None:
             mode = mode.replace("b", "")
@@ -176,14 +176,21 @@ class Capture(object):
 
 
 class StdCaptureFD(Capture):
-    """ This class allows capturing writes to FD1 and FD2
+    """ This class allows to capture writes to FD1 and FD2
         and may connect a NULL file to FD0 (and prevent
         reads from sys.stdin).  If any of the 0,1,2 file descriptors
         is invalid it will not be captured.
     """
     def __init__(self, out=True, err=True, mixed=False,
         in_=True, patchsys=True, now=True):
-        self._options = locals()
+        self._options = {
+            "out": out,
+            "err": err,
+            "mixed": mixed,
+            "in_": in_,
+            "patchsys": patchsys,
+            "now": now,
+        }
         self._save()
         if now:
             self.startall()
@@ -251,24 +258,30 @@ class StdCaptureFD(Capture):
 
     def readouterr(self):
         """ return snapshot value of stdout/stderr capturings. """
-        l = []
-        for name in ('out', 'err'):
-            res = ""
-            if hasattr(self, name):
-                f = getattr(self, name).tmpfile
-                f.seek(0)
-                res = f.read()
-                enc = getattr(f, 'encoding', None)
-                if enc:
-                    res = py.builtin._totext(res, enc, 'replace')
-                f.truncate(0)
-                f.seek(0)
-            l.append(res)
-        return l
+        if hasattr(self, "out"):
+            out = self._readsnapshot(self.out.tmpfile)
+        else:
+            out = ""
+        if hasattr(self, "err"):
+            err = self._readsnapshot(self.err.tmpfile)
+        else:
+            err = ""
+        return [out, err]
+
+    def _readsnapshot(self, f):
+        f.seek(0)
+        res = f.read()
+        enc = getattr(f, "encoding", None)
+        if enc:
+            res = py.builtin._totext(res, enc, "replace")
+        f.truncate(0)
+        f.seek(0)
+        return res
+
 
 class StdCapture(Capture):
-    """ This class allows capturing writes to sys.stdout|stderr "in-memory"
-        and will raise errors on read attempts from sys.stdin. It only
+    """ This class allows to capture writes to sys.stdout|stderr "in-memory"
+        and will raise errors on tries to read from sys.stdin. It only
         modifies sys.stdout|stderr|stdin attributes and does not
         touch underlying File Descriptors (use StdCaptureFD for that).
     """
diff --git a/py/_io/saferepr.py b/py/_io/saferepr.py
index afc968d3ab..8518290efd 100644
--- a/py/_io/saferepr.py
+++ b/py/_io/saferepr.py
@@ -1,5 +1,5 @@
 import py
-import sys, os.path
+import sys
 
 builtin_repr = repr
 
@@ -12,6 +12,23 @@ class SafeRepr(reprlib.Repr):
     def repr(self, x):
         return self._callhelper(reprlib.Repr.repr, self, x)
 
+    def repr_unicode(self, x, level):
+        # Strictly speaking wrong on narrow builds
+        def repr(u):
+            if "'" not in u:
+                return py.builtin._totext("'%s'") % u
+            elif '"' not in u:
+                return py.builtin._totext('"%s"') % u
+            else:
+                return py.builtin._totext("'%s'") % u.replace("'", r"\'")
+        s = repr(x[:self.maxstring])
+        if len(s) > self.maxstring:
+            i = max(0, (self.maxstring-3)//2)
+            j = max(0, self.maxstring-3-i)
+            s = repr(x[:i] + x[len(x)-j:])
+            s = s[:i] + '...' + s[len(s)-j:]
+        return s
+
     def repr_instance(self, x, level):
         return self._callhelper(builtin_repr, x)
 
@@ -26,7 +43,7 @@ class SafeRepr(reprlib.Repr):
             exc_name = getattr(cls, '__name__', 'unknown')
             try:
                 exc_info = str(e)
-            except sysex:
+            except py.builtin._sysex:
                 raise
             except:
                 exc_info = 'unknown'
diff --git a/py/_io/terminalwriter.py b/py/_io/terminalwriter.py
index a1d92bc003..6aca55d570 100644
--- a/py/_io/terminalwriter.py
+++ b/py/_io/terminalwriter.py
@@ -7,14 +7,21 @@ Helper functions for writing to terminals and files.
 
 import sys, os
 import py
+py3k = sys.version_info[0] >= 3
+from py.builtin import text, bytes
 
 win32_and_ctypes = False
+colorama = None
 if sys.platform == "win32":
     try:
-        import ctypes
-        win32_and_ctypes = True
+        import colorama
     except ImportError:
-        pass
+        try:
+            import ctypes
+            win32_and_ctypes = True
+        except ImportError:
+            pass
+
 
 def _getdimensions():
     import termios,fcntl,struct
@@ -94,6 +101,10 @@ def ansi_print(text, esc, file=None, newline=True, flush=False):
         file.flush()
 
 def should_do_markup(file):
+    if os.environ.get('PY_COLORS') == '1':
+        return True
+    if os.environ.get('PY_COLORS') == '0':
+        return False
     return hasattr(file, 'isatty') and file.isatty() \
            and os.environ.get('TERM') != 'dumb' \
            and not (sys.platform.startswith('java') and os._name == 'nt')
@@ -105,8 +116,6 @@ class TerminalWriter(object):
                      Blue=44, Purple=45, Cyan=46, White=47,
                      bold=1, light=2, blink=5, invert=7)
 
-    _newline = None   # the last line printed
-
     # XXX deprecate stringio argument
     def __init__(self, file=None, stringio=False, encoding=None):
         if file is None:
@@ -114,12 +123,16 @@ class TerminalWriter(object):
                 self.stringio = file = py.io.TextIO()
             else:
                 file = py.std.sys.stdout
-        elif hasattr(file, '__call__'):
+        elif py.builtin.callable(file) and not (
+             hasattr(file, "write") and hasattr(file, "flush")):
             file = WriteFile(file, encoding=encoding)
+        if hasattr(file, "isatty") and file.isatty() and colorama:
+            file = colorama.AnsiToWin32(file).stream
         self.encoding = encoding or getattr(file, 'encoding', "utf-8")
         self._file = file
         self.fullwidth = get_terminal_width()
         self.hasmarkup = should_do_markup(file)
+        self._lastlen = 0
 
     def _escaped(self, text, esc):
         if esc and self.hasmarkup:
@@ -141,6 +154,12 @@ class TerminalWriter(object):
             fullwidth = self.fullwidth
         # the goal is to have the line be as long as possible
         # under the condition that len(line) <= fullwidth
+        if sys.platform == "win32":
+            # if we print in the last column on windows we are on a
+            # new line but there is no way to verify/neutralize this
+            # (we may not know the exact line width)
+            # so let's be defensive to avoid empty lines in the output
+            fullwidth -= 1
         if title is not None:
             # we want 2 + 2*len(fill) + len(title) <= fullwidth
             # i.e.    2 + 2*len(sepchar)*N + len(title) <= fullwidth
@@ -161,56 +180,39 @@ class TerminalWriter(object):
 
         self.line(line, **kw)
 
-    def write(self, s, **kw):
-        if s:
-            if not isinstance(self._file, WriteFile):
-                s = self._getbytestring(s)
-                if self.hasmarkup and kw:
-                    s = self.markup(s, **kw)
-            self._file.write(s)
-            self._file.flush()
-
-    def _getbytestring(self, s):
-        # XXX review this and the whole logic
-        if self.encoding and sys.version_info[0] < 3 and isinstance(s, unicode):
-            return s.encode(self.encoding)
-        elif not isinstance(s, str):
-            try:
-                return str(s)
-            except UnicodeEncodeError:
-                return "<print-error '%s' object>" % type(s).__name__
-        return s
+    def write(self, msg, **kw):
+        if msg:
+            if not isinstance(msg, (bytes, text)):
+                msg = text(msg)
+            if self.hasmarkup and kw:
+                markupmsg = self.markup(msg, **kw)
+            else:
+                markupmsg = msg
+            write_out(self._file, markupmsg)
 
     def line(self, s='', **kw):
-        if self._newline == False:
-            self.write("\n")
         self.write(s, **kw)
+        self._checkfill(s)
         self.write('\n')
-        self._newline = True
 
-    def reline(self, line, **opts):
+    def reline(self, line, **kw):
         if not self.hasmarkup:
             raise ValueError("cannot use rewrite-line without terminal")
-        if not self._newline:
-            self.write("\r")
-        self.write(line, **opts)
-        # see if we need to fill up some spaces at the end
-        # xxx have a more exact lastlinelen working from self.write?
-        lenline = len(line)
-        try:
-            lastlen = self._lastlinelen
-        except AttributeError:
-            pass
-        else:
-            if lenline < lastlen:
-                self.write(" " * (lastlen - lenline + 1))
-        self._lastlinelen = lenline
-        self._newline = False
+        self.write(line, **kw)
+        self._checkfill(line)
+        self.write('\r')
+        self._lastlen = len(line)
 
+    def _checkfill(self, line):
+        diff2last = self._lastlen - len(line)
+        if diff2last > 0:
+            self.write(" " * diff2last)
 
 class Win32ConsoleWriter(TerminalWriter):
-    def write(self, s, **kw):
-        if s:
+    def write(self, msg, **kw):
+        if msg:
+            if not isinstance(msg, (bytes, text)):
+                msg = text(msg)
             oldcolors = None
             if self.hasmarkup and kw:
                 handle = GetStdHandle(STD_OUTPUT_HANDLE)
@@ -232,17 +234,10 @@ class Win32ConsoleWriter(TerminalWriter):
                     attr |= oldcolors & 0x0007
 
                 SetConsoleTextAttribute(handle, attr)
-            if not isinstance(self._file, WriteFile):
-                s = self._getbytestring(s)
-            self._file.write(s)
-            self._file.flush()
+            write_out(self._file, msg)
             if oldcolors:
                 SetConsoleTextAttribute(handle, oldcolors)
 
-    def line(self, s="", **kw):
-        self.write(s, **kw) # works better for resetting colors
-        self.write("\n")
-
 class WriteFile(object):
     def __init__(self, writemethod, encoding=None):
         self.encoding = encoding
@@ -250,7 +245,7 @@ class WriteFile(object):
 
     def write(self, data):
         if self.encoding:
-            data = data.encode(self.encoding)
+            data = data.encode(self.encoding, "replace")
         self._writemethod(data)
 
     def flush(self):
@@ -321,3 +316,26 @@ if win32_and_ctypes:
         # and the ending \n causes an empty line to display.
         return info.dwSize.Y, info.dwSize.X - 1
 
+def write_out(fil, msg):
+    # XXX sometimes "msg" is of type bytes, sometimes text which
+    # complicates the situation.  Should we try to enforce unicode?
+    try:
+        # on py27 and above writing out to sys.stdout with an encoding
+        # should usually work for unicode messages (if the encoding is
+        # capable of it)
+        fil.write(msg)
+    except UnicodeEncodeError:
+        # on py26 it might not work because stdout expects bytes
+        if fil.encoding:
+            try:
+                fil.write(msg.encode(fil.encoding))
+            except UnicodeEncodeError:
+                # it might still fail if the encoding is not capable
+                pass
+            else:
+                fil.flush()
+                return
+        # fallback: escape all unicode characters
+        msg = msg.encode("unicode-escape").decode("ascii")
+        fil.write(msg)
+    fil.flush()
diff --git a/py/_path/common.py b/py/_path/common.py
index 642e3ad0c8..aeb374c087 100644
--- a/py/_path/common.py
+++ b/py/_path/common.py
@@ -67,7 +67,7 @@ class Checkers:
             except (py.error.ENOENT, py.error.ENOTDIR, py.error.EBUSY):
                 # EBUSY feels not entirely correct,
                 # but its kind of necessary since ENOMEDIUM
-                # is not accessible in python     
+                # is not accessible in python
                 for name in self._depend_on_existence:
                     if name in kw:
                         if kw.get(name):
@@ -177,7 +177,7 @@ newline will be removed from the end of each line. """
                 exists=1  # exists
 
             You can specify multiple checker definitions, for example::
-                
+
                 path.check(file=1, link=1)  # a link pointing to a file
         """
         if not kw:
@@ -223,6 +223,10 @@ newline will be removed from the end of each line. """
             return strself[len(strrelpath):]
         return ""
 
+    def ensure_dir(self, *args):
+        """ ensure the path joined with args is a directory. """
+        return self.ensure(*args, **{"dir": True})
+
     def bestrelpath(self, dest):
         """ return a string which is a relative path from self
             (assumed to be a directory) to dest such that
@@ -249,6 +253,14 @@ newline will be removed from the end of each line. """
         except AttributeError:
             return str(dest)
 
+    def exists(self):
+        return self.check()
+
+    def isdir(self):
+        return self.check(dir=1)
+
+    def isfile(self):
+        return self.check(file=1)
 
     def parts(self, reverse=False):
         """ return a root-first list of all ancestor directories
@@ -261,8 +273,8 @@ newline will be removed from the end of each line. """
             current = current.dirpath()
             if last == current:
                 break
-            l.insert(0, current)
-        if reverse:
+            l.append(current)
+        if not reverse:
             l.reverse()
         return l
 
@@ -331,7 +343,7 @@ class Visitor:
         if isinstance(fil, str):
             fil = FNMatcher(fil)
         if isinstance(rec, str):
-            self.rec = fnmatch(fil)
+            self.rec = FNMatcher(rec)
         elif not hasattr(rec, '__call__') and rec:
             self.rec = lambda path: True
         else:
@@ -364,12 +376,14 @@ class Visitor:
 class FNMatcher:
     def __init__(self, pattern):
         self.pattern = pattern
+
     def __call__(self, path):
         pattern = self.pattern
         if pattern.find(path.sep) == -1:
             name = path.basename
         else:
             name = str(path) # path.strpath # XXX svn?
-            pattern = '*' + path.sep + pattern
+            if not os.path.isabs(pattern):
+                pattern = '*' + path.sep + pattern
         return py.std.fnmatch.fnmatch(name, pattern)
 
diff --git a/py/_path/local.py b/py/_path/local.py
index cd49bd5433..af09f43014 100644
--- a/py/_path/local.py
+++ b/py/_path/local.py
@@ -1,12 +1,22 @@
 """
 local path implementation.
 """
-import sys, os, stat, re, atexit
+from contextlib import contextmanager
+import sys, os, re, atexit
 import py
 from py._path import common
+from stat import S_ISLNK, S_ISDIR, S_ISREG
+
+from os.path import abspath, normpath, isabs, exists, isdir, isfile, islink
 
 iswin32 = sys.platform == "win32" or (getattr(os, '_name', False) == 'nt')
 
+if sys.version_info > (3,0):
+    def map_as_list(func, iter):
+        return list(map(func, iter))
+else:
+    map_as_list = map
+
 class Stat(object):
     def __getattr__(self, name):
         return getattr(self._osstatresult, "st_" + name)
@@ -15,14 +25,15 @@ class Stat(object):
         self.path = path
         self._osstatresult = osstatresult
 
+    @property
     def owner(self):
         if iswin32:
             raise NotImplementedError("XXX win32")
         import pwd
         entry = py.error.checked_call(pwd.getpwuid, self.uid)
         return entry[0]
-    owner = property(owner, None, None, "owner of path")
 
+    @property
     def group(self):
         """ return group name of file. """
         if iswin32:
@@ -30,7 +41,16 @@ class Stat(object):
         import grp
         entry = py.error.checked_call(grp.getgrgid, self.gid)
         return entry[0]
-    group = property(group)
+
+    def isdir(self):
+        return S_ISDIR(self._osstatresult.st_mode)
+
+    def isfile(self):
+        return S_ISREG(self._osstatresult.st_mode)
+
+    def islink(self):
+        st = self.path.lstat()
+        return S_ISLNK(self._osstatresult.st_mode)
 
 class PosixPath(common.PathBase):
     def chown(self, user, group, rec=0):
@@ -102,71 +122,75 @@ class LocalPath(FSBase):
                 return self._statcache
 
         def dir(self):
-            return stat.S_ISDIR(self._stat().mode)
+            return S_ISDIR(self._stat().mode)
 
         def file(self):
-            return stat.S_ISREG(self._stat().mode)
+            return S_ISREG(self._stat().mode)
 
         def exists(self):
             return self._stat()
 
         def link(self):
             st = self.path.lstat()
-            return stat.S_ISLNK(st.mode)
+            return S_ISLNK(st.mode)
 
-    def __new__(cls, path=None):
+    def __init__(self, path=None, expanduser=False):
         """ Initialize and return a local Path instance.
 
         Path can be relative to the current directory.
-        If it is None then the current working directory is taken.
+        If path is None it defaults to the current working directory.
+        If expanduser is True, tilde-expansion is performed.
         Note that Path instances always carry an absolute path.
         Note also that passing in a local path object will simply return
         the exact same path object. Use new() to get a new copy.
         """
-        if isinstance(path, common.PathBase):
-            if path.__class__ == cls:
-                return path
-            path = path.strpath
-        # initialize the path
-        self = object.__new__(cls)
-        if not path:
-            self.strpath = os.getcwd()
+        if path is None:
+            self.strpath = py.error.checked_call(os.getcwd)
+        elif isinstance(path, common.PathBase):
+            self.strpath = path.strpath
         elif isinstance(path, py.builtin._basestring):
-            self.strpath = os.path.abspath(os.path.normpath(str(path)))
+            if expanduser:
+                path = os.path.expanduser(path)
+            self.strpath = abspath(normpath(path))
         else:
             raise ValueError("can only pass None, Path instances "
                              "or non-empty strings to LocalPath")
-        assert isinstance(self.strpath, str)
-        return self
 
     def __hash__(self):
         return hash(self.strpath)
 
     def __eq__(self, other):
-        s1 = str(self)
-        s2 = str(other)
+        s1 = self.strpath
+        s2 = getattr(other, "strpath", other)
         if iswin32:
             s1 = s1.lower()
-            s2 = s2.lower()
+            try:
+                s2 = s2.lower()
+            except AttributeError:
+                return False
         return s1 == s2
 
     def __ne__(self, other):
         return not (self == other)
 
     def __lt__(self, other):
-        return str(self) < str(other)
+        return self.strpath < getattr(other, "strpath", other)
+
+    def __gt__(self, other):
+        return self.strpath > getattr(other, "strpath", other)
 
     def samefile(self, other):
         """ return True if 'other' references the same file as 'self'.
         """
-        if not isinstance(other, py.path.local):
-            other = os.path.abspath(str(other))
+        other = getattr(other, "strpath", other)
+        if not isabs(other):
+            other = abspath(other)
         if self == other:
             return True
         if iswin32:
-            return False # ther is no samefile
+            return False # there is no samefile
         return py.error.checked_call(
-                os.path.samefile, str(self), str(other))
+                os.path.samefile, self.strpath, other)
 
     def remove(self, rec=1, ignore_errors=False):
         """ remove a file or directory (or a directory tree if rec=1).
@@ -221,6 +245,9 @@ class LocalPath(FSBase):
                                      xxx   ext
         """
         obj = object.__new__(self.__class__)
+        if not kw:
+            obj.strpath = self.strpath
+            return obj
         drive, dirname, basename, purebasename,ext = self._getbyspec(
              "drive,dirname,basename,purebasename,ext")
         if 'basename' in kw:
@@ -242,7 +269,7 @@ class LocalPath(FSBase):
         else:
             kw.setdefault('dirname', dirname)
         kw.setdefault('sep', self.sep)
-        obj.strpath = os.path.normpath(
+        obj.strpath = normpath(
             "%(dirname)s%(sep)s%(basename)s" % kw)
         return obj
 
@@ -281,46 +308,76 @@ class LocalPath(FSBase):
         components.  if abs=1 is used restart from root if any
         of the args is an absolute path.
         """
-        if not args:
-            return self
-        strpath = self.strpath
         sep = self.sep
-        strargs = [str(x) for x in args]
-        if kwargs.get('abs', 0):
-            for i in range(len(strargs)-1, -1, -1):
-                if os.path.isabs(strargs[i]):
-                    strpath = strargs[i]
-                    strargs = strargs[i+1:]
+        strargs = [getattr(arg, "strpath", arg) for arg in args]
+        strpath = self.strpath
+        if kwargs.get('abs'):
+            newargs = []
+            for arg in reversed(strargs):
+                if isabs(arg):
+                    strpath = arg
+                    strargs = newargs
                     break
+                newargs.insert(0, arg)
         for arg in strargs:
             arg = arg.strip(sep)
             if iswin32:
                 # allow unix style paths even on windows.
                 arg = arg.strip('/')
                 arg = arg.replace('/', sep)
-            if arg:
-                if not strpath.endswith(sep):
-                    strpath += sep
-                strpath += arg
-        obj = self.new()
-        obj.strpath = os.path.normpath(strpath)
+            strpath = strpath + sep + arg
+        obj = object.__new__(self.__class__)
+        obj.strpath = normpath(strpath)
         return obj
 
-    def open(self, mode='r'):
-        """ return an opened file with the given mode. """
+    def open(self, mode='r', ensure=False):
+        """ return an opened file with the given mode.
+
+        If ensure is True, create parent directories if needed.
+        """
+        if ensure:
+            self.dirpath().ensure(dir=1)
         return py.error.checked_call(open, self.strpath, mode)
 
+    def _fastjoin(self, name):
+        child = object.__new__(self.__class__)
+        child.strpath = self.strpath + self.sep + name
+        return child
+
+    def islink(self):
+        return islink(self.strpath)
+
+    def check(self, **kw):
+        if not kw:
+            return exists(self.strpath)
+        if len(kw) == 1:
+            if "dir" in kw:
+                return not kw["dir"] ^ isdir(self.strpath)
+            if "file" in kw:
+                return not kw["file"] ^ isfile(self.strpath)
+        return super(LocalPath, self).check(**kw)
+
+    _patternchars = set("*?[" + os.path.sep)
     def listdir(self, fil=None, sort=None):
         """ list directory contents, possibly filter by the given fil func
             and possibly sorted.
         """
-        if isinstance(fil, str):
+        if fil is None and sort is None:
+            names = py.error.checked_call(os.listdir, self.strpath)
+            return map_as_list(self._fastjoin, names)
+        if isinstance(fil, py.builtin._basestring):
+            if not self._patternchars.intersection(fil):
+                child = self._fastjoin(fil)
+                if exists(child.strpath):
+                    return [child]
+                return []
             fil = common.FNMatcher(fil)
+        names = py.error.checked_call(os.listdir, self.strpath)
         res = []
-        for name in py.error.checked_call(os.listdir, self.strpath):
-            childurl = self.join(name)
-            if fil is None or fil(childurl):
-                res.append(childurl)
+        for name in names:
+            child = self._fastjoin(name)
+            if fil is None or fil(child):
+                res.append(child)
         self._sortlist(res, sort)
         return res
 
@@ -332,14 +389,15 @@ class LocalPath(FSBase):
         """ return last modification time of the path. """
         return self.stat().mtime
 
-    def copy(self, target, archive=False):
+    def copy(self, target, mode=False):
         """ copy path to target."""
-        assert not archive, "XXX archive-mode not supported"
         if self.check(file=1):
             if target.check(dir=1):
                 target = target.join(self.basename)
             assert self!=target
             copychunked(self, target)
+            if mode:
+                copymode(self, target)
         else:
             def rec(p):
                 return p.check(link=0)
@@ -349,14 +407,18 @@ class LocalPath(FSBase):
                 newx.dirpath().ensure(dir=1)
                 if x.check(link=1):
                     newx.mksymlinkto(x.readlink())
+                    continue
                 elif x.check(file=1):
                     copychunked(x, newx)
                 elif x.check(dir=1):
                     newx.ensure(dir=1)
+                if mode:
+                    copymode(x, newx)
 
     def rename(self, target):
         """ rename this path to target. """
-        return py.error.checked_call(os.rename, str(self), str(target))
+        target = getattr(target, "strpath", target)
+        return py.error.checked_call(os.rename, self.strpath, target)
 
     def dump(self, obj, bin=1):
         """ pickle object into path location"""
@@ -369,11 +431,15 @@ class LocalPath(FSBase):
     def mkdir(self, *args):
         """ create & return the directory joined with args. """
         p = self.join(*args)
-        py.error.checked_call(os.mkdir, str(p))
+        py.error.checked_call(os.mkdir, getattr(p, "strpath", p))
         return p
 
-    def write(self, data, mode='w'):
-        """ write data into path. """
+    def write(self, data, mode='w', ensure=False):
+        """ write data into path.   If ensure is True create
+        missing parent directories.
+        """
+        if ensure:
+            self.dirpath().ensure(dir=1)
         if 'b' in mode:
             if not py.builtin._isbytes(data):
                 raise ValueError("can only process bytes")
@@ -419,9 +485,16 @@ class LocalPath(FSBase):
                 p.open('w').close()
             return p
 
-    def stat(self):
+    def stat(self, raising=True):
         """ Return an os.stat() tuple. """
-        return Stat(self, py.error.checked_call(os.stat, self.strpath))
+        if raising == True:
+            return Stat(self, py.error.checked_call(os.stat, self.strpath))
+        try:
+            return Stat(self, os.stat(self.strpath))
+        except KeyboardInterrupt:
+            raise
+        except Exception:
+            return None
 
     def lstat(self):
         """ Return an os.lstat() tuple. """
@@ -442,10 +515,25 @@ class LocalPath(FSBase):
 
     def chdir(self):
         """ change directory to self and return old current directory """
-        old = self.__class__()
+        try:
+            old = self.__class__()
+        except py.error.ENOENT:
+            old = None
         py.error.checked_call(os.chdir, self.strpath)
         return old
 
+
+    @contextmanager
+    def as_cwd(self):
+        """ return context manager which changes to current dir during the
+        managed "with" context. On __enter__ it returns the old dir.
+        """
+        old = self.chdir()
+        try:
+            yield old
+        finally:
+            old.chdir()
+
     def realpath(self):
         """ return a new path which contains no symbolic links."""
         return self.__class__(os.path.realpath(self.strpath))
@@ -517,7 +605,8 @@ class LocalPath(FSBase):
             if pkgpath is not None:
                 if ensuresyspath:
                     self._prependsyspath(pkgpath.dirpath())
-                pkg = __import__(pkgpath.basename, None, None, [])
+                __import__(pkgpath.basename)
+                pkg = sys.modules[pkgpath.basename]
                 names = self.new(ext='').relto(pkgpath.dirpath())
                 names = names.split(self.sep)
                 if names and names[-1] == "__init__":
@@ -528,7 +617,8 @@ class LocalPath(FSBase):
                 if ensuresyspath:
                     self._prependsyspath(self.dirpath())
                 modname = self.purebasename
-            mod = __import__(modname, None, None, ['__doc__'])
+            __import__(modname)
+            mod = sys.modules[modname]
             if self.basename == "__init__.py":
                 return mod # we don't check anything as we might
                        # we in a namespace package ... too icky to check
@@ -569,9 +659,9 @@ class LocalPath(FSBase):
             The process is directly invoked and not through a system shell.
         """
         from subprocess import Popen, PIPE
-        argv = map(str, argv)
+        argv = map_as_list(str, argv)
         popen_opts['stdout'] = popen_opts['stderr'] = PIPE
-        proc = Popen([str(self)] + list(argv), **popen_opts)
+        proc = Popen([str(self)] + argv, **popen_opts)
         stdout, stderr = proc.communicate()
         ret = proc.wait()
         if py.builtin._isbytes(stdout):
@@ -583,7 +673,7 @@ class LocalPath(FSBase):
                                            stdout, stderr,)
         return stdout
 
-    def sysfind(cls, name, checker=None):
+    def sysfind(cls, name, checker=None, paths=None):
         """ return a path object found by looking at the systems
             underlying PATH specification. If the checker is not None
             it will be invoked to filter matching paths.  If a binary
@@ -591,26 +681,28 @@ class LocalPath(FSBase):
             Note: This is probably not working on plain win32 systems
             but may work on cygwin.
         """
-        if os.path.isabs(name):
+        if isabs(name):
             p = py.path.local(name)
             if p.check(file=1):
                 return p
         else:
-            if iswin32:
-                paths = py.std.os.environ['Path'].split(';')
-                if '' not in paths and '.' not in paths:
-                    paths.append('.')
-                try:
-                    systemroot = os.environ['SYSTEMROOT']
-                except KeyError:
-                    pass
+            if paths is None:
+                if iswin32:
+                    paths = py.std.os.environ['Path'].split(';')
+                    if '' not in paths and '.' not in paths:
+                        paths.append('.')
+                    try:
+                        systemroot = os.environ['SYSTEMROOT']
+                    except KeyError:
+                        pass
+                    else:
+                        paths = [re.sub('%SystemRoot%', systemroot, path)
+                                 for path in paths]
                 else:
-                    paths = [re.sub('%SystemRoot%', systemroot, path)
-                             for path in paths]
-                tryadd = [''] + os.environ['PATHEXT'].split(os.pathsep)
-            else:
-                paths = py.std.os.environ['PATH'].split(':')
-                tryadd = ('',)
+                    paths = py.std.os.environ['PATH'].split(':')
+            tryadd = ['']
+            if iswin32:
+                tryadd += os.environ['PATHEXT'].split(os.pathsep)
 
             for x in paths:
                 for addext in tryadd:
@@ -630,7 +722,10 @@ class LocalPath(FSBase):
         try:
             x = os.environ['HOME']
         except KeyError:
-            x = os.environ["HOMEDRIVE"] + os.environ['HOMEPATH']
+            try:
+                x = os.environ["HOMEDRIVE"] + os.environ['HOMEPATH']
+            except KeyError:
+                return None
         return cls(x)
     _gethomedir = classmethod(_gethomedir)
 
@@ -655,8 +750,7 @@ class LocalPath(FSBase):
     mkdtemp = classmethod(mkdtemp)
 
     def make_numbered_dir(cls, prefix='session-', rootdir=None, keep=3,
-                          lock_timeout = 172800,   # two days
-                          min_timeout = 300):      # five minutes
+                          lock_timeout = 172800):   # two days
         """ return unique directory with a number greater than the current
             maximum one.  The number is assumed to start directly after prefix.
             if keep is true directories with a number less than (maxnum-keep)
@@ -724,20 +818,6 @@ class LocalPath(FSBase):
             for path in rootdir.listdir():
                 num = parse_num(path)
                 if num is not None and num <= (maxnum - keep):
-                    if min_timeout:
-                        # NB: doing this is needed to prevent (or reduce
-                        # a lot the chance of) the following situation:
-                        # 'keep+1' processes call make_numbered_dir() at
-                        # the same time, they create dirs, but then the
-                        # last process notices the first dir doesn't have
-                        # (yet) a .lock in it and kills it.
-                        try:
-                            t1 = path.lstat().mtime
-                            t2 = lockfile.lstat().mtime
-                            if abs(t2-t1) < min_timeout:
-                                continue   # skip directories too recent
-                        except py.error.Error:
-                            continue   # failure to get a time, better skip
                     lf = path.join('.lock')
                     try:
                         t1 = lf.lstat().mtime
@@ -776,6 +856,9 @@ class LocalPath(FSBase):
         return udir
     make_numbered_dir = classmethod(make_numbered_dir)
 
+def copymode(src, dest):
+    py.std.shutil.copymode(str(src), str(dest))
+
 def copychunked(src, dest):
     chunksize = 524288 # half a meg of bytes
     fsrc = src.open('rb')
diff --git a/py/_process/cmdexec.py b/py/_process/cmdexec.py
index 4ceb647a90..f83a249402 100644
--- a/py/_process/cmdexec.py
+++ b/py/_process/cmdexec.py
@@ -1,8 +1,4 @@
-"""
-
-"""
-
-import os, sys
+import sys
 import subprocess
 import py
 from subprocess import Popen, PIPE
@@ -10,7 +6,7 @@ from subprocess import Popen, PIPE
 def cmdexec(cmd):
     """ return unicode output of executing 'cmd' in a separate process.
 
-    raise cmdexec.ExecutionFailed exeception if the command failed.
+    raise cmdexec.Error exeception if the command failed.
     the exception will provide an 'err' attribute containing
     the error-output from the command.
     if the subprocess module does not provide a proper encoding/unicode strings
diff --git a/py/_xmlgen.py b/py/_xmlgen.py
index 07f6b76599..2ffcaa14b8 100644
--- a/py/_xmlgen.py
+++ b/py/_xmlgen.py
@@ -4,13 +4,12 @@ by using simple python objects.
 
 (c) holger krekel, holger at merlinux eu. 2009
 """
-import py
 import sys, re
 
 if sys.version_info >= (3,0):
     def u(s):
         return s
-    def unicode(x):
+    def unicode(x, errors=None):
         if hasattr(x, '__unicode__'):
             return x.__unicode__()
         return str(x)
@@ -245,7 +244,10 @@ class _escape:
 
     def __call__(self, ustring):
         """ xml-escape the given unicode string. """
-        ustring = unicode(ustring)
+        try:
+            ustring = unicode(ustring)
+        except UnicodeDecodeError:
+            ustring = unicode(ustring, 'utf-8', errors='replace')
         return self.charef_rex.sub(self._replacer, ustring)
 
 escape = _escape()
diff --git a/py/bin/_findpy.py b/py/bin/_findpy.py
deleted file mode 100644
index 92b14b6c5a..0000000000
--- a/py/bin/_findpy.py
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-
-#
-# find and import a version of 'py'
-#
-import sys
-import os
-from os.path import dirname as opd, exists, join, basename, abspath
-
-def searchpy(current):
-    while 1:
-        last = current
-        initpy = join(current, '__init__.py')
-        if not exists(initpy):
-            pydir = join(current, 'py')
-            # recognize py-package and ensure it is importable
-            if exists(pydir) and exists(join(pydir, '__init__.py')):
-                #for p in sys.path:
-                #    if p == current:
-                #        return True
-                if current != sys.path[0]:  # if we are already first, then ok
-                    sys.stderr.write("inserting into sys.path: %s\n" % current)
-                    sys.path.insert(0, current)
-                return True
-        current = opd(current)
-        if last == current:
-            return False
-
-if not searchpy(abspath(os.curdir)):
-    if not searchpy(opd(abspath(sys.argv[0]))):
-        if not searchpy(opd(__file__)):
-            pass # let's hope it is just on sys.path
-
-import py
-import pytest
-
-if __name__ == '__main__':
-    print ("py lib is at %s" % py.__file__)
diff --git a/py/bin/py.test b/py/bin/py.test
deleted file mode 100755
index 8ecba0e406..0000000000
--- a/py/bin/py.test
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/usr/bin/env python
-from _findpy import pytest
-raise SystemExit(pytest.main())