allow arbitrary python streams to be converted to SBFile

Summary:
This patch adds SWIG typemaps that can convert arbitrary python
file objects into lldb_private::File.

A SBFile may be initialized from a python file using the
constructor.   There are also alternate, tagged constructors
that allow python files to be borrowed, and for the caller
to control whether or not the python I/O methods will be
called even when a file descriptor is available.I

Reviewers: JDevlieghere, jasonmolenda, labath

Reviewed By: labath

Subscribers: zturner, amccarth, lldb-commits

Tags: #lldb

Differential Revision: https://reviews.llvm.org/D68188

llvm-svn: 374225

lldb/include/lldb/API/SBFile.h

@@ -18,6 +18,7 @@ class LLDB_API SBFile {
 
 public:
   SBFile();
+  SBFile(FileSP file_sp);
   SBFile(FILE *file, bool transfer_ownership);
   SBFile(int fd, const char *mode, bool transfer_ownership);
   ~SBFile();
@@ -33,7 +34,6 @@ public:
 
 private:
   FileSP m_opaque_sp;
-  SBFile(FileSP file_sp);
 };
 
 } // namespace lldb

lldb/packages/Python/lldbsuite/test/python_api/file_handle/TestFileHandle.py

@@ -13,8 +13,53 @@ from contextlib import contextmanager
 import lldb
 from lldbsuite.test import  lldbtest
 from lldbsuite.test.decorators import (
-    add_test_categories, no_debug_info_test, skipIf)
+    add_test_categories, skipIf, skipIfWindows)
 
+class OhNoe(Exception):
+    pass
+
+class BadIO(io.TextIOBase):
+    @property
+    def closed(self):
+        return False
+    def writable(self):
+        return True
+    def readable(self):
+        return True
+    def write(self, s):
+        raise OhNoe('OH NOE')
+    def read(self, n):
+        raise OhNoe("OH NOE")
+    def flush(self):
+        raise OhNoe('OH NOE')
+
+# This class will raise an exception while it's being
+# converted into a C++ object by swig
+class ReallyBadIO(io.TextIOBase):
+    def fileno(self):
+        return 999
+    def writable(self):
+        raise OhNoe("OH NOE!!!")
+
+class MutableBool():
+    def __init__(self, value):
+        self.value = value
+    def set(self, value):
+        self.value = bool(value)
+    def __bool__(self):
+        return self.value
+
+class FlushTestIO(io.StringIO):
+    def __init__(self, mutable_flushed, mutable_closed):
+        super(FlushTestIO, self).__init__()
+        self.mut_flushed = mutable_flushed
+        self.mut_closed = mutable_closed
+    def close(self):
+        self.mut_closed.set(True)
+        return super(FlushTestIO, self).close()
+    def flush(self):
+        self.mut_flushed.set(True)
+        return super(FlushTestIO, self).flush()
 
 @contextmanager
 def replace_stdout(new):
@@ -36,6 +81,7 @@ def readStrippedLines(f):
 
 class FileHandleTestCase(lldbtest.TestBase):
 
+    NO_DEBUG_INFO_TESTCASE = True
     mydir = lldbtest.Base.compute_mydir(__file__)
 
     # The way this class interacts with the debugger is different
@@ -84,7 +130,8 @@ class FileHandleTestCase(lldbtest.TestBase):
 
 
     @add_test_categories(['pyapi'])
-    @no_debug_info_test
+    @skipIfWindows # FIXME pre-existing bug, should be fixed
+                   # when we delete the FILE* typemaps.
     def test_legacy_file_out_script(self):
         with open(self.out_filename, 'w') as f:
             self.debugger.SetOutputFileHandle(f, False)
@@ -100,7 +147,6 @@ class FileHandleTestCase(lldbtest.TestBase):
 
 
     @add_test_categories(['pyapi'])
-    @no_debug_info_test
     def test_legacy_file_out(self):
         with open(self.out_filename, 'w') as f:
             self.debugger.SetOutputFileHandle(f, False)
@@ -110,7 +156,8 @@ class FileHandleTestCase(lldbtest.TestBase):
             self.assertIn('deadbeef', f.read())
 
     @add_test_categories(['pyapi'])
-    @no_debug_info_test
+    @skipIfWindows # FIXME pre-existing bug, should be fixed
+                   # when we delete the FILE* typemaps.
     def test_legacy_file_err_with_get(self):
         with open(self.out_filename, 'w') as f:
             self.debugger.SetErrorFileHandle(f, False)
@@ -124,7 +171,6 @@ class FileHandleTestCase(lldbtest.TestBase):
 
 
     @add_test_categories(['pyapi'])
-    @no_debug_info_test
     def test_legacy_file_err(self):
         with open(self.out_filename, 'w') as f:
             self.debugger.SetErrorFileHandle(f, False)
@@ -135,7 +181,6 @@ class FileHandleTestCase(lldbtest.TestBase):
 
 
     @add_test_categories(['pyapi'])
-    @no_debug_info_test
     def test_sbfile_type_errors(self):
         sbf = lldb.SBFile()
         self.assertRaises(TypeError, sbf.Write, None)
@@ -146,8 +191,7 @@ class FileHandleTestCase(lldbtest.TestBase):
 
 
     @add_test_categories(['pyapi'])
-    @no_debug_info_test
+    def test_sbfile_write_fileno(self):
-    def test_sbfile_write(self):
         with open(self.out_filename, 'w') as f:
             sbf = lldb.SBFile(f.fileno(), "w", False)
             self.assertTrue(sbf.IsValid())
@@ -161,8 +205,20 @@ class FileHandleTestCase(lldbtest.TestBase):
 
 
     @add_test_categories(['pyapi'])
-    @no_debug_info_test
+    def test_sbfile_write(self):
-    def test_sbfile_read(self):
+        with open(self.out_filename, 'w') as f:
+            sbf = lldb.SBFile(f)
+            e, n = sbf.Write(b'FOO\n')
+            self.assertTrue(e.Success())
+            self.assertEqual(n, 4)
+            sbf.Close()
+            self.assertTrue(f.closed)
+        with open(self.out_filename, 'r') as f:
+            self.assertEqual(f.read().strip(), 'FOO')
+
+
+    @add_test_categories(['pyapi'])
+    def test_sbfile_read_fileno(self):
         with open(self.out_filename, 'w') as f:
             f.write('FOO')
         with open(self.out_filename, 'r') as f:
@@ -175,7 +231,21 @@ class FileHandleTestCase(lldbtest.TestBase):
 
 
     @add_test_categories(['pyapi'])
-    @no_debug_info_test
+    def test_sbfile_read(self):
+        with open(self.out_filename, 'w') as f:
+            f.write('foo')
+        with open(self.out_filename, 'r') as f:
+            sbf = lldb.SBFile(f)
+            buf = bytearray(100)
+            e, n = sbf.Read(buf)
+            self.assertTrue(e.Success())
+            self.assertEqual(n, 3)
+            self.assertEqual(buf[:n], b'foo')
+            sbf.Close()
+            self.assertTrue(f.closed)
+
+
+    @add_test_categories(['pyapi'])
     def test_fileno_out(self):
         with open(self.out_filename, 'w') as f:
             sbf = lldb.SBFile(f.fileno(), "w", False)
@@ -189,7 +259,6 @@ class FileHandleTestCase(lldbtest.TestBase):
 
 
     @add_test_categories(['pyapi'])
-    @no_debug_info_test
     def test_fileno_help(self):
         with open(self.out_filename, 'w') as f:
             sbf = lldb.SBFile(f.fileno(), "w", False)
@@ -201,7 +270,6 @@ class FileHandleTestCase(lldbtest.TestBase):
 
 
     @add_test_categories(['pyapi'])
-    @no_debug_info_test
     def test_immediate(self):
         with open(self.out_filename, 'w') as f:
             ret = lldb.SBCommandReturnObject()
@@ -220,7 +288,6 @@ class FileHandleTestCase(lldbtest.TestBase):
 
 
     @add_test_categories(['pyapi'])
-    @no_debug_info_test
     def test_fileno_inout(self):
         with open(self.in_filename, 'w') as f:
             f.write("help help\n")
@@ -244,7 +311,6 @@ class FileHandleTestCase(lldbtest.TestBase):
 
 
     @add_test_categories(['pyapi'])
-    @no_debug_info_test
     def test_fileno_error(self):
         with open(self.out_filename, 'w') as f:
 
@@ -263,7 +329,6 @@ class FileHandleTestCase(lldbtest.TestBase):
 
     #FIXME This shouldn't fail for python2 either.
     @add_test_categories(['pyapi'])
-    @no_debug_info_test
     @skipIf(py_version=['<', (3,)])
     def test_replace_stdout(self):
         f = io.StringIO()
@@ -272,3 +337,222 @@ class FileHandleTestCase(lldbtest.TestBase):
             self.handleCmd('script sys.stdout.write("lol")',
                 collect_result=False, check=False)
             self.assertEqual(sys.stdout, f)
+
+
+    @add_test_categories(['pyapi'])
+    def test_sbfile_write_borrowed(self):
+        with open(self.out_filename, 'w') as f:
+            sbf = lldb.SBFile.Create(f, borrow=True)
+            e, n = sbf.Write(b'FOO')
+            self.assertTrue(e.Success())
+            self.assertEqual(n, 3)
+            sbf.Close()
+            self.assertFalse(f.closed)
+            f.write('BAR\n')
+        with open(self.out_filename, 'r') as f:
+            self.assertEqual(f.read().strip(), 'FOOBAR')
+
+
+
+    @add_test_categories(['pyapi'])
+    @skipIf(py_version=['<', (3,)])
+    def test_sbfile_write_forced(self):
+        with open(self.out_filename, 'w') as f:
+            written = MutableBool(False)
+            orig_write = f.write
+            def mywrite(x):
+                written.set(True)
+                return orig_write(x)
+            f.write = mywrite
+            sbf = lldb.SBFile.Create(f, force_io_methods=True)
+            e, n = sbf.Write(b'FOO')
+            self.assertTrue(written)
+            self.assertTrue(e.Success())
+            self.assertEqual(n, 3)
+            sbf.Close()
+            self.assertTrue(f.closed)
+        with open(self.out_filename, 'r') as f:
+            self.assertEqual(f.read().strip(), 'FOO')
+
+
+    @add_test_categories(['pyapi'])
+    @skipIf(py_version=['<', (3,)])
+    def test_sbfile_write_forced_borrowed(self):
+        with open(self.out_filename, 'w') as f:
+            written = MutableBool(False)
+            orig_write = f.write
+            def mywrite(x):
+                written.set(True)
+                return orig_write(x)
+            f.write = mywrite
+            sbf = lldb.SBFile.Create(f, borrow=True, force_io_methods=True)
+            e, n = sbf.Write(b'FOO')
+            self.assertTrue(written)
+            self.assertTrue(e.Success())
+            self.assertEqual(n, 3)
+            sbf.Close()
+            self.assertFalse(f.closed)
+        with open(self.out_filename, 'r') as f:
+            self.assertEqual(f.read().strip(), 'FOO')
+
+
+    @add_test_categories(['pyapi'])
+    @skipIf(py_version=['<', (3,)])
+    def test_sbfile_write_string(self):
+        f = io.StringIO()
+        sbf = lldb.SBFile(f)
+        e, n = sbf.Write(b'FOO')
+        self.assertEqual(f.getvalue().strip(), "FOO")
+        self.assertTrue(e.Success())
+        self.assertEqual(n, 3)
+        sbf.Close()
+        self.assertTrue(f.closed)
+
+    @add_test_categories(['pyapi'])
+    @skipIf(py_version=['<', (3,)])
+    def test_sbfile_write_bytes(self):
+        f = io.BytesIO()
+        sbf = lldb.SBFile(f)
+        e, n = sbf.Write(b'FOO')
+        self.assertEqual(f.getvalue().strip(), b"FOO")
+        self.assertTrue(e.Success())
+        self.assertEqual(n, 3)
+        sbf.Close()
+        self.assertTrue(f.closed)
+
+    @add_test_categories(['pyapi'])
+    @skipIf(py_version=['<', (3,)])
+    def test_sbfile_read_string(self):
+        f = io.StringIO('zork')
+        sbf = lldb.SBFile(f)
+        buf = bytearray(100)
+        e, n = sbf.Read(buf)
+        self.assertTrue(e.Success())
+        self.assertEqual(buf[:n], b'zork')
+
+
+    @add_test_categories(['pyapi'])
+    @skipIf(py_version=['<', (3,)])
+    def test_sbfile_read_string_one_byte(self):
+        f = io.StringIO('z')
+        sbf = lldb.SBFile(f)
+        buf = bytearray(1)
+        e, n = sbf.Read(buf)
+        self.assertTrue(e.Fail())
+        self.assertEqual(n, 0)
+        self.assertEqual(e.GetCString(), "can't read less than 6 bytes from a utf8 text stream")
+
+
+    @add_test_categories(['pyapi'])
+    @skipIf(py_version=['<', (3,)])
+    def test_sbfile_read_bytes(self):
+        f = io.BytesIO(b'zork')
+        sbf = lldb.SBFile(f)
+        buf = bytearray(100)
+        e, n = sbf.Read(buf)
+        self.assertTrue(e.Success())
+        self.assertEqual(buf[:n], b'zork')
+
+
+    @add_test_categories(['pyapi'])
+    @skipIf(py_version=['<', (3,)])
+    def test_sbfile_out(self):
+        with open(self.out_filename, 'w') as f:
+            sbf = lldb.SBFile(f)
+            status = self.debugger.SetOutputFile(sbf)
+            self.assertTrue(status.Success())
+            self.handleCmd('script 2+2')
+        with open(self.out_filename, 'r') as f:
+            self.assertEqual(f.read().strip(), '4')
+
+
+    @add_test_categories(['pyapi'])
+    def test_sbfile_error(self):
+        with open(self.out_filename, 'w') as f:
+            sbf = lldb.SBFile(f)
+            status = self.debugger.SetErrorFile(sbf)
+            self.assertTrue(status.Success())
+            self.handleCmd('lolwut', check=False, collect_result=False)
+        with open(self.out_filename, 'r') as f:
+            errors = f.read()
+            self.assertTrue(re.search(r'error:.*lolwut', errors))
+
+
+    @add_test_categories(['pyapi'])
+    def test_exceptions(self):
+        self.assertRaises(TypeError, lldb.SBFile, None)
+        self.assertRaises(TypeError, lldb.SBFile, "ham sandwich")
+        if sys.version_info[0] < 3:
+            self.assertRaises(TypeError, lldb.SBFile, ReallyBadIO())
+        else:
+            self.assertRaises(OhNoe, lldb.SBFile, ReallyBadIO())
+            error, n = lldb.SBFile(BadIO()).Write(b"FOO")
+            self.assertEqual(n, 0)
+            self.assertTrue(error.Fail())
+            self.assertEqual(error.GetCString(), "OhNoe('OH NOE')")
+            error, n = lldb.SBFile(BadIO()).Read(bytearray(100))
+            self.assertEqual(n, 0)
+            self.assertTrue(error.Fail())
+            self.assertEqual(error.GetCString(), "OhNoe('OH NOE')")
+
+
+    @add_test_categories(['pyapi'])
+    @skipIf(py_version=['<', (3,)])
+    def test_exceptions_logged(self):
+        messages = list()
+        self.debugger.SetLoggingCallback(messages.append)
+        self.handleCmd('log enable lldb script')
+        self.debugger.SetOutputFile(lldb.SBFile(BadIO()))
+        self.handleCmd('script 1+1')
+        self.assertTrue(any('OH NOE' in msg for msg in messages))
+
+
+    @add_test_categories(['pyapi'])
+    @skipIf(py_version=['<', (3,)])
+    def test_flush(self):
+        flushed = MutableBool(False)
+        closed = MutableBool(False)
+        f = FlushTestIO(flushed, closed)
+        self.assertFalse(flushed)
+        self.assertFalse(closed)
+        sbf = lldb.SBFile(f)
+        self.assertFalse(flushed)
+        self.assertFalse(closed)
+        sbf = None
+        self.assertFalse(flushed)
+        self.assertTrue(closed)
+        self.assertTrue(f.closed)
+
+        flushed = MutableBool(False)
+        closed = MutableBool(False)
+        f = FlushTestIO(flushed, closed)
+        self.assertFalse(flushed)
+        self.assertFalse(closed)
+        sbf = lldb.SBFile.Create(f, borrow=True)
+        self.assertFalse(flushed)
+        self.assertFalse(closed)
+        sbf = None
+        self.assertTrue(flushed)
+        self.assertFalse(closed)
+        self.assertFalse(f.closed)
+
+
+    @add_test_categories(['pyapi'])
+    def test_fileno_flush(self):
+        with open(self.out_filename, 'w') as f:
+            f.write("foo")
+            sbf = lldb.SBFile(f)
+            sbf.Write(b'bar')
+            sbf = None
+            self.assertTrue(f.closed)
+        with open(self.out_filename, 'r') as f:
+            self.assertEqual(f.read(), 'foobar')
+
+        with open(self.out_filename, 'w+') as f:
+            f.write("foo")
+            sbf = lldb.SBFile.Create(f, borrow=True)
+            sbf.Write(b'bar')
+            sbf = None
+            self.assertFalse(f.closed)
+            f.seek(0)
+            self.assertEqual(f.read(), 'foobar')

lldb/scripts/Python/python-typemaps.swig

@@ -372,6 +372,69 @@ bool SetNumberFromPyObject<double>(double &number, PyObject *obj) {
   $1 = $1 || PyCallable_Check(reinterpret_cast<PyObject*>($input));
 }
 
+
+%typemap(in) lldb::FileSP {
+  using namespace lldb_private;
+  PythonFile py_file(PyRefType::Borrowed, $input);
+  if (!py_file) {
+    PyErr_SetString(PyExc_TypeError, "not a file");
+    return nullptr;
+  }
+  auto sp = unwrapOrSetPythonException(py_file.ConvertToFile());
+  if (!sp)
+    return nullptr;
+  $1 = sp;
+}
+
+%typemap(in) lldb::FileSP FORCE_IO_METHODS {
+  using namespace lldb_private;
+  PythonFile py_file(PyRefType::Borrowed, $input);
+  if (!py_file) {
+    PyErr_SetString(PyExc_TypeError, "not a file");
+    return nullptr;
+  }
+  auto sp = unwrapOrSetPythonException(py_file.ConvertToFileForcingUseOfScriptingIOMethods());
+  if (!sp)
+    return nullptr;
+  $1 = sp;
+}
+
+%typemap(in) lldb::FileSP BORROWED {
+ using namespace lldb_private;
+  PythonFile py_file(PyRefType::Borrowed, $input);
+  if (!py_file) {
+    PyErr_SetString(PyExc_TypeError, "not a file");
+    return nullptr;
+  }
+  auto sp = unwrapOrSetPythonException(py_file.ConvertToFile(/*borrowed=*/true));
+  if (!sp)
+    return nullptr;
+  $1 = sp;
+}
+
+%typemap(in) lldb::FileSP BORROWED_FORCE_IO_METHODS {
+  using namespace lldb_private;
+  PythonFile py_file(PyRefType::Borrowed, $input);
+  if (!py_file) {
+    PyErr_SetString(PyExc_TypeError, "not a file");
+    return nullptr;
+  }
+  auto sp = unwrapOrSetPythonException(py_file.ConvertToFileForcingUseOfScriptingIOMethods(/*borrowed=*/true));
+  if (!sp)
+    return nullptr;
+  $1 = sp;
+}
+
+%typecheck(SWIG_TYPECHECK_POINTER) lldb::FileSP {
+  if (lldb_private::PythonFile::Check($input)) {
+    $1 = 1;
+  } else {
+    PyErr_Clear();
+    $1 = 0;
+  }
+}
+
+
 // FIXME both of these paths wind up calling fdopen() with no provision for ever calling
 // fclose() on the result.  SB interfaces that use FILE* should be deprecated for scripting
 // use and this typemap should eventually be removed.

lldb/scripts/interface/SBFile.i

@@ -15,9 +15,53 @@ namespace lldb {
 class SBFile
 {
 public:
+
     SBFile();
+
+    %feature("docstring", "
+    Initialize a SBFile from a file descriptor.  mode is
+    'r', 'r+', or 'w', like fdopen.");
     SBFile(int fd, const char *mode, bool transfer_ownership);
 
+    %feature("docstring", "initialize a SBFile from a python file object");
+    SBFile(FileSP file);
+
+    %extend {
+        static lldb::SBFile MakeBorrowed(lldb::FileSP BORROWED) {
+            return lldb::SBFile(BORROWED);
+        }
+        static lldb::SBFile MakeForcingIOMethods(lldb::FileSP FORCE_IO_METHODS) {
+            return lldb::SBFile(FORCE_IO_METHODS);
+        }
+        static lldb::SBFile MakeBorrowedForcingIOMethods(lldb::FileSP BORROWED_FORCE_IO_METHODS) {
+            return lldb::SBFile(BORROWED_FORCE_IO_METHODS);
+        }
+    }
+
+    %pythoncode {
+        @classmethod
+        def Create(cls, file, borrow=False, force_io_methods=False):
+            """
+            Create a SBFile from a python file object, with options.
+
+            If borrow is set then the underlying file will
+            not be closed when the SBFile is closed or destroyed.
+
+            If force_scripting_io is set then the python read/write
+            methods will be called even if a file descriptor is available.
+            """
+            if borrow:
+                if force_io_methods:
+                    return cls.MakeBorrowedForcingIOMethods(file)
+                else:
+                    return cls.MakeBorrowed(file)
+            else:
+                if force_io_methods:
+                    return cls.MakeForcingIOMethods(file)
+                else:
+                    return cls(file)
+    }
+
     ~SBFile ();
 
     %feature("autodoc", "Read(buffer) -> SBError, bytes_read") Read;

lldb/source/API/SBFile.cpp

@@ -16,7 +16,9 @@ using namespace lldb_private;
 
 SBFile::~SBFile() {}
 
-SBFile::SBFile(FileSP file_sp) : m_opaque_sp(file_sp) {}
+SBFile::SBFile(FileSP file_sp) : m_opaque_sp(file_sp) {
+  LLDB_RECORD_DUMMY(void, SBfile, SBFile, (FileSP), file_sp);
+}
 
 SBFile::SBFile() { LLDB_RECORD_CONSTRUCTOR_NO_ARGS(SBFile); }
 

lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp

@@ -1028,22 +1028,23 @@ bool PythonFile::Check(PyObject *py_obj) {
   // first-class object type anymore.  `PyFile_FromFd` is just a thin wrapper
   // over `io.open()`, which returns some object derived from `io.IOBase`. As a
   // result, the only way to detect a file in Python 3 is to check whether it
-  // inherits from `io.IOBase`.  Since it is possible for non-files to also
+  // inherits from `io.IOBase`.
-  // inherit from `io.IOBase`, we additionally verify that it has the `fileno`
+  auto io_module = PythonModule::Import("io");
-  // attribute, which should guarantee that it is backed by the file system.
+  if (!io_module) {
-  PythonObject io_module(PyRefType::Owned, PyImport_ImportModule("io"));
+    llvm::consumeError(io_module.takeError());
-  PythonDictionary io_dict(PyRefType::Borrowed,
-                           PyModule_GetDict(io_module.get()));
-  PythonObject io_base_class = io_dict.GetItemForKey(PythonString("IOBase"));
-
-  PythonObject object_type(PyRefType::Owned, PyObject_Type(py_obj));
-
-  if (1 != PyObject_IsSubclass(object_type.get(), io_base_class.get()))
     return false;
-  if (!object_type.HasAttribute("fileno"))
+  }
+  auto iobase = io_module.get().Get("IOBase");
+  if (!iobase) {
+    llvm::consumeError(iobase.takeError());
     return false;
-
+  }
-  return true;
+  int r = PyObject_IsInstance(py_obj, iobase.get().get());
+  if (r < 0) {
+    llvm::consumeError(exception()); // clear the exception and log it.
+    return false;
+  }
+  return !!r;
 #endif
 }
 
@@ -1096,6 +1097,20 @@ FileUP PythonFile::GetUnderlyingFile() const {
   return file;
 }
 
+namespace {
+class GIL {
+public:
+  GIL() {
+    m_state = PyGILState_Ensure();
+    assert(!PyErr_Occurred());
+  }
+  ~GIL() { PyGILState_Release(m_state); }
+
+protected:
+  PyGILState_STATE m_state;
+};
+} // namespace
+
 const char *PythonException::toCString() const {
   if (!m_repr_bytes)
     return "unknown exception";
@@ -1150,4 +1165,376 @@ std::error_code PythonException::convertToErrorCode() const {
 
 char PythonException::ID = 0;
 
+llvm::Expected<uint32_t> GetOptionsForPyObject(const PythonObject &obj) {
+  uint32_t options = 0;
+#if PY_MAJOR_VERSION >= 3
+  auto readable = As<bool>(obj.CallMethod("readable"));
+  if (!readable)
+    return readable.takeError();
+  auto writable = As<bool>(obj.CallMethod("writable"));
+  if (!writable)
+    return writable.takeError();
+  if (readable.get())
+    options |= File::eOpenOptionRead;
+  if (writable.get())
+    options |= File::eOpenOptionWrite;
+#else
+  PythonString py_mode = obj.GetAttributeValue("mode").AsType<PythonString>();
+  options = File::GetOptionsFromMode(py_mode.GetString());
+#endif
+  return options;
+}
+
+// Base class template for python files.   All it knows how to do
+// is hold a reference to the python object and close or flush it
+// when the File is closed.
+namespace {
+template <typename Base> class OwnedPythonFile : public Base {
+public:
+  template <typename... Args>
+  OwnedPythonFile(const PythonFile &file, bool borrowed, Args... args)
+      : Base(args...), m_py_obj(file), m_borrowed(borrowed) {
+    assert(m_py_obj);
+  }
+
+  ~OwnedPythonFile() override {
+    assert(m_py_obj);
+    GIL takeGIL;
+    Close();
+    m_py_obj.Reset();
+  }
+
+  bool IsPythonSideValid() const {
+    GIL takeGIL;
+    auto closed = As<bool>(m_py_obj.GetAttribute("closed"));
+    if (!closed) {
+      llvm::consumeError(closed.takeError());
+      return false;
+    }
+    return !closed.get();
+  }
+
+  bool IsValid() const override {
+    return IsPythonSideValid() && Base::IsValid();
+  }
+
+  Status Close() override {
+    assert(m_py_obj);
+    Status py_error, base_error;
+    GIL takeGIL;
+    if (!m_borrowed) {
+      auto r = m_py_obj.CallMethod("close");
+      if (!r)
+        py_error = Status(r.takeError());
+    }
+    base_error = Base::Close();
+    if (py_error.Fail())
+      return py_error;
+    return base_error;
+  };
+
+protected:
+  PythonFile m_py_obj;
+  bool m_borrowed;
+};
+} // namespace
+
+// A SimplePythonFile is a OwnedPythonFile that just does all I/O as
+// a NativeFile
+namespace {
+class SimplePythonFile : public OwnedPythonFile<NativeFile> {
+public:
+  SimplePythonFile(const PythonFile &file, bool borrowed, int fd,
+                   uint32_t options)
+      : OwnedPythonFile(file, borrowed, fd, options, false) {}
+};
+} // namespace
+
+#if PY_MAJOR_VERSION >= 3
+
+namespace {
+class PythonBuffer {
+public:
+  PythonBuffer &operator=(const PythonBuffer &) = delete;
+  PythonBuffer(const PythonBuffer &) = delete;
+
+  static Expected<PythonBuffer> Create(PythonObject &obj,
+                                       int flags = PyBUF_SIMPLE) {
+    Py_buffer py_buffer = {};
+    PyObject_GetBuffer(obj.get(), &py_buffer, flags);
+    if (!py_buffer.obj)
+      return llvm::make_error<PythonException>();
+    return PythonBuffer(py_buffer);
+  }
+
+  PythonBuffer(PythonBuffer &&other) {
+    m_buffer = other.m_buffer;
+    other.m_buffer.obj = nullptr;
+  }
+
+  ~PythonBuffer() {
+    if (m_buffer.obj)
+      PyBuffer_Release(&m_buffer);
+  }
+
+  Py_buffer &get() { return m_buffer; }
+
+private:
+  // takes ownership of the buffer.
+  PythonBuffer(const Py_buffer &py_buffer) : m_buffer(py_buffer) {}
+  Py_buffer m_buffer;
+};
+} // namespace
+
+// Shared methods between TextPythonFile and BinaryPythonFile
+namespace {
+class PythonIOFile : public OwnedPythonFile<File> {
+public:
+  PythonIOFile(const PythonFile &file, bool borrowed)
+      : OwnedPythonFile(file, borrowed) {}
+
+  ~PythonIOFile() override { Close(); }
+
+  bool IsValid() const override { return IsPythonSideValid(); }
+
+  Status Close() override {
+    assert(m_py_obj);
+    GIL takeGIL;
+    if (m_borrowed)
+      return Flush();
+    auto r = m_py_obj.CallMethod("close");
+    if (!r)
+      return Status(r.takeError());
+    return Status();
+  }
+
+  Status Flush() override {
+    GIL takeGIL;
+    auto r = m_py_obj.CallMethod("flush");
+    if (!r)
+      return Status(r.takeError());
+    return Status();
+  }
+
+};
+} // namespace
+
+namespace {
+class BinaryPythonFile : public PythonIOFile {
+protected:
+  int m_descriptor;
+
+public:
+  BinaryPythonFile(int fd, const PythonFile &file, bool borrowed)
+      : PythonIOFile(file, borrowed),
+        m_descriptor(File::DescriptorIsValid(fd) ? fd
+                                                 : File::kInvalidDescriptor) {}
+
+  int GetDescriptor() const override { return m_descriptor; }
+
+  Status Write(const void *buf, size_t &num_bytes) override {
+    GIL takeGIL;
+    PyObject *pybuffer_p = PyMemoryView_FromMemory(
+        const_cast<char *>((const char *)buf), num_bytes, PyBUF_READ);
+    if (!pybuffer_p)
+      return Status(llvm::make_error<PythonException>());
+    auto pybuffer = Take<PythonObject>(pybuffer_p);
+    num_bytes = 0;
+    auto bytes_written = As<long long>(m_py_obj.CallMethod("write", pybuffer));
+    if (!bytes_written)
+      return Status(bytes_written.takeError());
+    if (bytes_written.get() < 0)
+      return Status(".write() method returned a negative number!");
+    static_assert(sizeof(long long) >= sizeof(size_t), "overflow");
+    num_bytes = bytes_written.get();
+    return Status();
+  }
+
+  Status Read(void *buf, size_t &num_bytes) override {
+    GIL takeGIL;
+    static_assert(sizeof(long long) >= sizeof(size_t), "overflow");
+    auto pybuffer_obj =
+        m_py_obj.CallMethod("read", (unsigned long long)num_bytes);
+    if (!pybuffer_obj)
+      return Status(pybuffer_obj.takeError());
+    num_bytes = 0;
+    if (pybuffer_obj.get().IsNone()) {
+      // EOF
+      num_bytes = 0;
+      return Status();
+    }
+    auto pybuffer = PythonBuffer::Create(pybuffer_obj.get());
+    if (!pybuffer)
+      return Status(pybuffer.takeError());
+    memcpy(buf, pybuffer.get().get().buf, pybuffer.get().get().len);
+    num_bytes = pybuffer.get().get().len;
+    return Status();
+  }
+};
+} // namespace
+
+namespace {
+class TextPythonFile : public PythonIOFile {
+protected:
+  int m_descriptor;
+
+public:
+  TextPythonFile(int fd, const PythonFile &file, bool borrowed)
+      : PythonIOFile(file, borrowed),
+        m_descriptor(File::DescriptorIsValid(fd) ? fd
+                                                 : File::kInvalidDescriptor) {}
+
+  int GetDescriptor() const override { return m_descriptor; }
+
+  Status Write(const void *buf, size_t &num_bytes) override {
+    GIL takeGIL;
+    auto pystring =
+        PythonString::FromUTF8(llvm::StringRef((const char *)buf, num_bytes));
+    if (!pystring)
+      return Status(pystring.takeError());
+    num_bytes = 0;
+    auto bytes_written =
+        As<long long>(m_py_obj.CallMethod("write", pystring.get()));
+    if (!bytes_written)
+      return Status(bytes_written.takeError());
+    if (bytes_written.get() < 0)
+      return Status(".write() method returned a negative number!");
+    static_assert(sizeof(long long) >= sizeof(size_t), "overflow");
+    num_bytes = bytes_written.get();
+    return Status();
+  }
+
+  Status Read(void *buf, size_t &num_bytes) override {
+    GIL takeGIL;
+    size_t num_chars = num_bytes / 6;
+    size_t orig_num_bytes = num_bytes;
+    num_bytes = 0;
+    if (orig_num_bytes < 6) {
+      return Status("can't read less than 6 bytes from a utf8 text stream");
+    }
+    auto pystring = As<PythonString>(
+        m_py_obj.CallMethod("read", (unsigned long long)num_chars));
+    if (!pystring)
+      return Status(pystring.takeError());
+    if (pystring.get().IsNone()) {
+      // EOF
+      return Status();
+    }
+    auto stringref = pystring.get().AsUTF8();
+    if (!stringref)
+      return Status(stringref.takeError());
+    num_bytes = stringref.get().size();
+    memcpy(buf, stringref.get().begin(), num_bytes);
+    return Status();
+  }
+};
+} // namespace
+
+#endif
+
+llvm::Expected<FileSP> PythonFile::ConvertToFile(bool borrowed) {
+  if (!IsValid())
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "invalid PythonFile");
+
+  int fd = PyObject_AsFileDescriptor(m_py_obj);
+  if (fd < 0) {
+    PyErr_Clear();
+    return ConvertToFileForcingUseOfScriptingIOMethods(borrowed);
+  }
+  auto options = GetOptionsForPyObject(*this);
+  if (!options)
+    return options.takeError();
+
+  // LLDB and python will not share I/O buffers.  We should probably
+  // flush the python buffers now.
+  auto r = CallMethod("flush");
+  if (!r)
+    return r.takeError();
+
+  FileSP file_sp;
+  if (borrowed) {
+    // In this case we we don't need to retain the python
+    // object at all.
+    file_sp = std::make_shared<NativeFile>(fd, options.get(), false);
+  } else {
+    file_sp = std::static_pointer_cast<File>(
+        std::make_shared<SimplePythonFile>(*this, borrowed, fd, options.get()));
+  }
+  if (!file_sp->IsValid())
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "invalid File");
+
+  return file_sp;
+}
+
+llvm::Expected<FileSP>
+PythonFile::ConvertToFileForcingUseOfScriptingIOMethods(bool borrowed) {
+
+  assert(!PyErr_Occurred());
+
+  if (!IsValid())
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "invalid PythonFile");
+
+#if PY_MAJOR_VERSION < 3
+
+  return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                 "not supported on python 2");
+
+#else
+
+  int fd = PyObject_AsFileDescriptor(m_py_obj);
+  if (fd < 0) {
+    PyErr_Clear();
+    fd = File::kInvalidDescriptor;
+  }
+
+  auto io_module = PythonModule::Import("io");
+  if (!io_module)
+    return io_module.takeError();
+  auto textIOBase = io_module.get().Get("TextIOBase");
+  if (!textIOBase)
+    return textIOBase.takeError();
+  auto rawIOBase = io_module.get().Get("RawIOBase");
+  if (!rawIOBase)
+    return rawIOBase.takeError();
+  auto bufferedIOBase = io_module.get().Get("BufferedIOBase");
+  if (!bufferedIOBase)
+    return bufferedIOBase.takeError();
+
+  FileSP file_sp;
+
+  auto isTextIO = IsInstance(textIOBase.get());
+  if (!isTextIO)
+    return isTextIO.takeError();
+  if (isTextIO.get())
+    file_sp = std::static_pointer_cast<File>(
+        std::make_shared<TextPythonFile>(fd, *this, borrowed));
+
+  auto isRawIO = IsInstance(rawIOBase.get());
+  if (!isRawIO)
+    return isRawIO.takeError();
+  auto isBufferedIO = IsInstance(bufferedIOBase.get());
+  if (!isBufferedIO)
+    return isBufferedIO.takeError();
+
+  if (isRawIO.get() || isBufferedIO.get()) {
+    file_sp = std::static_pointer_cast<File>(
+        std::make_shared<BinaryPythonFile>(fd, *this, borrowed));
+  }
+
+  if (!file_sp)
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "python file is neither text nor binary");
+
+  if (!file_sp->IsValid())
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "invalid File");
+
+  return file_sp;
+
+#endif
+}
+
 #endif