Changed python pickling code so it works with Python 3

dlib/python/serialize_pickle.h

@@ -6,6 +6,7 @@
 #include <dlib/serialize.h>
 #include <boost/python.hpp>
 #include <sstream>
+#include <dlib/vectorstream.h>
 
 template <typename T>
 struct serialize_pickle : boost::python::pickle_suite
@@ -15,9 +16,12 @@ struct serialize_pickle : boost::python::pickle_suite
     )
     {
         using namespace dlib;
-        std::ostringstream sout;
+        std::vector<char> buf;
+        buf.reserve(5000);
+        vectorstream sout(buf);
         serialize(item, sout);
-        return boost::python::make_tuple(sout.str());
+        return boost::python::make_tuple(boost::python::handle<>(
+                PyBytes_FromStringAndSize(buf.size()?&buf[0]:0, buf.size())));
     }
 
     static void setstate(
@@ -36,10 +40,30 @@ struct serialize_pickle : boost::python::pickle_suite
             throw_error_already_set();
         }
 
-        str data = extract<str>(state[0]);
-        std::string temp(extract<const char*>(data), len(data));
-        std::istringstream sin(temp);
-        deserialize(item, sin);
+        // We used to serialize by converting to a str but the boost.python routines for
+        // doing this don't work in Python 3.  You end up getting an error about invalid
+        // UTF-8 encodings.  So instead we access the python C interface directly and use
+        // bytes objects.  However, we keep the deserialization code that worked with str
+        // for backwards compatibility with previously pickled files.
+        if (extract<str>(state[0]).check())
+        {
+            str data = extract<str>(state[0]);
+            std::string temp(extract<const char*>(data), len(data));
+            std::istringstream sin(temp);
+            deserialize(item, sin);
+        }
+        else if(PyBytes_Check(object(state[0]).ptr()))
+        {
+            object obj = state[0];
+            char* data = PyBytes_AsString(obj.ptr());
+            unsigned long num = PyBytes_Size(obj.ptr());
+            std::istringstream sin(std::string(data, num));
+            deserialize(item, sin);
+        }
+        else
+        {
+            throw error("Unable to unpickle, error in input file.");
+        }
     }
 };