From: Geoffrey Chan <gc@qsensei.com>
Date: Sun, 10 May 2015 00:53:07 +0000 (-0400)
Subject: fix: don't decode stream bytes until json.decode
X-Git-Url: https://vcs.fsf.org/?a=commitdiff_plain;h=2bbef250ffaab8297c6b35e3225bfef28a5e65e5;p=tweepy.git

fix: don't decode stream bytes until json.decode
---

diff --git a/tests/test_streaming.py b/tests/test_streaming.py
index a2c0f2a..8b7abf8 100644
--- a/tests/test_streaming.py
+++ b/tests/test_streaming.py
@@ -174,6 +174,16 @@ class TweepyStreamReadBuffer(unittest.TestCase):
         # The mocked function not have been called at all since the stream looks closed
         self.assertEqual(mock_read.call_count, 0)
 
+    def test_read_unicode_tweet(self):
+        stream = '11\n{id:12345}\n\n23\n{id:23456, test:"\xe3\x81\x93"}\n\n'
+        for length in [1, 2, 5, 10, 20, 50]:
+            buf = ReadBuffer(six.StringIO(stream), length)
+            self.assertEqual('11\n', buf.read_line())
+            self.assertEqual('{id:12345}\n', buf.read_len(11))
+            self.assertEqual('\n', buf.read_line())
+            self.assertEqual('23\n', buf.read_line())
+            self.assertEqual('{id:23456, test:"\xe3\x81\x93"}\n', buf.read_len(23))
+
 
 class TweepyStreamBackoffTests(unittest.TestCase):
     def setUp(self):
diff --git a/tweepy/streaming.py b/tweepy/streaming.py
index 9b246bd..6ce3e5c 100644
--- a/tweepy/streaming.py
+++ b/tweepy/streaming.py
@@ -150,7 +150,7 @@ class ReadBuffer(object):
 
     def __init__(self, stream, chunk_size):
         self._stream = stream
-        self._buffer = u""
+        self._buffer = ''
         self._chunk_size = chunk_size
 
     def read_len(self, length):
@@ -158,7 +158,7 @@ class ReadBuffer(object):
             if len(self._buffer) >= length:
                 return self._pop(length)
             read_len = max(self._chunk_size, length - len(self._buffer))
-            self._buffer += self._stream.read(read_len).decode("ascii")
+            self._buffer += self._stream.read(read_len)
 
     def read_line(self, sep='\n'):
         start = 0
@@ -168,7 +168,7 @@ class ReadBuffer(object):
                 return self._pop(loc + len(sep))
             else:
                 start = len(self._buffer)
-            self._buffer += self._stream.read(self._chunk_size).decode("ascii")
+            self._buffer += self._stream.read(self._chunk_size)
 
     def _pop(self, length):
         r = self._buffer[:length]