From 905c8c3d8dfe081d91e399aa5fd93d1659655264 Mon Sep 17 00:00:00 2001
From: R David Murray <rdmurray@bitdance.com>
Date: Sat, 8 Feb 2014 11:48:20 -0500
Subject: [PATCH] #19772: Do not mutate message when downcoding to 7bit.

This is a bit of an ugly hack because of the way generator pieces together the
output message.  The deepcopys aren't too expensive, though, because we know it
is only called on messages that are not multiparts, and the payload (the thing
that could be large) is an immutable object.

Test and preliminary work on patch by Vajrasky Kok.
---
 Lib/email/generator.py            | 14 ++++++++++++++
 Lib/test/test_email/test_email.py | 12 +++++++++++-
 Misc/NEWS                         |  3 +++
 3 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/Lib/email/generator.py b/Lib/email/generator.py
index 4ea0b559b99..07a97c7e27b 100644
--- a/Lib/email/generator.py
+++ b/Lib/email/generator.py
@@ -12,6 +12,7 @@
 import random
 import warnings
 
+from copy import deepcopy
 from io import StringIO, BytesIO
 from email._policybase import compat32
 from email.header import Header
@@ -173,10 +174,18 @@ def _write(self, msg):
         # necessary.
         oldfp = self._fp
         try:
+            self._munge_cte = None
             self._fp = sfp = self._new_buffer()
             self._dispatch(msg)
         finally:
             self._fp = oldfp
+            munge_cte = self._munge_cte
+            del self._munge_cte
+        # If we munged the cte, copy the message again and re-fix the CTE.
+        if munge_cte:
+            msg = deepcopy(msg)
+            msg.replace_header('content-transfer-encoding', munge_cte[0])
+            msg.replace_header('content-type', munge_cte[1])
         # Write the headers.  First we see if the message object wants to
         # handle that itself.  If not, we'll do it generically.
         meth = getattr(msg, '_write_headers', None)
@@ -225,9 +234,14 @@ def _handle_text(self, msg):
         if _has_surrogates(msg._payload):
             charset = msg.get_param('charset')
             if charset is not None:
+                # XXX: This copy stuff is an ugly hack to avoid modifying the
+                # existing message.
+                msg = deepcopy(msg)
                 del msg['content-transfer-encoding']
                 msg.set_payload(payload, charset)
                 payload = msg.get_payload()
+                self._munge_cte = (msg['content-transfer-encoding'],
+                                   msg['content-type'])
         if self._mangle_from_:
             payload = fcre.sub('>From ', payload)
         self._write_lines(payload)
diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py
index 4157a067a2a..73ec2a6d0bc 100644
--- a/Lib/test/test_email/test_email.py
+++ b/Lib/test/test_email/test_email.py
@@ -3495,7 +3495,7 @@ def test_CRLFLF_at_end_of_part(self):
         self.assertTrue(msg.get_payload(0).get_payload().endswith('\r\n'))
 
 
-class Test8BitBytesHandling(unittest.TestCase):
+class Test8BitBytesHandling(TestEmailBase):
     # In Python3 all input is string, but that doesn't work if the actual input
     # uses an 8bit transfer encoding.  To hack around that, in email 5.1 we
     # decode byte streams using the surrogateescape error handler, and
@@ -3748,6 +3748,16 @@ def test_generator_handles_8bit(self):
         email.generator.Generator(out).flatten(msg)
         self.assertEqual(out.getvalue(), self.non_latin_bin_msg_as7bit_wrapped)
 
+    def test_str_generator_should_not_mutate_msg_when_handling_8bit(self):
+        msg = email.message_from_bytes(self.non_latin_bin_msg)
+        out = BytesIO()
+        BytesGenerator(out).flatten(msg)
+        orig_value = out.getvalue()
+        Generator(StringIO()).flatten(msg) # Should not mutate msg!
+        out = BytesIO()
+        BytesGenerator(out).flatten(msg)
+        self.assertEqual(out.getvalue(), orig_value)
+
     def test_bytes_generator_with_unix_from(self):
         # The unixfrom contains a current date, so we can't check it
         # literally.  Just make sure the first word is 'From' and the
diff --git a/Misc/NEWS b/Misc/NEWS
index ae5f83a1537..cedd4e52ac5 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -48,6 +48,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #19772: email.generator no longer mutates the message object when
+  doing a down-transform from 8bit to 7bit CTEs.
+
 - Issue #18805: the netmask/hostmask parsing in ipaddress now more reliably
   filters out illegal values and correctly allows any valid prefix length.