[geary/cherry-pick-850d7f1c] Merge branch 'mjog/771-html-not-wrapped-on-send' into 'mainline'

[Michael Gratton <mjog src gnome org>]

commit 4bbd576623e2de20abc0fd727bc57c98b0c5fad2
Author: Michael Gratton <mike vee net>
Date:   Sun Mar 29 05:52:02 2020 +0000

    Merge branch 'mjog/771-html-not-wrapped-on-send' into 'mainline'
    
    HTML not wrapped on send
    
    Closes #771
    
    See merge request GNOME/geary!468
    
    (cherry picked from commit 850d7f1c2860f370381daf8b2c9609a055bec0e8)
    
    42152d00 Geary.RFC822.Message: Work around long HTML lines exceeding SMTP max len
    f3c02c78 Geary.RFC822.Utils: Ensure best encoding/charset not missing data
    2eb597fe Geary.RFC822.Message: Always re-determine the encoding for body parts

 src/engine/rfc822/rfc822-message.vala | 26 +++++++--------
 src/engine/rfc822/rfc822-utils.vala   | 10 +++---
 test/engine/rfc822-message-test.vala  | 61 +++++++++++++++++++++++++++++++++--
 test/engine/rfc822-utils-test.vala    | 31 ++++++++++++++++++
 4 files changed, 106 insertions(+), 22 deletions(-)
---
diff --git a/src/engine/rfc822/rfc822-message.vala b/src/engine/rfc822/rfc822-message.vala
index b7df6725..4ef341c8 100644
--- a/src/engine/rfc822/rfc822-message.vala
+++ b/src/engine/rfc822/rfc822-message.vala
@@ -210,10 +210,12 @@ public class Geary.RFC822.Message : BaseObject, EmailHeaderSet {
 
         Gee.List<GMime.Object> body_parts = new Gee.LinkedList<GMime.Object>();
 
-        // Share the body charset and encoding between plain and HTML
-        // parts, so we don't need to work it out twice.
+        // Share the body charset between plain and HTML parts, so we
+        // don't need to work it out twice. This doesn't work for the
+        // content encoding however since the HTML encoding may need
+        // to be different, e.g. if it contains lines longer than
+        // allowed by RFC822/SMTP.
         string? body_charset = null;
-        GMime.ContentEncoding? body_encoding = null;
 
         // Body: text format (optional)
         if (email.body_text != null) {
@@ -222,7 +224,6 @@ public class Geary.RFC822.Message : BaseObject, EmailHeaderSet {
                 body_text = yield body_data_to_part(
                     email.body_text.data,
                     null,
-                    null,
                     "text/plain",
                     true,
                     cancellable
@@ -234,7 +235,6 @@ public class Geary.RFC822.Message : BaseObject, EmailHeaderSet {
                 body_charset = body_text.get_content_type().get_parameter(
                     "charset"
                 );
-                body_encoding = body_text.get_content_encoding();
                 body_parts.add(body_text);
             }
         }
@@ -324,7 +324,6 @@ public class Geary.RFC822.Message : BaseObject, EmailHeaderSet {
                 body_html = yield body_data_to_part(
                     email.body_html.data,
                     body_charset,
-                    body_encoding,
                     "text/html",
                     false,
                     cancellable
@@ -1146,7 +1145,6 @@ public class Geary.RFC822.Message : BaseObject, EmailHeaderSet {
      */
     private async GMime.Part body_data_to_part(uint8[] content,
                                                string? charset,
-                                               GMime.ContentEncoding? encoding,
                                                string content_type,
                                                bool is_flowed,
                                                GLib.Cancellable? cancellable)
@@ -1157,13 +1155,13 @@ public class Geary.RFC822.Message : BaseObject, EmailHeaderSet {
         }
         GMime.StreamFilter filter_stream = new GMime.StreamFilter(content_stream);
         filter_stream.add(new GMime.FilterCharset(UTF8_CHARSET, charset));
-        if (encoding == null) {
-            encoding = yield Utils.get_best_encoding(
-                filter_stream,
-                GMime.EncodingConstraint.7BIT,
-                cancellable
-            );
-        }
+
+        GMime.ContentEncoding encoding = yield Utils.get_best_encoding(
+            filter_stream,
+            GMime.EncodingConstraint.7BIT,
+            cancellable
+        );
+
         if (is_flowed && encoding == GMime.ContentEncoding.BASE64) {
             // Base64-encoded text needs to have CR's added after LF's
             // before encoding, otherwise it breaks format=flowed. See
diff --git a/src/engine/rfc822/rfc822-utils.vala b/src/engine/rfc822/rfc822-utils.vala
index d9278f64..f05c50a2 100644
--- a/src/engine/rfc822/rfc822-utils.vala
+++ b/src/engine/rfc822/rfc822-utils.vala
@@ -263,9 +263,7 @@ public string to_preview_text(string? text, TextFormat format) {
 public async string get_best_charset(GMime.Stream in_stream,
                                      GLib.Cancellable? cancellable)
     throws GLib.Error {
-    GMime.FilterBest filter = new GMime.FilterBest(
-        GMime.FilterBestFlags.CHARSET
-    );
+    GMime.FilterBest filter = new GMime.FilterBest(CHARSET);
     GMime.StreamFilter out_stream = new GMime.StreamFilter(
         new GMime.StreamNull()
     );
@@ -274,6 +272,7 @@ public async string get_best_charset(GMime.Stream in_stream,
     yield Nonblocking.Concurrent.global.schedule_async(() => {
             in_stream.write_to_stream(out_stream);
             in_stream.reset();
+            out_stream.close();
         },
         cancellable
     );
@@ -290,9 +289,7 @@ public async GMime.ContentEncoding get_best_encoding(GMime.Stream in_stream,
                                                      GMime.EncodingConstraint constraint,
                                                      GLib.Cancellable? cancellable)
     throws GLib.Error {
-    GMime.FilterBest filter = new GMime.FilterBest(
-        GMime.FilterBestFlags.ENCODING
-    );
+    GMime.FilterBest filter = new GMime.FilterBest(ENCODING);
     GMime.StreamFilter out_stream = new GMime.StreamFilter(
         new GMime.StreamNull()
     );
@@ -301,6 +298,7 @@ public async GMime.ContentEncoding get_best_encoding(GMime.Stream in_stream,
     yield Nonblocking.Concurrent.global.schedule_async(() => {
             in_stream.write_to_stream(out_stream);
             in_stream.reset();
+            out_stream.close();
         },
         cancellable
     );
diff --git a/test/engine/rfc822-message-test.vala b/test/engine/rfc822-message-test.vala
index 48b4bef0..2f1ab3f9 100644
--- a/test/engine/rfc822-message-test.vala
+++ b/test/engine/rfc822-message-test.vala
@@ -57,9 +57,11 @@ This is the second line.
         add_test("get_recipients", get_recipients);
         add_test("get_searchable_body", get_searchable_body);
         add_test("get_searchable_recipients", get_searchable_recipients);
-        add_test("get_network_buffer", get_network_buffer);
         add_test("from_composed_email", from_composed_email);
         add_test("from_composed_email_inline_attachments", from_composed_email_inline_attachments);
+        add_test("get_network_buffer", get_network_buffer);
+        add_test("get_network_buffer_dot_stuff", get_network_buffer_dot_stuff);
+        add_test("get_network_buffer_long_ascii_line", get_network_buffer_long_ascii_line);
     }
 
     public void basic_message_from_buffer() throws Error {
@@ -210,10 +212,65 @@ This is the second line.
 
     public void get_network_buffer() throws Error {
         Message test = resource_to_message(BASIC_TEXT_PLAIN);
-        Memory.Buffer buffer =  test.get_network_buffer(true);
+        Memory.Buffer buffer = test.get_network_buffer(true);
         assert_true(buffer.to_string() == NETWORK_BUFFER_EXPECTED, "Network buffer differs");
     }
 
+    public void get_network_buffer_dot_stuff() throws GLib.Error {
+        RFC822.MailboxAddress to = new RFC822.MailboxAddress(
+            "Test", "test example com"
+        );
+        RFC822.MailboxAddress from = new RFC822.MailboxAddress(
+            "Sender", "sender example com"
+        );
+        Geary.ComposedEmail composed = new Geary.ComposedEmail(
+            new GLib.DateTime.now_local(),
+            new Geary.RFC822.MailboxAddresses.single(from)
+        ).set_to(new Geary.RFC822.MailboxAddresses.single(to));
+        composed.body_text = ".newline\n.\n";
+
+        this.message_from_composed_email.begin(
+            composed,
+            async_complete_full
+        );
+        Geary.RFC822.Message message = message_from_composed_email.end(async_result());
+
+        string message_data = message.get_network_buffer(true).to_string();
+        assert_true(message_data.has_suffix("..newline\r\n..\r\n"));
+    }
+
+    public void get_network_buffer_long_ascii_line() throws GLib.Error {
+        RFC822.MailboxAddress to = new RFC822.MailboxAddress(
+            "Test", "test example com"
+        );
+        RFC822.MailboxAddress from = new RFC822.MailboxAddress(
+            "Sender", "sender example com"
+        );
+        Geary.ComposedEmail composed = new Geary.ComposedEmail(
+            new GLib.DateTime.now_local(),
+            new Geary.RFC822.MailboxAddresses.single(from)
+        ).set_to(new Geary.RFC822.MailboxAddresses.single(to));
+
+        GLib.StringBuilder buf = new GLib.StringBuilder();
+        for (int i = 0; i < 2000; i++) {
+            buf.append("long ");
+        }
+
+        //composed.body_text = buf.str;
+        composed.body_html = "<p>%s<p>".printf(buf.str);
+
+        this.message_from_composed_email.begin(
+            composed,
+            async_complete_full
+        );
+        Geary.RFC822.Message message = message_from_composed_email.end(async_result());
+
+        string message_data = message.get_network_buffer(true).to_string();
+        foreach (var line in message_data.split("\n")) {
+            assert_true(line.length < 1000, line);
+        }
+    }
+
     public void from_composed_email() throws GLib.Error {
         RFC822.MailboxAddress to = new RFC822.MailboxAddress(
             "Test", "test example com"
diff --git a/test/engine/rfc822-utils-test.vala b/test/engine/rfc822-utils-test.vala
index d478cb48..d150b71f 100644
--- a/test/engine/rfc822-utils-test.vala
+++ b/test/engine/rfc822-utils-test.vala
@@ -10,6 +10,9 @@ class Geary.RFC822.Utils.Test : TestCase {
     public Test() {
         base("Geary.RFC822.Utils.Test");
         add_test("to_preview_text", to_preview_text);
+        add_test("best_encoding_default", best_encoding_default);
+        add_test("best_encoding_long_line", best_encoding_long_line);
+        add_test("best_encoding_binary", best_encoding_binary);
     }
 
     public void to_preview_text() throws Error {
@@ -21,6 +24,34 @@ class Geary.RFC822.Utils.Test : TestCase {
                HTML_BODY_EXPECTED);
     }
 
+    public void best_encoding_default() throws GLib.Error {
+        string test = "abc";
+        var stream = new GMime.StreamMem.with_buffer(test.data);
+        get_best_encoding.begin(stream, 7BIT, null, async_complete_full);
+        var encoding = get_best_encoding.end(async_result());
+        assert_true(encoding == DEFAULT);
+    }
+
+    public void best_encoding_long_line() throws GLib.Error {
+        GLib.StringBuilder buf = new GLib.StringBuilder();
+        for (int i = 0; i < 2000; i++) {
+            buf.append("long ");
+        }
+        var stream = new GMime.StreamMem.with_buffer(buf.str.data);
+        get_best_encoding.begin(stream, 7BIT, null, async_complete_full);
+        var encoding = get_best_encoding.end(async_result());
+        assert_true(encoding == QUOTEDPRINTABLE);
+    }
+
+    public void best_encoding_binary() throws GLib.Error {
+        uint8 test[] = { 0x20, 0x00, 0x20 };
+        var stream = new GMime.StreamMem.with_buffer(test);
+        get_best_encoding.begin(stream, 7BIT, null, async_complete_full);
+        var encoding = get_best_encoding.end(async_result());
+        assert_true(encoding == BASE64);
+    }
+
+
     public static string PLAIN_BODY_ENCODED = "-----BEGIN PGP SIGNED MESSAGE-----\nHash: 
SHA512\n\n=============================================================================\nFreeBSD-EN-16:11.vmbus
                                          Errata Notice\n                                                     
     The FreeBSD Project\n\nTopic:          Avoid using spin locks for channel message locks\n\nCategory:     
  core\nModule:         vmbus\nAnnounced:      2016-08-12\nCredits:        Microsoft OSTC\nAffects:        
FreeBSD 10.3\nCorrected:      2016-06-15 09:52:01 UTC (stable/10, 10.3-STABLE)\n                2016-08-12 
04:01:16 UTC (releng/10.3, 10.3-RELEASE-p7)\n\nFor general information regarding FreeBSD Errata Notices and 
Security\nAdvisories, including descriptions of the fields above, security\nbranches, and the following 
sections, please visit\n<URL:https://security.FreeBSD.org/>.\n";
     public static string PLAIN_BODY_EXPECTED = "FreeBSD-EN-16:11.vmbus Errata Notice The FreeBSD Project 
Topic: Avoid using spin locks for channel message locks Category: core Module: vmbus Announced: 2016-08-12 
Credits: Microsoft OSTC Affects: FreeBSD 10.3 Corrected: 2016-06-15 09:52:01 UTC (stable/10, 10.3-STABLE) 
2016-08-12 04:01:16 UTC (releng/10.3, 10.3-RELEASE-p7) For general information regarding FreeBSD Errata 
Notices and Security Advisories, including descriptions of the fields above, security branches, and the 
following sections, please visit <URL:https://security.FreeBSD.org/>.";