[vte] pty: Add support for PTY packet mode on sysv-like systems

[Christian Persch <chpe src gnome org>]

commit 8586e4292e76943a100a8039f5ce9e33e5984bb3
Author: Jim Mason <jmason ibinx com>
Date:   Wed Dec 2 21:22:28 2020 +0100

    pty: Add support for PTY packet mode on sysv-like systems
    
    Fixes: https://gitlab.gnome.org/GNOME/vte/-/issues/318

 meson.build |  1 +
 src/pty.cc  | 14 ++++++++++---
 src/vte.cc  | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 79 insertions(+), 3 deletions(-)
---
diff --git a/meson.build b/meson.build
index ea1d7bf3..ce52cb8d 100644
--- a/meson.build
+++ b/meson.build
@@ -152,6 +152,7 @@ check_headers = [
   'stropts.h',
   'sys/resource.h',
   'sys/select.h',
+  'sys/stream.h',
   'sys/syslimits.h',
   'sys/termios.h',
   'sys/types.h',
diff --git a/src/pty.cc b/src/pty.cc
index ce77106b..c6fcdc9d 100644
--- a/src/pty.cc
+++ b/src/pty.cc
@@ -357,10 +357,18 @@ Pty::get_size(int* rows,
 static int
 fd_set_cpkt(vte::libc::FD& fd)
 {
+        auto ret = 0;
+#if defined(TIOCPKT)
         /* tty_ioctl(4) -> every read() gives an extra byte at the beginning
          * notifying us of stop/start (^S/^Q) events. */
         int one = 1;
-        return ioctl(fd.get(), TIOCPKT, &one);
+        ret = ioctl(fd.get(), TIOCPKT, &one);
+#elif defined(__sun) && defined(HAVE_STROPTS_H)
+        if (isastream(fd.get()) == 1 &&
+            ioctl(fd.get(), I_FIND, "pckt") == 0)
+                ret = ioctl(fd.get(), I_PUSH, "pckt");
+#endif
+        return ret;
 }
 
 static int
@@ -400,7 +408,7 @@ fd_setup(vte::libc::FD& fd)
                 auto errsv = vte::libc::ErrnoSaver{};
                 _vte_debug_print(VTE_DEBUG_PTY,
                                  "%s failed: %s",
-                                 "ioctl(TIOCPKT)", g_strerror(errsv));
+                                 "Setting packet mode", g_strerror(errsv));
                 return -1;
         }
 
@@ -466,7 +474,7 @@ _vte_pty_open_posix(void)
                 auto errsv = vte::libc::ErrnoSaver{};
                 _vte_debug_print(VTE_DEBUG_PTY,
                                  "%s failed: %s",
-                                 "ioctl(TIOCPKT)", g_strerror(errsv));
+                                 "Setting packet mode", g_strerror(errsv));
                 return {};
         }
 
diff --git a/src/vte.cc b/src/vte.cc
index b5df3d24..4674241d 100644
--- a/src/vte.cc
+++ b/src/vte.cc
@@ -29,6 +29,12 @@
 #ifdef HAVE_SYS_TERMIOS_H
 #include <sys/termios.h>
 #endif
+#ifdef HAVE_STROPTS_H
+#include <stropts.h>
+#endif
+#ifdef HAVE_SYS_STREAM_H
+#include <sys/stream.h>
+#endif
 
 #include <glib.h>
 #include <glib-unix.h>
@@ -3966,6 +3972,7 @@ Terminal::pty_io_read(int const fd,
                        bp = chunk->begin_writing();
                        len = 0;
                        do {
+#if defined(TIOCPKT)
                                 /* We'd like to read (fd, bp, rem); but due to TIOCPKT mode
                                  * there's an extra input byte returned at the beginning.
                                  * We need to see what that byte is, but otherwise drop it
@@ -4014,6 +4021,66 @@ Terminal::pty_io_read(int const fd,
                                                 }
                                                break;
                                }
+#elif defined(__sun) && defined(HAVE_STROPTS_H)
+                               static unsigned char ctl_s[128];
+                               struct strbuf ctlbuf, databuf;
+                               int ret, flags = 0;
+                               bool have_data = false;
+
+                               ctlbuf.buf = (caddr_t)ctl_s;
+                               ctlbuf.maxlen = sizeof(ctl_s);
+                               databuf.buf = (caddr_t)bp;
+                               databuf.maxlen = rem;
+
+                               ret = getmsg(fd, &ctlbuf, &databuf, &flags);
+                               if (ret == -1) {
+                                       err = errno;
+                                       goto out;
+                               } else if (ctlbuf.len == 1) {
+                                       switch (ctl_s[0]) {
+                                       case M_IOCTL:
+                                               pty_termios_changed();
+                                               break;
+                                       case M_STOP:
+                                               pty_scroll_lock_changed(true);
+                                               break;
+                                       case M_START:
+                                               pty_scroll_lock_changed(false);
+                                               break;
+                                       case M_DATA:
+                                               have_data = true;
+                                               break;
+                                       }
+                               } else if (ctlbuf.len == -1 && databuf.len != -1) {
+                                       // MOREDATA
+                                       have_data = true;
+                               }
+
+                               if (have_data) {
+                                       if (databuf.len == 0) {
+                                               eos = true;
+                                               goto out;
+                                       }
+                                       bp += databuf.len;
+                                       rem -= databuf.len;
+                                       len += databuf.len;
+                               }
+#else /* neither TIOCPKT nor STREAMS pty */
+                               int ret = read(fd, bp, rem);
+                               switch (ret) {
+                                       case -1:
+                                               err = errno;
+                                               goto out;
+                                       case 0:
+                                               eos = true;
+                                               goto out;
+                                       default:
+                                               bp += ret;
+                                               rem -= ret;
+                                               len += ret;
+                                               break;
+                               }
+#endif /* */
                        } while (rem);
 out:
                        chunk->add_size(len);