JLine3 should not allow building a system terminal if input *or* output is not a tty

Author: mattirn

terminal-jansi/src/main/java/org/jline/terminal/impl/jansi/JansiNativePty.java

@@ -176,4 +176,7 @@ public static boolean isConsoleOutput() {
         return CLibrary.isatty(1) == 1;
     }
 
+    public static boolean isConsoleInput() {
+        return CLibrary.isatty(0) == 1;
+    }
 }

terminal-jansi/src/main/java/org/jline/terminal/impl/jansi/JansiSupportImpl.java

@@ -18,6 +18,7 @@
 import org.jline.terminal.impl.jansi.win.JansiWinSysTerminal;
 import org.jline.terminal.spi.JansiSupport;
 import org.jline.terminal.spi.Pty;
+import org.jline.utils.OSUtils;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -137,7 +138,31 @@ public boolean isWindowsConsole() {
     }
 
     @Override
-    public boolean isConsoleOutput(boolean pty) {
-        return pty ? JansiNativePty.isConsoleOutput() : JansiWinSysTerminal.isConsoleOutput();
+    public boolean isConsoleOutput() {
+        if (OSUtils.IS_CYGWIN || OSUtils.IS_MSYSTEM) {
+            if (isAtLeast(2,1)) {
+                return JansiNativePty.isConsoleOutput();
+            } else {
+                throw new UnsupportedOperationException();
+            }
+        } else if (OSUtils.IS_WINDOWS) {
+            return JansiWinSysTerminal.isConsoleOutput();
+        }
+        return JansiNativePty.isConsoleOutput();
+    }
+
+    @Override
+    public boolean isConsoleInput() {
+        if (OSUtils.IS_CYGWIN || OSUtils.IS_MSYSTEM) {
+            if (isAtLeast(2,1)) {
+                return JansiNativePty.isConsoleInput();
+            } else {
+                throw new UnsupportedOperationException();
+            }
+        } else if (OSUtils.IS_WINDOWS) {
+            return JansiWinSysTerminal.isConsoleInput();
+        }
+        return JansiNativePty.isConsoleInput();
     }
+
 }

terminal-jansi/src/main/java/org/jline/terminal/impl/jansi/win/JansiWinSysTerminal.java

@@ -91,6 +91,11 @@ public static boolean isConsoleOutput() {
         return Kernel32.GetConsoleMode(consoleOut, mode) != 0;
     }
 
+    public static boolean isConsoleInput() {
+        int[] mode = new int[1];
+        return Kernel32.GetConsoleMode(consoleIn, mode) != 0;
+    }
+
     JansiWinSysTerminal(Writer writer, String name, String type, Charset encoding, int codepage, boolean nativeSignals, SignalHandler signalHandler) throws IOException {
         super(writer, name, type, encoding, codepage, nativeSignals, signalHandler);
     }

terminal-jna/src/main/java/org/jline/terminal/impl/jna/JnaNativePty.java

@@ -148,14 +148,22 @@ public String toString() {
     }
 
     public static boolean isConsoleOutput() {
+        return isatty(1);
+    }
+
+    public static boolean isConsoleInput() {
+        return isatty(0);
+    }
+
+    private static boolean isatty(int fd) {
         if (Platform.isMac()) {
-            return OsXNativePty.isatty(1) == 1;
+            return OsXNativePty.isatty(fd) == 1;
         } else if (Platform.isLinux()) {
-            return LinuxNativePty.isatty(1) == 1;
+            return LinuxNativePty.isatty(fd) == 1;
         } else if (Platform.isSolaris()) {
-            return SolarisNativePty.isatty(1) == 1;
+            return SolarisNativePty.isatty(fd) == 1;
         } else if (Platform.isFreeBSD()) {
-            return FreeBsdNativePty.isatty(1) == 1;
+            return FreeBsdNativePty.isatty(fd) == 1;
         } else {
             return false;
         }

terminal-jna/src/main/java/org/jline/terminal/impl/jna/JnaSupportImpl.java

@@ -14,6 +14,7 @@
 import org.jline.terminal.impl.jna.win.JnaWinSysTerminal;
 import org.jline.terminal.spi.JnaSupport;
 import org.jline.terminal.spi.Pty;
+import org.jline.utils.OSUtils;
 
 import java.io.IOException;
 import java.nio.charset.Charset;
@@ -45,8 +46,22 @@ public boolean isWindowsConsole() {
     }
 
     @Override
-    public boolean isConsoleOutput(boolean pty) {
-        return pty ? JnaNativePty.isConsoleOutput() : JnaWinSysTerminal.isConsoleOutput();
+    public boolean isConsoleOutput() {
+        if (OSUtils.IS_CYGWIN || OSUtils.IS_MSYSTEM) {
+            throw new UnsupportedOperationException();
+        } else if (OSUtils.IS_WINDOWS) {
+            return JnaWinSysTerminal.isConsoleOutput();
+        }
+        return JnaNativePty.isConsoleOutput();
     }
 
+    @Override
+    public boolean isConsoleInput() {
+        if (OSUtils.IS_CYGWIN || OSUtils.IS_MSYSTEM) {
+            throw new UnsupportedOperationException();
+        } else if (OSUtils.IS_WINDOWS) {
+            return JnaWinSysTerminal.isConsoleInput();
+        }
+        return JnaNativePty.isConsoleInput();
+    }
 }

terminal-jna/src/main/java/org/jline/terminal/impl/jna/win/JnaWinSysTerminal.java

@@ -87,6 +87,16 @@ public static boolean isConsoleOutput() {
         }
     }
 
+    public static boolean isConsoleInput() {
+        try {
+            IntByReference mode = new IntByReference();
+            Kernel32.INSTANCE.GetConsoleMode(consoleIn, mode);
+            return true;
+        } catch (LastErrorException e) {
+            return false;
+        }
+    }
+
     JnaWinSysTerminal(Writer writer, String name, String type, Charset encoding, int codepage, boolean nativeSignals, SignalHandler signalHandler) throws IOException {
         super(writer, name, type, encoding, codepage, nativeSignals, signalHandler);
         strings.put(InfoCmp.Capability.key_mouse, "\\E[M");