I used to think
System.console() != null
was a reliable way to determine whether the shell that launched my Java application was interactive or not. This allowed me to use ANSI escape sequences in interactive mode and plain System.out/System.err whenever the program's output was redirected to a file or piped to the stdin of some other process, similarly to --color=auto mode of many GNU utilities.
System.console() behaviour is different in Windows, however. While the method does return a non-null value when the JVM is launched from cmd.exe (which is useless for me, as cmd.exe doesn't understand escape sequences), the return value is always null when I launch my program from any of the terminal emulators available in Cygwin -- xterm, mintty or cygwin (the last one is merely a cmd.exe running a bash child process).
How do I test for an interactive shell in Java w/o reading $- in shell scripts and passing command-line args to my Java program? Testing for PS1 environment variable from Java is not an option, as Java is launched from a shell script, so the parent process is a non-interactive shell, and PS1 is unset.
There is a conversation where Cygwin's maintainer (Corinna Vinschen) explains that the Cygwin pseudo TTYs look like pipes to the Microsoft Visual C run-time library (MSVCRT). She also suggests to implement a wrapper around the isatty() function that recognizes Cygwin pseudo TTYs.
The idea is to fetch the name of the pipe associated with given file descriptor. The NtQueryInformationFile function fetches FILE_NAME_INFORMATION structure, where FileName member contains the pipe name. If the pipe name matches the following pattern, then it is very likely that the command is running in interactive mode:
\cygwin-%16llx-pty%d-{to,from}-master
The conversation is pretty old, but the format of pipe names is still the same:
"\\\\.\\pipe\\cygwin-" + "%S-" + + "pty%d-from-master", where "\\\\.\\pipe\\" is a convensional prefix for named pipes (see CreateNamedPipe).
So the Cygwin part is already hacked. The next step is to make a Java function from the C code.
Example
The following creates ttyjni.TestApp class with istty() method implemented via the Java Native Interface (JNI). The code is tested on GNU/Linux (x86_64) and Cygwin on Windows 7 (64-bit). The code can be easily ported to Windows (cmd.exe), maybe even works as is.
Required components
- Cygwin with
x86_64-w64-mingw32-gcccompiler - Windows with JDK
Layout
├── Makefile
├── TestApp.c
├── test.sh
├── ttyjni
│ └── TestApp.java
└── ttyjni_TestApp.h
Makefile
# Input: $JAVA_HOME
FINAL_TARGETS := TestApp.class
ifeq ($(OS),Windows_NT)
CC=x86_64-w64-mingw32-gcc
FINAL_TARGETS += testapp.dll
else
CC=gcc
FINAL_TARGETS += libtestapp.so
endif
all: $(FINAL_TARGETS)
TestApp.class: ttyjni/TestApp.java
javac $<
testapp.dll: TestApp.c TestApp.class
$(CC) \
-Wl,--add-stdcall-alias \
-D__int64="long long" \
-D_isatty=isatty -D_fileno=fileno \
-I"$(JAVA_HOME)/include" \
-I"$(JAVA_HOME)/include/win32" \
-shared -o $@ $<
libtestapp.so: TestApp.c
$(CC) \
-I"$(JAVA_HOME)/include" \
-I"$(JAVA_HOME)/include/linux" \
-fPIC \
-o $@ -shared -Wl,-soname,testapp.so $< \
-z noexecstack
clean:
rm -f *.o $(FINAL_TARGETS) ttyjni/*.class
TestApp.c
#include <jni.h>
#include <stdio.h>
#include "ttyjni_TestApp.h"
#if defined __CYGWIN__ || defined __MINGW32__ || defined __MINGW64__
#include <io.h>
#include <errno.h>
#include <wchar.h>
#include <windows.h>
#include <winternl.h>
#include <unistd.h>
/* vvvvvvvvvv From http://cygwin.com/ml/cygwin/2012-11/txt00003.txt vvvvvvvv */
#ifndef __MINGW64_VERSION_MAJOR
/* MS winternl.h defines FILE_INFORMATION_CLASS, but with only a
different single member. */
enum FILE_INFORMATION_CLASSX
{
FileNameInformation = 9
};
typedef struct _FILE_NAME_INFORMATION
{
ULONG FileNameLength;
WCHAR FileName[1];
} FILE_NAME_INFORMATION, *PFILE_NAME_INFORMATION;
NTSTATUS (NTAPI *pNtQueryInformationFile) (HANDLE, PIO_STATUS_BLOCK, PVOID,
ULONG, FILE_INFORMATION_CLASSX);
#else
NTSTATUS (NTAPI *pNtQueryInformationFile) (HANDLE, PIO_STATUS_BLOCK, PVOID,
ULONG, FILE_INFORMATION_CLASS);
#endif
jint
testapp_isatty(jint fd)
{
HANDLE fh;
NTSTATUS status;
IO_STATUS_BLOCK io;
long buf[66]; /* NAME_MAX + 1 + sizeof ULONG */
PFILE_NAME_INFORMATION pfni = (PFILE_NAME_INFORMATION) buf;
PWCHAR cp;
/* First check using _isatty.
Note that this returns the wrong result for NUL, for instance!
Workaround is not to use _isatty at all, but rather GetFileType
plus object name checking. */
if (_isatty(fd))
return 1;
/* Now fetch the underlying HANDLE. */
fh = (HANDLE)_get_osfhandle(fd);
if (!fh || fh == INVALID_HANDLE_VALUE) {
errno = EBADF;
return 0;
}
/* Must be a pipe. */
if (GetFileType (fh) != FILE_TYPE_PIPE)
goto no_tty;
/* Calling the native NT function NtQueryInformationFile is required to
support pre-Vista systems. If that's of no concern, Vista introduced
the GetFileInformationByHandleEx call with the FileNameInfo info class,
which can be used instead. */
if (!pNtQueryInformationFile) {
pNtQueryInformationFile = (NTSTATUS (NTAPI *)(HANDLE, PIO_STATUS_BLOCK,
PVOID, ULONG, FILE_INFORMATION_CLASS))
GetProcAddress(GetModuleHandle("ntdll.dll"), "NtQueryInformationFile");
if (!pNtQueryInformationFile)
goto no_tty;
}
if (!NT_SUCCESS (pNtQueryInformationFile (fh, &io, pfni, sizeof buf,
FileNameInformation)))
goto no_tty;
/* The filename is not guaranteed to be NUL-terminated. */
pfni->FileName[pfni->FileNameLength / sizeof (WCHAR)] = L'\0';
/* Now check the name pattern. The filename of a Cygwin pseudo tty pipe
looks like this:
\cygwin-%16llx-pty%d-{to,from}-master
%16llx is the hash of the Cygwin installation, (to support multiple
parallel installations), %d id the pseudo tty number, "to" or "from"
differs the pipe direction. "from" is a stdin, "to" a stdout-like
pipe. */
cp = pfni->FileName;
if (!wcsncmp(cp, L"\\cygwin-", 8)
&& !wcsncmp (cp + 24, L"-pty", 4))
{
cp = wcschr(cp + 28, '-');
if (!cp)
goto no_tty;
if (!wcscmp (cp, L"-from-master") || !wcscmp (cp, L"-to-master"))
return 1;
}
no_tty:
errno = EINVAL;
return 0;
}
/* ^^^^^^^^^^ From http://cygwin.com/ml/cygwin/2012-11/txt00003.txt ^^^^^^^^ */
#elif _WIN32
#include <io.h>
static jint
testapp_isatty(jint fd)
{
return _isatty(fd);
}
#elif defined __linux__ || defined __sun || defined __FreeBSD__
#include <unistd.h>
static jint
testapp_isatty(jint fd)
{
return isatty(fd);
}
#else
#error Unsupported platform
#endif /* __CYGWIN__ */
JNIEXPORT jboolean JNICALL Java_ttyjni_TestApp_istty
(JNIEnv *env, jobject obj)
{
return testapp_isatty(fileno(stdin)) &&
testapp_isatty(fileno(stdout)) ?
JNI_TRUE : JNI_FALSE;
}
ttyjni_TestApp.h
/* DO NOT EDIT THIS FILE - it is machine generated */
#include <jni.h>
/* Header for class ttyjni_TestApp */
#ifndef _Included_ttyjni_TestApp
#define _Included_ttyjni_TestApp
#ifdef __cplusplus
extern "C" {
#endif
/*
* Class: ttyjni_TestApp
* Method: istty
* Signature: ()Z
*/
JNIEXPORT jboolean JNICALL Java_ttyjni_TestApp_istty
(JNIEnv *, jobject);
#ifdef __cplusplus
}
#endif
#endif
ttyjni/TestApp.java
package ttyjni;
import java.io.Console;
import java.lang.reflect.Method;
class TestApp {
static {
System.loadLibrary("testapp");
}
private native boolean istty();
private static final String ISTTY_METHOD = "istty";
private static final String INTERACTIVE = "interactive";
private static final String NON_INTERACTIVE = "non-interactive";
protected static boolean isInteractive() {
try {
Method method = Console.class.getDeclaredMethod(ISTTY_METHOD);
method.setAccessible(true);
return (Boolean) method.invoke(Console.class);
} catch (Exception e) {
System.out.println(e.toString());
}
return false;
}
public static void main(String[] args) {
// Testing JNI
TestApp t = new TestApp();
boolean b = t.istty();
System.out.format("%s(jni)\n", b ?
"interactive" : "non-interactive");
// Testing pure Java
System.out.format("%s(console)\n", System.console() != null ?
INTERACTIVE : NON_INTERACTIVE);
System.out.format("%s(java)\n", isInteractive() ?
INTERACTIVE : NON_INTERACTIVE);
}
}
test.sh
#!/bin/bash -
java -Djava.library.path="$(dirname "$0")" ttyjni.TestApp
Compiling
make
Testing on Linux
$ ./test.sh
interactive(jni)
interactive(console)
interactive(java)
$ ./test.sh > 1
ruslan@pavilion ~/tmp/java $ cat 1
non-interactive(jni)
non-interactive(console)
non-interactive(java)
Testing on Cygwin
$ ./test.sh
interactive(jni)
non-interactive(console)
non-interactive(java)
$ ./test.sh > 1
$ cat 1
non-interactive(jni)
non-interactive(console)
non-interactive(java)