Merge branch 'master' into feature/search-request-refactoring

Author: colings86

core/src/main/java/org/elasticsearch/Version.java

@@ -253,7 +253,9 @@ public class Version {
     public static final int V_1_7_2_ID = 1070299;
     public static final Version V_1_7_2 = new Version(V_1_7_2_ID, false, org.apache.lucene.util.Version.LUCENE_4_10_4);
     public static final int V_1_7_3_ID = 1070399;
-    public static final Version V_1_7_3 = new Version(V_1_7_3_ID, true, org.apache.lucene.util.Version.LUCENE_4_10_4);
+    public static final Version V_1_7_3 = new Version(V_1_7_3_ID, false, org.apache.lucene.util.Version.LUCENE_4_10_4);
+    public static final int V_1_7_4_ID = 1070499;
+    public static final Version V_1_7_4 = new Version(V_1_7_4_ID, true, org.apache.lucene.util.Version.LUCENE_4_10_4);
 
     public static final int V_2_0_0_beta1_ID = 2000001;
     public static final Version V_2_0_0_beta1 = new Version(V_2_0_0_beta1_ID, false, org.apache.lucene.util.Version.LUCENE_5_2_1);
@@ -295,6 +297,8 @@ public static Version fromId(int id) {
                 return V_2_0_0_beta2;
             case V_2_0_0_beta1_ID:
                 return V_2_0_0_beta1;
+            case V_1_7_4_ID:
+                return V_1_7_4;
             case V_1_7_3_ID:
                 return V_1_7_3;
             case V_1_7_2_ID:

core/src/main/java/org/elasticsearch/bootstrap/JNANatives.java

@@ -46,6 +46,9 @@ private JNANatives() {}
     static boolean LOCAL_MLOCKALL = false;
     // Set to true, in case native seccomp call was successful
     static boolean LOCAL_SECCOMP = false;
+    // Set to true, in case policy can be applied to all threads of the process (even existing ones)
+    // otherwise they are only inherited for new threads (ES app threads)
+    static boolean LOCAL_SECCOMP_ALL = false;
 
     static void tryMlockall() {
         int errno = Integer.MIN_VALUE;
@@ -177,8 +180,11 @@ static void addConsoleCtrlHandler(ConsoleCtrlHandler handler) {
 
     static void trySeccomp(Path tmpFile) {
         try {
-            Seccomp.init(tmpFile);
+            int ret = Seccomp.init(tmpFile);
             LOCAL_SECCOMP = true;
+            if (ret == 1) {
+                LOCAL_SECCOMP_ALL = true;
+            }
         } catch (Throwable t) {
             // this is likely to happen unless the kernel is newish, its a best effort at the moment
             // so we log stacktrace at debug for now...

core/src/main/java/org/elasticsearch/bootstrap/Seccomp.java

@@ -120,7 +120,7 @@ static interface LinuxLibrary extends Library {
     static final int PR_SET_NO_NEW_PRIVS       =  38;   // since Linux 3.5
     static final int PR_GET_SECCOMP            =  21;   // since Linux 2.6.23
     static final int PR_SET_SECCOMP            =  22;   // since Linux 2.6.23
-    static final int SECCOMP_MODE_FILTER       =   2;   // since Linux Linux 3.5
+    static final long SECCOMP_MODE_FILTER      =   2;   // since Linux Linux 3.5
 
     /** corresponds to struct sock_filter */
     static final class SockFilter {
@@ -209,9 +209,10 @@ static SockFilter BPF_JUMP(int code, int k, int jt, int jf) {
     static final int NR_SYSCALL_FORK     = 57;
     static final int NR_SYSCALL_EXECVE   = 59;
     static final int NR_SYSCALL_EXECVEAT = 322;  // since Linux 3.19
+    static final int NR_SYSCALL_TUXCALL  = 184;  // should return ENOSYS
 
     /** try to install our BPF filters via seccomp() or prctl() to block execution */
-    private static void linuxImpl() {
+    private static int linuxImpl() {
         // first be defensive: we can give nice errors this way, at the very least.
         // also, some of these security features get backported to old versions, checking kernel version here is a big no-no! 
         boolean supported = Constants.LINUX && "amd64".equals(Constants.OS_ARCH);
@@ -224,24 +225,85 @@ private static void linuxImpl() {
             throw new UnsupportedOperationException("seccomp unavailable: could not link methods. requires kernel 3.5+ with CONFIG_SECCOMP and CONFIG_SECCOMP_FILTER compiled in");
         }
 
-        // check for kernel version
-        if (linux_libc.prctl(PR_GET_NO_NEW_PRIVS, 0, 0, 0, 0) < 0) {
+        // pure paranoia:
+
+        // check that unimplemented syscalls actually return ENOSYS
+        // you never know (e.g. https://code.google.com/p/chromium/issues/detail?id=439795)
+        if (linux_libc.syscall(NR_SYSCALL_TUXCALL) >= 0 || Native.getLastError() != ENOSYS) {
+            throw new UnsupportedOperationException("seccomp unavailable: your kernel is buggy and you should upgrade");
+        }
+
+        // try to check system calls really are who they claim
+        // you never know (e.g. https://chromium.googlesource.com/chromium/src.git/+/master/sandbox/linux/seccomp-bpf/sandbox_bpf.cc#57)
+        final int bogusArg = 0xf7a46a5c;
+
+        // test seccomp(BOGUS)
+        long ret = linux_libc.syscall(SECCOMP_SYSCALL_NR, bogusArg);
+        if (ret != -1) {
+            throw new UnsupportedOperationException("seccomp unavailable: seccomp(BOGUS_OPERATION) returned " + ret);
+        } else {
             int errno = Native.getLastError();
             switch (errno) {
-                case ENOSYS: throw new UnsupportedOperationException("seccomp unavailable: requires kernel 3.5+ with CONFIG_SECCOMP and CONFIG_SECCOMP_FILTER compiled in");
-                default: throw new UnsupportedOperationException("prctl(PR_GET_NO_NEW_PRIVS): " + JNACLibrary.strerror(errno));
+                case ENOSYS: break; // ok
+                case EINVAL: break; // ok
+                default: throw new UnsupportedOperationException("seccomp(BOGUS_OPERATION): " + JNACLibrary.strerror(errno));
             }
         }
-        // check for SECCOMP
-        if (linux_libc.prctl(PR_GET_SECCOMP, 0, 0, 0, 0) < 0) {
+
+        // test seccomp(VALID, BOGUS)
+        ret = linux_libc.syscall(SECCOMP_SYSCALL_NR, SECCOMP_SET_MODE_FILTER, bogusArg);
+        if (ret != -1) {
+            throw new UnsupportedOperationException("seccomp unavailable: seccomp(SECCOMP_SET_MODE_FILTER, BOGUS_FLAG) returned " + ret);
+        } else {
             int errno = Native.getLastError();
             switch (errno) {
-                case EINVAL: throw new UnsupportedOperationException("seccomp unavailable: CONFIG_SECCOMP not compiled into kernel, CONFIG_SECCOMP and CONFIG_SECCOMP_FILTER are needed");
-                default: throw new UnsupportedOperationException("prctl(PR_GET_SECCOMP): " + JNACLibrary.strerror(errno));
+                case ENOSYS: break; // ok
+                case EINVAL: break; // ok
+                default: throw new UnsupportedOperationException("seccomp(SECCOMP_SET_MODE_FILTER, BOGUS_FLAG): " + JNACLibrary.strerror(errno));
             }
         }
+
+        // test prctl(BOGUS)
+        ret = linux_libc.prctl(bogusArg, 0, 0, 0, 0);
+        if (ret != -1) {
+            throw new UnsupportedOperationException("seccomp unavailable: prctl(BOGUS_OPTION) returned " + ret);
+        } else {
+            int errno = Native.getLastError();
+            switch (errno) {
+                case ENOSYS: break; // ok
+                case EINVAL: break; // ok
+                default: throw new UnsupportedOperationException("prctl(BOGUS_OPTION): " + JNACLibrary.strerror(errno));
+            }
+        }
+
+        // now just normal defensive checks
+
+        // check for GET_NO_NEW_PRIVS
+        switch (linux_libc.prctl(PR_GET_NO_NEW_PRIVS, 0, 0, 0, 0)) {
+            case 0: break; // not yet set
+            case 1: break; // already set by caller
+            default:
+                int errno = Native.getLastError();
+                if (errno == ENOSYS) {
+                    throw new UnsupportedOperationException("seccomp unavailable: requires kernel 3.5+ with CONFIG_SECCOMP and CONFIG_SECCOMP_FILTER compiled in");
+                } else {
+                    throw new UnsupportedOperationException("prctl(PR_GET_NO_NEW_PRIVS): " + JNACLibrary.strerror(errno));
+                }
+        }
+        // check for SECCOMP
+        switch (linux_libc.prctl(PR_GET_SECCOMP, 0, 0, 0, 0)) {
+            case 0: break; // not yet set
+            case 2: break; // already in filter mode by caller
+            default:
+                int errno = Native.getLastError();
+                if (errno == EINVAL) {
+                    throw new UnsupportedOperationException("seccomp unavailable: CONFIG_SECCOMP not compiled into kernel, CONFIG_SECCOMP and CONFIG_SECCOMP_FILTER are needed");
+                } else {
+                    throw new UnsupportedOperationException("prctl(PR_GET_SECCOMP): " + JNACLibrary.strerror(errno));
+                }
+        }
         // check for SECCOMP_MODE_FILTER
-        if (linux_libc.prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, 0, 0, 0) < 0) {
+        if (linux_libc.prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, 0, 0, 0) != 0) {
             int errno = Native.getLastError();
             switch (errno) {
                 case EFAULT: break; // available
@@ -251,10 +313,15 @@ private static void linuxImpl() {
         }
 
         // ok, now set PR_SET_NO_NEW_PRIVS, needed to be able to set a seccomp filter as ordinary user
-        if (linux_libc.prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) < 0) {
+        if (linux_libc.prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0) {
             throw new UnsupportedOperationException("prctl(PR_SET_NO_NEW_PRIVS): " + JNACLibrary.strerror(Native.getLastError()));
         }
 
+        // check it worked
+        if (linux_libc.prctl(PR_GET_NO_NEW_PRIVS, 0, 0, 0, 0) != 1) {
+            throw new UnsupportedOperationException("seccomp filter did not really succeed: prctl(PR_GET_NO_NEW_PRIVS): " + JNACLibrary.strerror(Native.getLastError()));
+        }
+        
         // BPF installed to check arch, then syscall range. See https://www.kernel.org/doc/Documentation/prctl/seccomp_filter.txt for details.
         SockFilter insns[] = {
           /* 1 */ BPF_STMT(BPF_LD  + BPF_W   + BPF_ABS, SECCOMP_DATA_ARCH_OFFSET),               //
@@ -272,14 +339,16 @@ private static void linuxImpl() {
         prog.write();
         long pointer = Pointer.nativeValue(prog.getPointer());
 
+        int method = 1;
         // install filter, if this works, after this there is no going back!
         // first try it with seccomp(SECCOMP_SET_MODE_FILTER), falling back to prctl()
         if (linux_libc.syscall(SECCOMP_SYSCALL_NR, SECCOMP_SET_MODE_FILTER, SECCOMP_FILTER_FLAG_TSYNC, pointer) != 0) {
+            method = 0;
             int errno1 = Native.getLastError();
             if (logger.isDebugEnabled()) {
                 logger.debug("seccomp(SECCOMP_SET_MODE_FILTER): " + JNACLibrary.strerror(errno1) + ", falling back to prctl(PR_SET_SECCOMP)...");
             }
-            if (linux_libc.prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, pointer, 0, 0) < 0) {
+            if (linux_libc.prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, pointer, 0, 0) != 0) {
                 int errno2 = Native.getLastError();
                 throw new UnsupportedOperationException("seccomp(SECCOMP_SET_MODE_FILTER): " + JNACLibrary.strerror(errno1) + 
                                                         ", prctl(PR_SET_SECCOMP): " + JNACLibrary.strerror(errno2));
@@ -291,7 +360,8 @@ private static void linuxImpl() {
             throw new UnsupportedOperationException("seccomp filter installation did not really succeed. seccomp(PR_GET_SECCOMP): " + JNACLibrary.strerror(Native.getLastError()));
         }
 
-        logger.debug("Linux seccomp filter installation successful");
+        logger.debug("Linux seccomp filter installation successful, threads: [{}]", method == 1 ? "all" : "app" );
+        return method;
     }
 
     // OS X implementation via sandbox(7)
@@ -334,7 +404,7 @@ private static void macImpl(Path tmpFile) throws IOException {
         // first be defensive: we can give nice errors this way, at the very least.
         boolean supported = Constants.MAC_OS_X;
         if (supported == false) {
-            throw new IllegalStateException("bug: should not be trying to initialize seccomp for an unsupported OS");
+            throw new IllegalStateException("bug: should not be trying to initialize seatbelt for an unsupported OS");
         }
 
         // we couldn't link methods, could be some really ancient OS X (< Leopard) or some bug
@@ -372,12 +442,14 @@ private static void macImpl(Path tmpFile) throws IOException {
      * Attempt to drop the capability to execute for the process.
      * <p>
      * This is best effort and OS and architecture dependent. It may throw any Throwable.
+     * @return 0 if we can do this for application threads, 1 for the entire process
      */
-    static void init(Path tmpFile) throws Throwable {
+    static int init(Path tmpFile) throws Throwable {
         if (Constants.LINUX) {
-            linuxImpl();
+            return linuxImpl();
         } else if (Constants.MAC_OS_X) {
             macImpl(tmpFile);
+            return 1;
         } else {
             throw new UnsupportedOperationException("syscall filtering not supported for OS: '" + Constants.OS_NAME + "'");
         }

core/src/test/java/org/elasticsearch/bootstrap/SeccompTests.java

@@ -30,6 +30,16 @@ public void setUp() throws Exception {
         assumeTrue("requires seccomp filter installation", Natives.isSeccompInstalled());
         // otherwise security manager will block the execution, no fun
         assumeTrue("cannot test with security manager enabled", System.getSecurityManager() == null);
+        // otherwise, since we don't have TSYNC support, rules are not applied to the test thread
+        // (randomizedrunner class initialization happens in its own thread, after the test thread is created)
+        // instead we just forcefully run it for the test thread here.
+        if (!JNANatives.LOCAL_SECCOMP_ALL) {
+            try {
+                Seccomp.init(createTempDir());
+            } catch (Throwable e) {
+                throw new RuntimeException("unable to forcefully apply seccomp to test thread", e);
+            }
+        }
     }
 
     public void testNoExecution() throws Exception {

core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java

@@ -28,6 +28,7 @@
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.Version;
 import org.elasticsearch.test.ESTestCase;
@@ -359,4 +360,16 @@ public void testCount() throws Exception {
         w.close();
         dir.close();
     }
+
+    /**
+     * Test that the "unmap hack" is detected as supported by lucene.
+     * This works around the following bug: https://bugs.openjdk.java.net/browse/JDK-4724038
+     * <p>
+     * While not guaranteed, current status is "Critical Internal API": http://openjdk.java.net/jeps/260
+     * Additionally this checks we did not screw up the security logic around the hack.
+     */
+    public void testMMapHackSupported() throws Exception {
+        // add assume's here if needed for certain platforms, but we should know if it does not work.
+        assertTrue(MMapDirectory.UNMAP_SUPPORTED);
+    }
 }