Add GNU make jobserver client support

- add new TokenPool interface
- GNU make implementation for TokenPool parses and verifies the magic
  information from the MAKEFLAGS environment variable
- RealCommandRunner tries to acquire TokenPool
  * if no token pool is available then there is no change in behaviour
- When a token pool is available then RealCommandRunner behaviour
  changes as follows
  * CanRunMore() only returns true if TokenPool::Acquire() returns true
  * StartCommand() calls TokenPool::Reserve()
  * WaitForCommand() calls TokenPool::Release()

Documentation for GNU make jobserver

  http://make.mad-scientist.net/papers/jobserver-implementation/

Fixes ninja-build#1139

Author: Stefan Becker

configure.py

@@ -494,6 +494,7 @@ def has_re2c():
     objs += cxx(name)
 if platform.is_windows():
     for name in ['subprocess-win32',
+                 'tokenpool-none',
                  'includes_normalize-win32',
                  'msvc_helper-win32',
                  'msvc_helper_main-win32']:
@@ -503,6 +504,7 @@ def has_re2c():
     objs += cc('getopt')
 else:
     objs += cxx('subprocess-posix')
+    objs += cxx('tokenpool-gnu-make')
 if platform.is_aix():
     objs += cc('getopt')
 if platform.is_msvc():

src/build.cc

@@ -33,6 +33,7 @@
 #include "graph.h"
 #include "state.h"
 #include "subprocess.h"
+#include "tokenpool.h"
 #include "util.h"
 
 namespace {
@@ -495,8 +496,8 @@ void Plan::Dump() {
 }
 
 struct RealCommandRunner : public CommandRunner {
-  explicit RealCommandRunner(const BuildConfig& config) : config_(config) {}
-  virtual ~RealCommandRunner() {}
+  explicit RealCommandRunner(const BuildConfig& config);
+  virtual ~RealCommandRunner();
   virtual bool CanRunMore();
   virtual bool StartCommand(Edge* edge);
   virtual bool WaitForCommand(Result* result);
@@ -505,9 +506,18 @@ struct RealCommandRunner : public CommandRunner {
 
   const BuildConfig& config_;
   SubprocessSet subprocs_;
+  TokenPool *tokens_;
   map<Subprocess*, Edge*> subproc_to_edge_;
 };
 
+RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) {
+  tokens_ = TokenPool::Get();
+}
+
+RealCommandRunner::~RealCommandRunner() {
+  delete tokens_;
+}
+
 vector<Edge*> RealCommandRunner::GetActiveEdges() {
   vector<Edge*> edges;
   for (map<Subprocess*, Edge*>::iterator e = subproc_to_edge_.begin();
@@ -518,21 +528,27 @@ vector<Edge*> RealCommandRunner::GetActiveEdges() {
 
 void RealCommandRunner::Abort() {
   subprocs_.Clear();
+  if (tokens_)
+    tokens_->Clear();
 }
 
 bool RealCommandRunner::CanRunMore() {
   size_t subproc_number =
       subprocs_.running_.size() + subprocs_.finished_.size();
   return (int)subproc_number < config_.parallelism
-    && ((subprocs_.running_.empty() || config_.max_load_average <= 0.0f)
-        || GetLoadAverage() < config_.max_load_average);
+    && (subprocs_.running_.empty()
+      || ((!tokens_ || tokens_->Acquire())
+        && (config_.max_load_average <= 0.0f
+          || GetLoadAverage() < config_.max_load_average)));
 }
 
 bool RealCommandRunner::StartCommand(Edge* edge) {
   string command = edge->EvaluateCommand();
   Subprocess* subproc = subprocs_.Add(command, edge->use_console());
   if (!subproc)
     return false;
+  if (tokens_)
+    tokens_->Reserve();
   subproc_to_edge_.insert(make_pair(subproc, edge));
 
   return true;
@@ -546,6 +562,9 @@ bool RealCommandRunner::WaitForCommand(Result* result) {
       return false;
   }
 
+  if (tokens_)
+    tokens_->Release();
+
   result->status = subproc->Finish();
   result->output = subproc->GetOutput();
 

src/tokenpool-gnu-make.cc

@@ -0,0 +1,148 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tokenpool.h"
+
+#include <fcntl.h>
+#include <poll.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+// TokenPool implementation for GNU make jobserver
+// (http://make.mad-scientist.net/papers/jobserver-implementation/)
+struct GNUmakeTokenPool : public TokenPool {
+  GNUmakeTokenPool();
+  virtual ~GNUmakeTokenPool();
+
+  virtual bool Acquire();
+  virtual void Reserve();
+  virtual void Release();
+  virtual void Clear();
+
+  bool Setup();
+
+ private:
+  int available_;
+  int used_;
+
+#ifdef _WIN32
+  // @TODO
+#else
+  int rfd_;
+  int wfd_;
+
+  bool CheckFd(int fd);
+#endif
+
+  void Return();
+};
+
+// every instance owns an implicit token -> available_ == 1
+GNUmakeTokenPool::GNUmakeTokenPool() : available_(1), used_(0),
+                                       rfd_(-1), wfd_(-1) {
+}
+
+GNUmakeTokenPool::~GNUmakeTokenPool() {
+  Clear();
+}
+
+bool GNUmakeTokenPool::CheckFd(int fd) {
+  if (fd < 0)
+    return false;
+  int ret = fcntl(fd, F_GETFD);
+  if (ret < 0)
+    return false;
+  return true;
+}
+
+bool GNUmakeTokenPool::Setup() {
+  const char *value = getenv("MAKEFLAGS");
+  if (value) {
+    const char *jobserver = strstr(value, "--jobserver-fds=");
+    if (jobserver) {
+      int rfd = -1;
+      int wfd = -1;
+      if ((sscanf(jobserver, "--jobserver-fds=%d,%d", &rfd, &wfd) == 2) &&
+          CheckFd(rfd) &&
+          CheckFd(wfd)) {
+        rfd_ = rfd;
+        wfd_ = wfd;
+        return true;
+      }
+    }
+  }
+
+  return false;
+}
+
+bool GNUmakeTokenPool::Acquire() {
+  if (available_ > 0)
+    return true;
+
+#ifdef USE_PPOLL
+  pollfd pollfds[] = {{rfd_, POLLIN, 0}};
+  int ret = poll(pollfds, 1, 0);
+#else
+  fd_set set;
+  struct timeval timeout = { 0, 0 };
+  FD_ZERO(&set);
+  FD_SET(rfd_, &set);
+  int ret = select(rfd_ + 1, &set, NULL, NULL, &timeout);
+#endif
+  if (ret > 0) {
+    char buf;
+    int ret = read(rfd_, &buf, 1);
+    if (ret > 0) {
+      available_++;
+      return true;
+    }
+  }
+  return false;
+}
+
+void GNUmakeTokenPool::Reserve() {
+  available_--;
+  used_++;
+}
+
+void GNUmakeTokenPool::Return() {
+  const char buf = '+';
+  if (write(wfd_, &buf, 1) > 0)
+    available_--;
+}
+
+void GNUmakeTokenPool::Release() {
+  available_++;
+  used_--;
+  if (available_ > 1)
+    Return();
+}
+
+void GNUmakeTokenPool::Clear() {
+  while (used_ > 0)
+    Release();
+  while (available_ > 1)
+    Return();
+}
+
+struct TokenPool *TokenPool::Get(void) {
+  GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool;
+  if (tokenpool->Setup())
+    return tokenpool;
+  else
+    delete tokenpool;
+  return NULL;
+}

src/tokenpool-none.cc

@@ -0,0 +1,27 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tokenpool.h"
+
+#include <fcntl.h>
+#include <poll.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+// No-op TokenPool implementation
+struct TokenPool *TokenPool::Get(void) {
+  return NULL;
+}

src/tokenpool.h

@@ -0,0 +1,26 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// interface to token pool
+struct TokenPool {
+  virtual ~TokenPool() {}
+
+  virtual bool Acquire() = 0;
+  virtual void Reserve() = 0;
+  virtual void Release() = 0;
+  virtual void Clear() = 0;
+
+  // returns NULL if token pool is not available
+  static struct TokenPool *Get(void);
+};