Merge branch 'master' into patch-1

Author: BanzaiMan

.gitignore

@@ -12,3 +12,4 @@ tmp
 .env
 NOTES.md
 .envrc
+/db/

.travis.yml

@@ -1,27 +1,25 @@
+language: ruby
 dist: trusty
-sudo: false
-dist: trusty
-
 rvm: 2.4.2
 
-cache: bundler
+cache:
+  bundler: true
 
 services:
-  - redis
-  - rabbitmq
+- redis
+- rabbitmq
 
 addons:
-  postgresql: 9.3
+  postgresql: 9.6
 
 env:
   matrix:
   - RAKE_TASK=spec
   global:
     secure: BdVC3OHqYcgePLrkKIk28Ewn/dxCYFf3Cx+Q8P+BCDj6UPJyRSbKmILBzuX96H5xhKmUFo0A/upUhJI9UUP9aXHO7MzRe04/c88QdO4wGacVUaIyB20S0pr262zbc/nA50K9cVgpmWc64n6uQR1tgM6ZyyBnBeXkLzCAOHPq99I=
 
-before_install:
-  - gem install bundler
-before_script:
-  - bundle exec rake db:create --trace
-script:
-  - bundle exec rspec spec
+before_install: gem install bundler
+
+before_script: bundle exec rake db:create --trace
+
+script: bundle exec rspec spec

Gemfile

@@ -5,7 +5,7 @@ ruby '2.4.2' if ENV['DYNO']
 gem 'travis-config',      '~> 1.1.3'
 gem 'travis-lock'
 gem 'travis-metrics',     git: 'https://github.com/travis-ci/travis-metrics'
-gem 'travis-rollout',     git: 'https://github.com/travis-ci/travis-rollout', ref: 'sf-refactor'
+gem 'travis-rollout',     git: 'https://github.com/travis-ci/travis-rollout'
 gem 'travis-exceptions',  git: 'https://github.com/travis-ci/travis-exceptions'
 gem 'travis-logger',      git: 'https://github.com/travis-ci/travis-logger'
 gem 'travis-settings',    git: 'https://github.com/travis-ci/travis-settings'
@@ -22,7 +22,7 @@ gem 'sidekiq-pro', require: 'sidekiq-pro', source: 'https://gems.contribsys.com'
 gem 'jemalloc'
 gem 'redis-namespace'
 gem 'activerecord',       '~> 4.2.7'
-gem 'bunny',              '~> 2.6.1'
+gem 'bunny',              '~> 2.9.2'
 gem 'pg'
 gem 'concurrent-ruby'
 gem 'sentry-raven'
@@ -33,9 +33,13 @@ gem 'rack', '1.6.4'
 
 gem 'libhoney'
 
+group :development, :test do
+  gem 'pry'
+end
+
 group :test do
   gem 'rake'
-  gem 'database_cleaner', '~> 1.5.1'
+  gem 'database_cleaner', '~> 1.7'
   gem 'factory_girl',     '~> 4.7.0'
   gem 'mocha',            '~> 0.10.0'
   gem 'rspec'

Gemfile.lock

@@ -62,8 +62,7 @@ GIT
 
 GIT
   remote: https://github.com/travis-ci/travis-rollout
-  revision: 643477fb44ea35d483ed24cc89ced459719c6e6c
-  ref: sf-refactor
+  revision: ac0184411b3618dd5e5706db8fe7676a107c7b07
   specs:
     travis-rollout (0.0.1)
 
@@ -94,7 +93,7 @@ GEM
       tzinfo (~> 1.1)
     addressable (2.5.1)
       public_suffix (~> 2.0, >= 2.0.2)
-    amq-protocol (2.0.1)
+    amq-protocol (2.3.0)
     arel (6.0.4)
     atomic (1.1.99)
     avl_tree (1.2.1)
@@ -105,19 +104,20 @@ GEM
       thread_safe (~> 0.3, >= 0.3.1)
     backports (3.10.3)
     builder (3.2.3)
-    bunny (2.6.1)
-      amq-protocol (>= 2.0.1)
+    bunny (2.9.2)
+      amq-protocol (~> 2.3.0)
     cl (0.0.4)
+    coderay (1.1.2)
     coercible (1.0.0)
       descendants_tracker (~> 0.0.1)
     concurrent-ruby (1.0.5)
     connection_pool (2.2.1)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
-    database_cleaner (1.5.3)
+    database_cleaner (1.7.0)
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
-    diff-lcs (1.2.5)
+    diff-lcs (1.3)
     domain_name (0.5.20170404)
       unf (>= 0.0.5, < 1.0.0)
     equalizer (0.0.11)
@@ -144,6 +144,7 @@ GEM
     libhoney (1.3.2)
       http (~> 2.0)
     metaclass (0.0.4)
+    method_source (0.9.0)
     minitest (5.11.3)
     mocha (0.10.5)
       metaclass (~> 0.0.1)
@@ -152,9 +153,12 @@ GEM
     net-http-persistent (2.9.4)
     net-http-pipeline (1.0.1)
     pg (0.21.0)
+    pry (0.11.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
     public_suffix (2.0.5)
     rack (1.6.4)
-    rack-protection (1.5.3)
+    rack-protection (2.0.1)
       rack
     rake (11.3.0)
     redis (3.3.3)
@@ -163,19 +167,19 @@ GEM
     redlock (0.1.8)
       redis (~> 3, >= 3.0.0)
     rollout (1.1.0)
-    rspec (3.5.0)
-      rspec-core (~> 3.5.0)
-      rspec-expectations (~> 3.5.0)
-      rspec-mocks (~> 3.5.0)
-    rspec-core (3.5.4)
-      rspec-support (~> 3.5.0)
-    rspec-expectations (3.5.0)
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.1)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.5.0)
-    rspec-mocks (3.5.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.5.0)
-    rspec-support (3.5.0)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.1)
     safe_yaml (1.0.4)
     sentry-raven (2.6.3)
       faraday (>= 0.7.6, < 1.0)
@@ -210,11 +214,11 @@ PLATFORMS
 
 DEPENDENCIES
   activerecord (~> 4.2.7)
-  bunny (~> 2.6.1)
+  bunny (~> 2.9.2)
   cl
   coder!
   concurrent-ruby
-  database_cleaner (~> 1.5.1)
+  database_cleaner (~> 1.7)
   factory_girl (~> 4.7.0)
   gh!
   jemalloc
@@ -225,6 +229,7 @@ DEPENDENCIES
   mocha (~> 0.10.0)
   multi_json (~> 1.11)
   pg
+  pry
   rack (= 1.6.4)
   rake
   redis-namespace

Procfile

@@ -1,3 +1,3 @@
 console:   bundle exec je ./bin/console
-sidekiq:   bundle exec je bin/sidekiq ${SIDEKIQ_CONCURRENCY:-5} ${SIDEKIQ_QUEUE:-scheduler}
-scheduler: bundle exec je bin/sidekiq ${SIDEKIQ_CONCURRENCY:-5} ${SIDEKIQ_QUEUE:-scheduler}
+sidekiq:   bundle exec je bin/sidekiq-pgbouncer ${SIDEKIQ_CONCURRENCY:-5} ${SIDEKIQ_QUEUE:-scheduler}
+scheduler: bundle exec je bin/sidekiq-pgbouncer ${SIDEKIQ_CONCURRENCY:-5} ${SIDEKIQ_QUEUE:-scheduler}

README.md

@@ -29,30 +29,26 @@ only queue 2 more jobs in order to respect the limit of 5.
 The log output for each of these evaluation rounds looks like this:
 
 ```
-one-org: total: 5, running: 8, max: 20, queueable: 5
-another-org: total: 20, running: 2, max: 2, queueable: 0
+user sven public capacity: total=3 running=1 selected=2
+user sven boost capacity: total=2 running=0 selected=2
+repo sven/repo: queueable=5 running=1 selected=4 waiting=1
+user sven: queueable=5 running=1 selected=4 total_waiting=1 waiting_for_concurrency=1
 ```
 
 The terminology used here can be confusing. The terms mean:
 
-* `total` is the number of jobs waiting in the `created` state
-* `running` is the number of jobs already running, so the number of jobs that
-   have state `queued`, `received`, or `started`
-* `max` is the concurrency limit for the given owner, i.e. the maximum number
-   of jobs
-* `queueable` is the result of the evaluation: how many jobs can be queued up,
-   i.e. "sent to the workers", at this moment
-
-In the example log output above, on the first line the owner `one-org` has a
-concurrency limit of 20. It has 5 jobs waiting to be queued ("total"), and
-already has 8 jobs running at this moment. Therefore the scheduler can queue up
-5 more jobs for the workers to pick them up and execute them. (And, if someone
-on this org would have pushed 12 more jobs in this moment, it could queue up
-all 12 of them, too.)
-
-On the second line the owner `another-org` has a concurrency limit of 2, 20
-jobs waiting ("total"), and 2 already running. Therefore the scheduler can not
-queue up any more jobs (`queueable: 0`) at this moment.
+* `total` - number of concurrent jobs, provided by public capacity, plan, boost, etc.
+* `running` - number of jobs currently running, i.e. in the state `queued`, `received`, or `started`
+* `queueable` - number of jobs in the state `created`
+* `selected` - number of queueable jobs that are being selected to be queued based on concurrency limits
+* `total_waiting` - total number of queueable jobs that have not been selected to be queued
+* `waiting_for_concurrency` - number of queueable jobs that have not been selected, and have not been found to be limited by repo settings, queue, or stages
+
+In the example log output above, the owner has a capacity of 3 concurrent jobs
+provided by `public` capacity (line 1), and 2 jobs provided by `boost` capacity
+(line 2). The job selection finds 5 jobs to be queueable (i.e. in the state
+`created`), and 1 job to be running. As the total capacity is 5 jobs it can
+select 4 jobs to be queued for the workers, leaving 1 job waiting.
 
 ## Contributing
 

TODO.md

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# this script enables a pgbouncer wrapper when the
+# $PGBOUNCER_ENABLED variable is set to 'true'
+# or if the space-delimited list
+# $PGBOUNCER_ENABLED_FOR_DYNOS contains $DYNO
+
+cd "$(dirname "$0")/.."
+
+for d in $PGBOUNCER_ENABLED_FOR_DYNOS; do
+  if [ "$d" = "$DYNO" ]; then
+    export PGBOUNCER_ENABLED=true
+  fi
+done
+
+if [ ! -f "bin/start-pgbouncer-stunnel" ]; then
+  echo "warning: pgbouncer buildpack not found, setting PGBOUNCER_ENABLED=false"
+  export PGBOUNCER_ENABLED=false
+fi
+
+if [ "$PGBOUNCER_ENABLED" = 'true' ]; then
+  export PGBOUNCER_PREPARED_STATEMENTS=false
+  exec bin/start-pgbouncer-stunnel bin/sidekiq "$@"
+else
+  exec bin/sidekiq "$@"
+fi

bin/sidekiq-pgbouncer

@@ -7,13 +7,16 @@
 
 module Travis
   module Owners
+    ArgumentError = Class.new(::ArgumentError)
+
     class << self
-      def group(owner, config)
+      def group(owner, config, logger = nil)
         owner = find(owner) if owner.is_a?(Hash)
-        Group.new(owners(owner, config), config)
+        Group.new(owners(owner, config), config, logger)
       end
 
       def find(owner)
+        raise ArgumentError, 'Invalid owner data: %p' % owner unless owner[:owner_type]
         Kernel.const_get(owner[:owner_type]).find(owner[:owner_id])
       end
 

lib/travis/owners.rb

@@ -27,7 +27,7 @@ def login
         end
 
         def delegate
-          config[login.to_sym]
+          config[login.try(:to_sym)]
         end
 
         def delegatees(*logins)

lib/travis/owners/config.rb

@@ -1,6 +1,6 @@
 module Travis
   module Owners
-    class Group < Struct.new(:all, :config)
+    class Group < Struct.new(:all, :config, :logger)
       include Enumerable
 
       def each(&block)
@@ -12,12 +12,13 @@ def key
       end
 
       def logins
-        @login ||= all.map(&:login).sort
+        @logins ||= all.map(&:login).compact.sort
       end
 
-      def max_jobs
-        subscriptions.max_jobs
+      def paid_capacity
+        subscriptions.capacity
       end
+      alias max_jobs paid_capacity
 
       def subscribed?
         subscriptions.active?
@@ -27,6 +28,10 @@ def subscribed_owners
         subscriptions.subscribers
       end
 
+      def educational?
+        all.any?(&:educational?)
+      end
+
       def ==(other)
         key == other.key
       end
@@ -43,7 +48,7 @@ def public_mode?(redis)
       private
 
         def subscriptions
-          @subscriptions ||= Subscriptions.new(self, plans)
+          @subscriptions ||= Subscriptions.new(self, plans, logger)
         end
 
         def plans

lib/travis/owners/group.rb

@@ -1,13 +1,18 @@
 module Travis
   module Owners
-    class Subscriptions < Struct.new(:owners, :config)
+    class Subscriptions < Struct.new(:owners, :config, :logger)
+      MSGS = {
+        missing_plan: '[missing_plan] Plan missing from application config: %s (%s)'
+      }
+
       def active?
         subscriptions.any?
       end
 
-      def max_jobs
-        @max_jobs ||= plan_limits.inject(&:+).to_i
+      def capacity
+        @capacity ||= plan_limits.inject(&:+).to_i
       end
+      alias max_jobs capacity
 
       def subscribers
         @subscribers ||= subscriptions.map(&:owner).map(&:login)
@@ -20,7 +25,11 @@ def plan_limits
         end
 
         def plan_limit(plan)
-          config[plan.to_sym]
+          config[plan.to_sym].tap { |limit| missing_plan(plan) unless limit }
+        end
+
+        def missing_plan(plan)
+          logger.warn MSGS[:missing_plan] % [plan, owners.to_s]
         end
 
         def plans