implement a build cache for build watch events that come before bc events for bc creation, then trigger build job when bc arrives

Author: gabemontero

src/main/java/io/fabric8/jenkins/openshiftsync/BuildConfigToJobMapper.java

@@ -55,7 +55,7 @@ public class BuildConfigToJobMapper {
 
     public static FlowDefinition mapBuildConfigToFlow(BuildConfig bc)
             throws IOException {
-        if (!OpenShiftUtils.isJenkinsBuildConfig(bc)) {
+        if (!OpenShiftUtils.isPipelineStrategyBuildConfig(bc)) {
             return null;
         }
 

src/main/java/io/fabric8/jenkins/openshiftsync/BuildConfigWatcher.java

@@ -99,6 +99,10 @@ public void doRun() {
                                 e);
                     }
                 }
+                // poke the BuildWatcher builds with no BC list and see if we
+                // can create job
+                // runs for premature builds
+                BuildWatcher.flushBuildsWithNoBCList();
             }
         };
     }
@@ -139,6 +143,10 @@ public synchronized void eventReceived(Watcher.Action action,
                 modifyEventToJenkinsJob(buildConfig);
                 break;
             }
+            // if bc event came after build events, let's
+            // poke the BuildWatcher builds with no BC list to create job
+            // runs
+            BuildWatcher.flushBuildsWithNoBCList();
         } catch (Exception e) {
             logger.log(Level.WARNING, "Caught: " + e, e);
         }
@@ -163,7 +171,7 @@ private void updateJob(WorkflowJob job, InputStream jobStream,
     }
 
     private void upsertJob(final BuildConfig buildConfig) throws Exception {
-        if (isJenkinsBuildConfig(buildConfig)) {
+        if (isPipelineStrategyBuildConfig(buildConfig)) {
             // sync on intern of name should guarantee sync on same actual obj
             synchronized (buildConfig.getMetadata().getUid().intern()) {
                 ACL.impersonate(ACL.SYSTEM,
@@ -298,7 +306,7 @@ public Void call() throws Exception {
 
     private synchronized void modifyEventToJenkinsJob(BuildConfig buildConfig)
             throws Exception {
-        if (isJenkinsBuildConfig(buildConfig)) {
+        if (isPipelineStrategyBuildConfig(buildConfig)) {
             upsertJob(buildConfig);
             return;
         }

src/main/java/io/fabric8/jenkins/openshiftsync/BuildWatcher.java

@@ -34,6 +34,7 @@
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.logging.Level;
@@ -55,6 +56,18 @@ public class BuildWatcher extends BaseWatcher implements Watcher<Build> {
     private static final Logger logger = Logger.getLogger(BuildWatcher.class
             .getName());
 
+    // the fabric8 classes like Build have equal/hashcode annotations that
+    // should allow
+    // us to index via the objects themselves;
+    // now that listing interval is 5 minutes (used to be 10 seconds), we have
+    // seen
+    // timing windows where if the build watch events come before build config
+    // watch events
+    // when both are created in a simultaneous fashion, there is an up to 5
+    // minute delay
+    // before the job run gets kicked off
+    private static final HashSet<Build> buildsWithNoBCList = new HashSet<Build>();
+
     @SuppressFBWarnings("EI_EXPOSE_REP2")
     public BuildWatcher(String[] namespaces) {
         super(namespaces);
@@ -69,6 +82,11 @@ public void doRun() {
                     logger.fine("No Openshift Token credential defined.");
                     return;
                 }
+                // prior to finding new builds poke the BuildWatcher builds with
+                // no BC list and see if we
+                // can create job runs for premature builds we already know
+                // about
+                BuildWatcher.flushBuildsWithNoBCList();
                 for (String namespace : namespaces) {
                     try {
                         logger.fine("listing Build resources");
@@ -113,6 +131,8 @@ public void start() {
     @SuppressFBWarnings("SF_SWITCH_NO_DEFAULT")
     @Override
     public synchronized void eventReceived(Action action, Build build) {
+        if (!OpenShiftUtils.isPipelineStrategyBuild(build))
+            return;
         try {
             switch (action) {
             case ADDED:
@@ -151,6 +171,8 @@ public int compare(Build b1, Build b2) {
             Map<BuildConfig, List<Build>> buildConfigBuildMap = new HashMap<>(
                     items.size());
             for (Build b : items) {
+                if (!OpenShiftUtils.isPipelineStrategyBuild(b))
+                    continue;
                 String buildConfigName = b.getStatus().getConfig().getName();
                 if (StringUtils.isEmpty(buildConfigName)) {
                     continue;
@@ -163,6 +185,9 @@ public int compare(Build b1, Build b2) {
                             .inNamespace(namespace).withName(buildConfigName)
                             .get();
                     if (bc == null) {
+                        // if the bc is not there via a REST get, then it is not
+                        // going to be, and we are not handling manual creation
+                        // of pipeline build objects, so don't bother with "no bc list"
                         continue;
                     }
                     buildConfigMap.put(bcMapKey, bc);
@@ -185,11 +210,25 @@ public int compare(Build b1, Build b2) {
                 }
                 WorkflowJob job = getJobFromBuildConfig(bc);
                 if (job == null) {
+                    List<Build> builds = buildConfigBuilds.getValue();
+                    for (Build b : builds) {
+                        logger.info("skipping listed new build "
+                                + b.getMetadata().getName()
+                                + " no job at this time");
+                        addBuildToNoBCList(b);
+                    }
                     continue;
                 }
                 BuildConfigProjectProperty bcp = job
                         .getProperty(BuildConfigProjectProperty.class);
                 if (bcp == null) {
+                    List<Build> builds = buildConfigBuilds.getValue();
+                    for (Build b : builds) {
+                        logger.info("skipping listed new build "
+                                + b.getMetadata().getName()
+                                + " no prop at this time");
+                        addBuildToNoBCList(b);
+                    }
                     continue;
                 }
                 List<Build> builds = buildConfigBuilds.getValue();
@@ -204,12 +243,20 @@ private static synchronized void modifyEventToJenkinsJobRun(Build build) {
             WorkflowJob job = getJobFromBuild(build);
             if (job != null) {
                 cancelBuild(job, build);
+            } else {
+                removeBuildFromNoBCList(build);
             }
+        } else {
+            // see if any pre-BC cached builds can now be flushed
+            flushBuildsWithNoBCList();
         }
     }
 
     public static synchronized boolean addEventToJenkinsJobRun(Build build)
             throws IOException {
+        // should have been caught upstack, but just in case since public method
+        if (!OpenShiftUtils.isPipelineStrategyBuild(build))
+            return false;
         BuildStatus status = build.getStatus();
         if (status != null) {
             if (isCancelled(status)) {
@@ -225,9 +272,48 @@ public static synchronized boolean addEventToJenkinsJobRun(Build build)
         if (job != null) {
             return triggerJob(job, build);
         }
+        logger.info("skipping watch event for build "
+                + build.getMetadata().getName() + " no job at this time");
+        addBuildToNoBCList(build);
         return false;
     }
 
+    public static synchronized void addBuildToNoBCList(Build build) {
+        // should have been caught upstack, but just in case since public method
+        if (!OpenShiftUtils.isPipelineStrategyBuild(build))
+            return;
+        buildsWithNoBCList.add(build);
+    }
+
+    private static synchronized void removeBuildFromNoBCList(Build build) {
+        buildsWithNoBCList.remove(build);
+    }
+
+    private static synchronized void clearNoBCList() {
+        buildsWithNoBCList.clear();
+    }
+
+    // trigger any builds whose watch events arrived before the
+    // corresponding build config watch events
+    public static synchronized void flushBuildsWithNoBCList() {
+        HashSet<Build> clone = (HashSet<Build>) buildsWithNoBCList.clone();
+        clearNoBCList();
+        for (Build build : clone) {
+            WorkflowJob job = getJobFromBuild(build);
+            if (job != null)
+                try {
+                    logger.info("triggering job run for previously skipped build "
+                            + build.getMetadata().getName());
+                    triggerJob(job, build);
+                } catch (IOException e) {
+                    logger.log(Level.WARNING, "flushCachedBuilds", e);
+                }
+            else
+                addBuildToNoBCList(build);
+        }
+
+    }
+
     // innerDeleteEventToJenkinsJobRun is the actual delete logic at the heart
     // of deleteEventToJenkinsJobRun
     // that is either in a sync block or not based on the presence of a BC uid
@@ -243,6 +329,10 @@ public Void call() throws Exception {
                             return null;
                         }
                     });
+        } else {
+            // in case build was created and deleted quickly, prior to seeing BC
+            // event, clear out from pre-BC cache
+            removeBuildFromNoBCList(build);
         }
     }
 

src/main/java/io/fabric8/jenkins/openshiftsync/JenkinsUtils.java

@@ -623,6 +623,8 @@ public int compare(Build b1, Build b2) {
         boolean jobIsBuilding = job.isBuilding();
         for (int i = 0; i < builds.size(); i++) {
             Build b = builds.get(i);
+            if (!OpenShiftUtils.isPipelineStrategyBuild(b))
+                continue;
             // For SerialLatestOnly we should try to cancel all builds before
             // the latest one requested.
             if (isSerialLatestOnly) {

src/main/java/io/fabric8/jenkins/openshiftsync/OpenShiftUtils.java

@@ -124,7 +124,7 @@ public synchronized static void shutdownOpenShiftClient() {
      * @return true if this is an OpenShift BuildConfig which should be mirrored
      *         to a Jenkins Job
      */
-    public static boolean isJenkinsBuildConfig(BuildConfig bc) {
+    public static boolean isPipelineStrategyBuildConfig(BuildConfig bc) {
         if (BuildConfigToJobMapper.JENKINS_PIPELINE_BUILD_STRATEGY
                 .equalsIgnoreCase(bc.getSpec().getStrategy().getType())
                 && bc.getSpec().getStrategy().getJenkinsPipelineStrategy() != null) {
@@ -144,6 +144,15 @@ public static boolean isJenkinsBuildConfig(BuildConfig bc) {
         return false;
     }
 
+    public static boolean isPipelineStrategyBuild(Build b) {
+        if (BuildConfigToJobMapper.JENKINS_PIPELINE_BUILD_STRATEGY
+                .equalsIgnoreCase(b.getSpec().getStrategy().getType())
+                && b.getSpec().getStrategy().getJenkinsPipelineStrategy() != null) {
+            return true;
+        }
+        return false;
+    }
+
     /**
      * Finds the Jenkins job name for the given {@link BuildConfig}.
      *