(maint) integration: use puppetserver uberjar to install gems

Currently, puppetserver has transitioned to jetty10 and puppetdb is
still using jetty9 creating dependency conflicts when integration tests
are run. Once we transition to jetty10 as well, there will still be
dependency conflicts when we run integration tests against puppetserver
7.x, which is still on jetty9. Remove puppetserver from the project file
to avoid dependency conflicts.

d7e619d moved puppetserver to run in
its own JVM for integration tests, so it is no longer necessary to have
puppetserver in our project.clj file, but we still had it there to
facilitate installing the vendored gems that puppetserver needs. This
moves gem installation for puppetserver out of our project file and
clojure code, and instead uses the test setup scripts and the
puppetserver uberjar to create the vendored gem directory.

The jetty service name changed from jetty9-service to jetty10-service.
Adds an additional bootstrap file, used only for 7.x integration
testing, so each puppetserver version loads the correct tk services.

Author: austb

Diff for: ext/bin/config-puppetserver-test-ref

@@ -78,6 +78,23 @@ fi
  dep_ver="$(lein-pprint :version)"
  echo "$dep_ver" > "$top/ext/test-conf/puppetserver-dep")
 
-# Install Puppet Server Ruby dependencies using JRuby
-# Procedure defined in src-gems/puppetlabs/puppetdb/integration/install_gems.clj
-lein install-gems
+gem_list=()
+while read LINE
+do
+  gem_name=$(echo $LINE |awk '{print $1}')
+  gem_version=$(echo $LINE |awk '{print $2}')
+  gem_list+=("$gem_name:$gem_version")
+done < ./puppetserver/resources/ext/build-scripts/jruby-gem-list.txt
+
+gem-install() {
+  java -cp puppetserver/target/puppet-server-release.jar clojure.main \
+    -m puppetlabs.puppetserver.cli.gem \
+    --config ./test-resources/puppetserver/puppetserver.conf -- install --no-document "$@"
+}
+
+echo "Installing gems"
+gem-install facter
+gem-install hiera
+gem-install puppet-pson
+gem-install "${gem_list[@]}"
+echo "Finished install gems"

Diff for: project.clj

@@ -42,31 +42,6 @@
     [["releases" "https://artifactory.delivery.puppetlabs.net/artifactory/list/clojure-releases__local/"]
      ["snapshots" "https://artifactory.delivery.puppetlabs.net/artifactory/list/clojure-snapshots__local/"]]))
 
-;; See the integration tests section in documentation/CONTRIBUTING.md.
-(def puppetserver-test-dep-ver
-  (some-> (try
-            (slurp (str "ext/test-conf/puppetserver-dep"))
-            (catch java.io.FileNotFoundException ex
-              (binding [*out* *err*]
-                (println "puppetserver test dependency unconfigured (ignoring)"))
-              nil))
-          clojure.string/trim))
-
-(def puppetserver-test-dep-gem-list
-  (when puppetserver-test-dep-ver
-    (let [[major minor] (->> (re-matches #"^([0-9]+)\.([0-9]+)\..*" puppetserver-test-dep-ver)
-                             next
-                             (take 2)
-                             (map #(Integer/parseInt %)))]
-      (if (neg? (compare [major minor] [5 3]))
-        "gem-list.txt"
-        "jruby-gem-list.txt"))))
-
-(def puppetserver-test-deps
-  (when puppetserver-test-dep-ver
-    `[[puppetlabs/puppetserver ~puppetserver-test-dep-ver]
-      [puppetlabs/puppetserver ~puppetserver-test-dep-ver :classifier "test"]]))
-
 (def pdb-dev-deps
   (concat
    '[[ring/ring-mock]
@@ -82,8 +57,7 @@
      [org.yaml/snakeyaml]
 
      ;; Only needed for :integration tests
-     [puppetlabs/trapperkeeper-filesystem-watcher nil]]
-   puppetserver-test-deps))
+     [puppetlabs/trapperkeeper-filesystem-watcher nil]]))
 
 ;; Don't use lein :clean-targets so that we don't have to repeat
 ;; ourselves, given that we need to remove some protected files, and
@@ -319,9 +293,6 @@
                          ;; compile test files, and crashes because
                          ;; "src" namespaces aren't available.
                          :aot ^:replace []}
-             :install-gems {:source-paths ^:replace ["src-gems"]
-                            :target-path "target-gems"
-                            :dependencies ~puppetserver-test-deps}
              :ci {:plugins [[lein-pprint "1.1.1"]
                             [lein-exec "0.3.7"]]}
              ; We only want to include bouncycastle in the FOSS uberjar.
@@ -356,14 +327,7 @@
              ;; yet, disable it for now.
              :exclude-linters [:local-shadows-var]}
 
-  :aliases {"gem" ["with-profile" "install-gems,dev"
-                   "trampoline" "run" "-m" "puppetlabs.puppetserver.cli.gem"
-                   "--config" "./test-resources/puppetserver/puppetserver.conf"]
-            "install-gems" ["with-profile" "install-gems,dev"
-                            "trampoline" "run" "-m" "puppetlabs.puppetdb.integration.install-gems"
-                            ~puppetserver-test-dep-gem-list
-                            "--config" "./test-resources/puppetserver/puppetserver.conf"]
-            "kondo" ["with-profile" "+kondo" "run" "-m" "clj-kondo.main"]
+  :aliases {"kondo" ["with-profile" "+kondo" "run" "-m" "clj-kondo.main"]
             "clean" ~(pdb-run-clean pdb-clean-paths)
             "distclean" ~(pdb-run-clean pdb-distclean-paths)
             "time-shift-export" ^{:doc (clojure.string/join "" ["Shifts all timestamps from a PuppetDB archive with"

Diff for: src-gems/puppetlabs/puppetdb/integration/install_gems.clj

@@ -0,0 +1,21 @@
+puppetlabs.services.request-handler.request-handler-service/request-handler-service
+puppetlabs.services.jruby.jruby-puppet-service/jruby-puppet-pooled-service
+puppetlabs.services.jruby-pool-manager.jruby-pool-manager-service/jruby-pool-manager-service
+puppetlabs.services.puppet-profiler.puppet-profiler-service/puppet-profiler-service
+puppetlabs.trapperkeeper.services.webserver.jetty9-service/jetty9-service
+puppetlabs.trapperkeeper.services.webrouting.webrouting-service/webrouting-service
+puppetlabs.services.config.puppet-server-config-service/puppet-server-config-service
+puppetlabs.services.master.master-service/master-service
+puppetlabs.services.legacy-routes.legacy-routes-service/legacy-routes-service
+puppetlabs.services.puppet-admin.puppet-admin-service/puppet-admin-service
+puppetlabs.trapperkeeper.services.authorization.authorization-service/authorization-service
+puppetlabs.services.versioned-code-service.versioned-code-service/versioned-code-service
+puppetlabs.trapperkeeper.services.scheduler.scheduler-service/scheduler-service
+puppetlabs.trapperkeeper.services.status.status-service/status-service
+puppetlabs.trapperkeeper.services.metrics.metrics-service/metrics-service
+puppetlabs.services.jruby.jruby-metrics-service/jruby-metrics-service
+puppetlabs.trapperkeeper.services.watcher.filesystem-watch-service/filesystem-watch-service
+# To enable the CA service, leave the following line uncommented
+puppetlabs.services.ca.certificate-authority-service/certificate-authority-service
+# To disable the CA service, comment out the above line and uncomment the line below
+#puppetlabs.services.ca.certificate-authority-disabled-service/certificate-authority-disabled-service

Diff for: test-resources/puppetserver/bootstrap-7.x.cfg

@@ -2,7 +2,7 @@ puppetlabs.services.request-handler.request-handler-service/request-handler-serv
 puppetlabs.services.jruby.jruby-puppet-service/jruby-puppet-pooled-service
 puppetlabs.services.jruby-pool-manager.jruby-pool-manager-service/jruby-pool-manager-service
 puppetlabs.services.puppet-profiler.puppet-profiler-service/puppet-profiler-service
-puppetlabs.trapperkeeper.services.webserver.jetty9-service/jetty9-service
+puppetlabs.trapperkeeper.services.webserver.jetty10-service/jetty10-service
 puppetlabs.trapperkeeper.services.webrouting.webrouting-service/webrouting-service
 puppetlabs.services.config.puppet-server-config-service/puppet-server-config-service
 puppetlabs.services.master.master-service/master-service

Diff for: test-resources/puppetserver/bootstrap.cfg

@@ -20,7 +20,8 @@
             [puppetlabs.puppetdb.utils :as utils])
   (:import
    (com.typesafe.config ConfigValueFactory)
-   (java.lang ProcessBuilder ProcessBuilder$Redirect)))
+   (java.lang ProcessBuilder ProcessBuilder$Redirect)
+   (java.io FileNotFoundException)))
 
 (defprotocol TestServer
   (server-info [this]))
@@ -164,8 +165,14 @@
     (doto process .destroy .waitFor) ;; Assumes destroy sends a SIGTERM
     (doseq [f files-to-cleanup] (fs/delete f))))
 
+(defn pup-srv-version []
+  (try
+    (string/trim (slurp "./ext/test-conf/puppetserver-dep"))
+    (catch FileNotFoundException e
+      (log/error "No ext/test-conf/puppetserver-dep file found. Did you run ext/bin/config-puppetserver-test-ref ?")
+      (throw e))))
+
 (def dev-config-file "./test-resources/puppetserver/puppetserver.conf")
-(def dev-bootstrap-file "./test-resources/puppetserver/bootstrap.cfg")
 
 (defn mri-agent-dir []
   (-> (sh "bundle" "show" "puppet") :out clojure.string/trim))
@@ -245,6 +252,9 @@
                      (merge puppetserver-config-overrides)
                      (assoc-in [:webserver :ssl-port] port))
           config-file (ks/temp-file "puppetserver-conf" ".edn")
+          dev-bootstrap-file (if (string/starts-with? (pup-srv-version) "7.")
+                               "./test-resources/puppetserver/bootstrap-7.x.cfg"
+                               "./test-resources/puppetserver/bootstrap.cfg")
           cmd ["java" "-cp" "puppetserver/target/puppet-server-release.jar"
                "clojure.main" "-m" "puppetlabs.trapperkeeper.main"
                "services"