Merge pull request #312 from agrare/use_api_chunking

API Chunking Support

Author: cben

app/models/manageiq/providers/kubernetes/container_manager/refresher_mixin.rb

@@ -63,8 +63,18 @@ def parse_targeted_inventory(ems, target, inventory)
 
         def fetch_entities(client, entities)
           entities.each_with_object({}) do |entity, h|
+            continue = nil
+            h[entity[:name].singularize] ||= []
+
             begin
-              h[entity[:name].singularize] = client.send("get_#{entity[:name]}")
+              loop do
+                entities = client.send("get_#{entity[:name]}", :limit => refresher_options.chunk_size, :continue => continue)
+
+                h[entity[:name].singularize].concat(entities)
+                break if entities.last?
+
+                continue = entities.continue
+              end
             rescue KubeException => e
               raise e if entity[:default].nil?
               $log.warn("Unexpected Exception during refresh: #{e}")

config/settings.yml

@@ -59,6 +59,7 @@
   :kubernetes:
     :refresh_interval: 15.minutes
     :streaming_refresh: false
+    :chunk_size: 1_000
     :inventory_object_refresh: true
     :inventory_collections:
       :saver_strategy: batch

manageiq-providers-kubernetes.gemspec

@@ -15,7 +15,7 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency("hawkular-client",                 "~> 4.1")
   s.add_runtime_dependency("image-inspector-client",          "~> 2.0")
-  s.add_runtime_dependency("kubeclient",                      "~> 4.0")
+  s.add_runtime_dependency("kubeclient",                      "~> 4.1", ">= 4.1.1")
   s.add_runtime_dependency("prometheus-alert-buffer-client",  "~> 0.2.0")
   s.add_runtime_dependency("prometheus-api-client",           "~> 0.6")
   s.add_runtime_dependency("more_core_extensions",            "~> 3.6")

spec/models/manageiq/providers/kubernetes/container_manager/refresh_mixin_spec.rb

@@ -2,7 +2,8 @@
 
 describe ManageIQ::Providers::Kubernetes::ContainerManager::RefresherMixin do
   let(:client)  { double("client") }
-  let(:dummy) { (Class.new { include ManageIQ::Providers::Kubernetes::ContainerManager::RefresherMixin }).new }
+  let(:ems)     { FactoryBot.create(:ems_kubernetes) }
+  let(:dummy)   { ManageIQ::Providers::Kubernetes::ContainerManager::Refresher.new([ems]) }
 
   context 'when an exception is thrown' do
     before { allow(client).to receive(:get_pods) { raise KubeException.new(0, 'oh-no', nil) } }

spec/models/manageiq/providers/kubernetes/container_manager/refresher_spec.rb

@@ -28,7 +28,12 @@
   end
 
   def full_refresh_test(expected_extra_tags: [])
-    VCR.use_cassette(described_class.name.underscore) do # , :record => :new_episodes) do
+    # VCR by default matches on :method and the whole :uri
+    # In this case we are sending :limit in the :query section but we
+    # want to simulate an older kube API that doesn't respond to that
+    # param.  This can be done by having VCR ignore the :query component
+    # of the URI and return the legacy style responses.
+    VCR.use_cassette(described_class.name.underscore, :match_requests_on => [:method, :host, :path]) do # , :record => :new_episodes) do
       EmsRefresh.refresh(@ems)
     end
     @ems.reload
@@ -424,15 +429,15 @@ def tag_in_category_with_description(category, description)
       FactoryBot.create(:container_node, :name => "node", :ems_id => @ems.id)
     end
 
-    let(:container_volumes_count) { 21 }
+    let(:container_volumes_count) { 68 }
     let(:persintent_volumes_count) { 3 }
     let(:object_counts) do
       # using strings instead of actual model classes for compact rspec diffs
       {
         'ContainerNode'         => 2, # including the fake node
-        'ContainerGroup'        => 9,
-        'Container'             => 9,
-        'ContainerService'      => 12,
+        'ContainerGroup'        => 22,
+        'Container'             => 22,
+        'ContainerService'      => 16,
         'ContainerQuota'        => 9,
         'ContainerQuotaScope'   => 9,
         'ContainerQuotaItem'    => 30,
@@ -588,7 +593,7 @@ def assert_specific_persistent_volume_claim
         end
         # ContainerVolume records don't get archived themselves, but some belong to archived pods.
         expect(ContainerVolume.where(:type => 'ContainerVolume').count).to eq(container_volumes_count)
-        expect(@ems.container_volumes.count).to eq(container_volumes_count - 12)
+        expect(@ems.container_volumes.count).to eq(container_volumes_count - 18)
 
         container0 = Container.find_by(:name => "my-container", :container_group => pod0)
         container1 = Container.find_by(:name => "my-container", :container_group => pod1)