|
| 1 | +# frozen_string_literal: true |
| 2 | + |
| 3 | +module Import |
| 4 | + module PlaceholderReferences |
| 5 | + class LoadService < BaseService |
| 6 | + BATCH_LIMIT = 500 |
| 7 | + |
| 8 | + def initialize(import_source:, import_uid:) |
| 9 | + super(import_source: import_source, import_uid: import_uid) |
| 10 | + |
| 11 | + @processed_count = 0 |
| 12 | + @error_count = 0 |
| 13 | + end |
| 14 | + |
| 15 | + def execute |
| 16 | + log_info(message: 'Processing placeholder references') |
| 17 | + |
| 18 | + while (batch = next_batch).present? |
| 19 | + load!(batch) |
| 20 | + |
| 21 | + # End this loop if we know that we cleared the set earlier. |
| 22 | + # This prevents processing just a few records at a time if an import is simultaneously writing data to Redis. |
| 23 | + break if batch.size < BATCH_LIMIT |
| 24 | + end |
| 25 | + |
| 26 | + log_info( |
| 27 | + message: 'Processed placeholder references', |
| 28 | + processed_count: processed_count, |
| 29 | + error_count: error_count |
| 30 | + ) |
| 31 | + |
| 32 | + success(processed_count: processed_count, error_count: error_count) |
| 33 | + end |
| 34 | + |
| 35 | + private |
| 36 | + |
| 37 | + attr_accessor :error_count, :processed_count |
| 38 | + |
| 39 | + def next_batch |
| 40 | + cache.limited_values_from_set(cache_key, limit: BATCH_LIMIT) |
| 41 | + end |
| 42 | + |
| 43 | + def load!(batch) |
| 44 | + to_load = batch.filter_map do |item| |
| 45 | + SourceUserPlaceholderReference.from_serialized(item) |
| 46 | + rescue JSON::ParserError, SourceUserPlaceholderReference::SerializationError => e |
| 47 | + log_error(item, e) |
| 48 | + nil |
| 49 | + end |
| 50 | + |
| 51 | + begin |
| 52 | + bulk_insert!(to_load) |
| 53 | + rescue ActiveRecord::RecordInvalid => e |
| 54 | + # We optimise for all records being valid and only filter for validity |
| 55 | + # when there was a problem |
| 56 | + to_load.reject! do |item| |
| 57 | + next false if item.valid? |
| 58 | + |
| 59 | + log_error(item.attributes, e) |
| 60 | + true |
| 61 | + end |
| 62 | + |
| 63 | + # Try again |
| 64 | + bulk_insert!(to_load) |
| 65 | + rescue ActiveRecord::InvalidForeignKey => e |
| 66 | + # This is an unrecoverable situation where we allow the error to clear the batch |
| 67 | + log_error(to_load, e) |
| 68 | + end |
| 69 | + |
| 70 | + clear_batch!(batch) |
| 71 | + end |
| 72 | + |
| 73 | + def bulk_insert!(to_load) |
| 74 | + Import::SourceUserPlaceholderReference.bulk_insert!(to_load) |
| 75 | + end |
| 76 | + |
| 77 | + def clear_batch!(batch) |
| 78 | + processed_count = batch.size |
| 79 | + |
| 80 | + self.processed_count += processed_count |
| 81 | + |
| 82 | + cache.set_remove(cache_key, batch) |
| 83 | + end |
| 84 | + |
| 85 | + def log_error(item, exception) |
| 86 | + super( |
| 87 | + message: 'Error processing placeholder reference', |
| 88 | + item: item, |
| 89 | + exception: { |
| 90 | + class: exception.class, |
| 91 | + message: exception.message |
| 92 | + } |
| 93 | + ) |
| 94 | + |
| 95 | + self.error_count += 1 |
| 96 | + end |
| 97 | + end |
| 98 | + end |
| 99 | +end |
0 commit comments