NC | lifecycle | continue last run

nadavMiz · nadavMiz · commit d04893f74604 · 2025-04-02T12:25:20.000+03:00
Signed-off-by: nadav mizrahi &lt;nadav.mizrahi16@gmail.com&gt;
diff --git a/src/cmd/manage_nsfs.js b/src/cmd/manage_nsfs.js
@@ -889,8 +889,9 @@ async function lifecycle_management(args) {
     const disable_service_validation = get_boolean_or_string_value(args.disable_service_validation);
     const disable_runtime_validation = get_boolean_or_string_value(args.disable_runtime_validation);
     const short_status = get_boolean_or_string_value(args.short_status);
+    const should_continue_last_run = get_boolean_or_string_value(args.continue);
     try {
-        const options = { disable_service_validation, disable_runtime_validation, short_status };
+        const options = { disable_service_validation, disable_runtime_validation, short_status, should_continue_last_run };
         const { should_run, lifecycle_run_status } = await noobaa_cli_lifecycle.run_lifecycle_under_lock(config_fs, options);
         if (should_run) {
             write_stdout_response(ManageCLIResponse.LifecycleSuccessful, lifecycle_run_status);
diff --git a/src/manage_nsfs/health.js b/src/manage_nsfs/health.js
@@ -3,7 +3,6 @@
 
 const dbg = require('../util/debug_module')(__filename);
 const _ = require('lodash');
-const path = require('path');
 const P = require('../util/promise');
 const config = require('../../config');
 const os_util = require('../util/os_utils');
@@ -18,6 +17,7 @@ const { get_boolean_or_string_value, throw_cli_error, write_stdout_response,
 const { ManageCLIResponse } = require('./manage_nsfs_cli_responses');
 const ManageCLIError = require('./manage_nsfs_cli_errors').ManageCLIError;
 const notifications_util = require('../util/notifications_util');
+const lifecycle_utils = require('../util/lifecycle_utils');
 
 
 const HOSTNAME = 'localhost';
@@ -462,7 +462,9 @@ class NSFSHealth {
      * @returns {Promise<object>}
      */
     async get_lifecycle_health_status() {
-        const latest_lifecycle_run_status = await this.get_latest_lifecycle_run_status({ silent_if_missing: true });
+        const latest_lifecycle_run_status = await lifecycle_utils.get_latest_nc_lifecycle_run_status(
+            this.config_fs,
+            { silent_if_missing: true });
         if (!latest_lifecycle_run_status) return {};
         return {
             total_stats: latest_lifecycle_run_status.total_stats,
@@ -471,29 +473,6 @@ class NSFSHealth {
         };
     }
 
-
-     /**
-     * get_latest_lifecycle_run_status returns the latest lifecycle run status
-     * latest run can be found by maxing the lifecycle log entry names, log entry name is the lifecycle_run_{timestamp}.json of the run
-     * @params {{silent_if_missing: boolean}} options
-     * @returns {Promise<object | undefined >}
-     */
-    async get_latest_lifecycle_run_status(options) {
-        const { silent_if_missing = false } = options;
-        try {
-            const lifecycle_log_entries = await nb_native().fs.readdir(this.config_fs.fs_context, config.NC_LIFECYCLE_LOGS_DIR);
-            const latest_lifecycle_run = _.maxBy(lifecycle_log_entries, entry => entry.name);
-            const latest_lifecycle_run_status_path = path.join(config.NC_LIFECYCLE_LOGS_DIR, latest_lifecycle_run.name);
-            const latest_lifecycle_run_status = await this.config_fs.get_config_data(latest_lifecycle_run_status_path, options);
-            return latest_lifecycle_run_status;
-        } catch (err) {
-            if (err.code === 'ENOENT' && silent_if_missing) {
-                return;
-            }
-            throw err;
-        }
-    }
-
     /**
      * get_config_file_data_or_error_object return an object containing config_data or err_obj if error occurred
      * @param {string} type
diff --git a/src/manage_nsfs/manage_nsfs_constants.js b/src/manage_nsfs/manage_nsfs_constants.js
@@ -96,7 +96,7 @@ const VALID_OPTIONS_CONNECTION = {
     'status': new Set(['name', 'decrypt', ...CLI_MUTUAL_OPTIONS]),
 };
 
-const VALID_OPTIONS_LIFECYCLE = new Set(['disable_service_validation', 'disable_runtime_validation', 'short_status', ...CLI_MUTUAL_OPTIONS]);
+const VALID_OPTIONS_LIFECYCLE = new Set(['disable_service_validation', 'disable_runtime_validation', 'short_status', 'continue', ...CLI_MUTUAL_OPTIONS]);
 
 const VALID_OPTIONS_WHITELIST = new Set(['ips', ...CLI_MUTUAL_OPTIONS]);
 
@@ -159,7 +159,8 @@ const OPTION_TYPE = {
     // lifecycle options
     disable_service_validation: 'boolean',
     disable_runtime_validation: 'boolean',
-    short: 'boolean',
+    short_status: 'boolean',
+    continue: 'boolean',
     //connection
     notification_protocol: 'string',
     agent_request_object: 'string',
@@ -175,7 +176,7 @@ const OPTION_TYPE = {
 const BOOLEAN_STRING_VALUES = ['true', 'false'];
 const BOOLEAN_STRING_OPTIONS = new Set(['allow_bucket_creation', 'regenerate', 'wide', 'show_secrets', 'force',
     'force_md5_etag', 'iam_operate_on_root_account', 'all_account_details', 'all_bucket_details', 'anonymous',
-    'disable_service_validation', 'disable_runtime_validation', 'short_status', 'skip_verification']);
+    'disable_service_validation', 'disable_runtime_validation', 'short_status', 'skip_verification', 'continue']);
 
 // CLI UNSET VALUES
 const CLI_EMPTY_STRING = '';
diff --git a/src/manage_nsfs/nc_lifecycle.js b/src/manage_nsfs/nc_lifecycle.js
@@ -18,6 +18,7 @@ const { throw_cli_error, get_service_status, NOOBAA_SERVICE_NAME,
     is_desired_time, record_current_time } = require('./manage_nsfs_cli_utils');
 const SensitiveString = require('../util/sensitive_string');
 const {CONFIG_TYPES} = require('../sdk/config_fs');
+const lifecycle_utils = require('../util/lifecycle_utils');
 
 // TODO:
 // implement
@@ -53,11 +54,12 @@ const TIMED_OPS = Object.freeze({
  * run_lifecycle_under_lock runs the lifecycle workflow under a file system lock
  * lifecycle workflow is being locked to prevent multiple instances from running the lifecycle workflow
  * @param {import('../sdk/config_fs').ConfigFS} config_fs
- * @param {{disable_service_validation?: boolean, disable_runtime_validation?: boolean, short_status?: boolean}} flags
+ * @param {{disable_service_validation?: boolean, disable_runtime_validation?: boolean, short_status?: boolean, should_continue_last_run?: boolean}} flags
  * @returns {Promise<{should_run: Boolean, lifecycle_run_status: Object}>}
  */
 async function run_lifecycle_under_lock(config_fs, flags) {
-    const { disable_service_validation = false, disable_runtime_validation = false, short_status = false } = flags;
+    const { disable_service_validation = false, disable_runtime_validation = false, short_status = false,
+        should_continue_last_run = false } = flags;
     return_short_status = short_status;
     const fs_context = config_fs.fs_context;
     const lifecyle_logs_dir_path = config.NC_LIFECYCLE_LOGS_DIR;
@@ -76,7 +78,7 @@ async function run_lifecycle_under_lock(config_fs, flags) {
         try {
             dbg.log0('run_lifecycle_under_lock acquired lock - start lifecycle');
             new NoobaaEvent(NoobaaEvent.LIFECYCLE_STARTED).create_event();
-            await run_lifecycle_or_timeout(config_fs, disable_service_validation);
+            await run_lifecycle_or_timeout(config_fs, disable_service_validation, should_continue_last_run);
         } catch (err) {
             dbg.error('run_lifecycle_under_lock failed with error', err, err.code, err.message);
             throw err;
@@ -96,13 +98,13 @@ async function run_lifecycle_under_lock(config_fs, flags) {
  * @param {boolean} disable_service_validation
  * @returns {Promise<Void>}
  */
-async function run_lifecycle_or_timeout(config_fs, disable_service_validation) {
+async function run_lifecycle_or_timeout(config_fs, disable_service_validation, should_continue_last_run) {
     await _call_op_and_update_status({
         op_name: TIMED_OPS.RUN_LIFECYLE,
         op_func: async () => {
             await P.timeout(
                 config.NC_LIFECYCLE_TIMEOUT_MS,
-                run_lifecycle(config_fs, disable_service_validation),
+                run_lifecycle(config_fs, disable_service_validation, should_continue_last_run),
                 () => ManageCLIError.LifecycleWorkerReachedTimeout
             );
         }
@@ -115,7 +117,7 @@ async function run_lifecycle_or_timeout(config_fs, disable_service_validation) {
  * @param {boolean} disable_service_validation
  * @returns {Promise<Void>}
  */
-async function run_lifecycle(config_fs, disable_service_validation) {
+async function run_lifecycle(config_fs, disable_service_validation, should_continue_last_run) {
     const system_json = await config_fs.get_system_config_file(config_fs_options);
     if (!disable_service_validation) await throw_if_noobaa_not_active(config_fs, system_json);
 
@@ -124,6 +126,10 @@ async function run_lifecycle(config_fs, disable_service_validation) {
         op_func: async () => config_fs.list_buckets()
     });
 
+    if (should_continue_last_run) {
+        await load_previous_run_state(bucket_names, config_fs);
+    }
+
     await _call_op_and_update_status({
         op_name: TIMED_OPS.PROCESS_BUCKETS,
         op_func: async () => process_buckets(config_fs, bucket_names, system_json)
@@ -735,6 +741,32 @@ async function _call_op_and_update_status({ bucket_name = undefined, rule_id = u
     }
 }
 
+/**
+ * init the bucket status object statuses if they don't exist
+ * @param {string} bucket_name
+ * @returns {Object} created or existing bucket status
+ */
+function init_bucket_status(bucket_name) {
+    lifecycle_run_status.buckets_statuses[bucket_name] ??= {};
+    lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses ??= {};
+    lifecycle_run_status.buckets_statuses[bucket_name].bucket_process_times = {};
+    lifecycle_run_status.buckets_statuses[bucket_name].bucket_stats = {};
+    return lifecycle_run_status.buckets_statuses[bucket_name];
+}
+
+/**
+ * init the rule status object statuses if they don't exist
+ * @param {string} bucket_name
+ * @param {string} rule_id
+ * @returns {Object} created or existing rule status
+ */
+function init_rule_status(bucket_name, rule_id) {
+    lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id] ??= {};
+    lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id].rule_process_times = {};
+    lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id].rule_stats = {};
+    return lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id];
+}
+
 /**
  * update_status updates rule/bucket/global based on the given parameters
  * 1. initalize statuses/times/stats per level
@@ -755,9 +787,9 @@ function update_status({ bucket_name, rule_id, op_name, op_times, reply = [], er
     // TODO - check errors
     if (op_times.start_time) {
         if (op_name === TIMED_OPS.PROCESS_RULE) {
-            lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id] = { rule_process_times: {}, rule_stats: {} };
+            init_rule_status(bucket_name, rule_id);
         } else if (op_name === TIMED_OPS.PROCESS_BUCKET) {
-            lifecycle_run_status.buckets_statuses[bucket_name] = { bucket_process_times: {}, bucket_stats: {}, rules_statuses: {} };
+            init_bucket_status(bucket_name);
         }
     }
     _update_times_on_status({ op_name, op_times, bucket_name, rule_id });
@@ -882,6 +914,26 @@ async function write_lifecycle_log_file(fs_context, lifecyle_logs_dir_path) {
     );
 }
 
+async function load_previous_run_state(buckets, config_fs) {
+    const previous_run = await lifecycle_utils.get_latest_nc_lifecycle_run_status(config_fs, { silent_if_missing: true });
+    if (previous_run) {
+        lifecycle_run_status.state = previous_run.state;
+        for (const [bucket_name, prev_bucket_status] of Object.entries(previous_run.buckets_statuses)) {
+            if (!buckets.includes(bucket_name)) continue;
+            const bucket_json = await config_fs.get_bucket_by_name(bucket_name, config_fs_options);
+            if (!bucket_json.lifecycle_configuration_rules) continue;
+            const bucket_status = init_bucket_status(bucket_name);
+            bucket_status.state = prev_bucket_status.state;
+            const bucket_rules = bucket_json.lifecycle_configuration_rules.map(rule => rule.id);
+            for (const [rule_id, prev_rule_status] of Object.entries(prev_bucket_status.rules_statuses)) {
+                if (!bucket_rules.includes(rule_id)) return;
+                const rule_status = init_rule_status(bucket_name, rule_id);
+                rule_status.state = prev_rule_status.state;
+            }
+        }
+    }
+}
+
 // EXPORTS
 exports.run_lifecycle_under_lock = run_lifecycle_under_lock;
 
diff --git a/src/test/unit_tests/jest_tests/test_nc_lifecycle_cli.test.js b/src/test/unit_tests/jest_tests/test_nc_lifecycle_cli.test.js
@@ -481,13 +481,14 @@ describe('noobaa cli - lifecycle batching', () => {
         object_sdk = new NsfsObjectSDK('', config_fs, json_account, "DISABLED", config_fs.config_root, undefined);
         object_sdk.requesting_account = json_account;
         await object_sdk.create_bucket({ name: test_bucket });
-        config.NC_LIFECYCLE_LOGS_DIR = tmp_lifecycle_logs_dir_path;
-        config.NC_LIFECYCLE_LIST_BATCH_SIZE = 2;
-        config.NC_LIFECYCLE_BUCKET_BATCH_SIZE = 5;
     });
 
     beforeEach(async () => {
-        await config_fs.create_config_json_file(JSON.stringify({ NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path }));
+        await config_fs.create_config_json_file(JSON.stringify({
+            NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path,
+            NC_LIFECYCLE_LIST_BATCH_SIZE: 2,
+            NC_LIFECYCLE_BUCKET_BATCH_SIZE: 5,
+        }));
     });
 
     afterEach(async () => {
@@ -583,7 +584,9 @@ describe('noobaa cli - lifecycle batching', () => {
 
         await config_fs.update_config_json_file(JSON.stringify({
             NC_LIFECYCLE_TIMEOUT_MS: 1,
-            NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path
+            NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path,
+            NC_LIFECYCLE_BUCKET_BATCH_SIZE: 5,
+            NC_LIFECYCLE_LIST_BATCH_SIZE: 2
         }));
         await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, true, undefined);
 
@@ -595,6 +598,91 @@ describe('noobaa cli - lifecycle batching', () => {
         const object_list = await object_sdk.list_objects({bucket: test_bucket});
         expect(object_list.objects.length).not.toBe(0);
     });
+
+    it("lifecycle batching - continue finished lifecycle should do nothing", async () => {
+        await bucketspace_fs.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all });
+
+        await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, true, undefined);
+        await create_object(object_sdk, test_bucket, test_key1, 100, true);
+        await create_object(object_sdk, test_bucket, test_key2, 100, true);
+
+        //continue finished run
+        await exec_manage_cli(TYPES.LIFECYCLE, '', { continue: 'true', disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, true, undefined);
+
+        const lifecycle_log_entries = await nb_native().fs.readdir(config_fs.fs_context, tmp_lifecycle_logs_dir_path);
+        expect(lifecycle_log_entries.length).toBe(2);
+        const log_file_path = path.join(tmp_lifecycle_logs_dir_path, lifecycle_log_entries[0].name);
+        const lifecycle_log_json = await config_fs.get_config_data(log_file_path, {silent_if_missing: true});
+        expect(lifecycle_log_json.state.is_finished).toBe(true);
+        const object_list = await object_sdk.list_objects({bucket: test_bucket});
+        expect(object_list.objects.length).toBe(2);
+    });
+
+    it("continue lifecycle batching should finish the run - delete all", async () => {
+        await bucketspace_fs.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all });
+        await config_fs.update_config_json_file(JSON.stringify({
+            NC_LIFECYCLE_TIMEOUT_MS: 60,
+            NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path,
+            NC_LIFECYCLE_BUCKET_BATCH_SIZE: 5,
+            NC_LIFECYCLE_LIST_BATCH_SIZE: 2
+
+        }));
+        const keys = [test_key1, test_key2, "key3", "key4", "key5", "key6", "key7"];
+        for (const key of keys) {
+            await create_object(object_sdk, test_bucket, key, 100, false);
+        }
+        try {
+            await exec_manage_cli(TYPES.LIFECYCLE, '', {disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined);
+        } catch (e) {
+            //ignore error
+        }
+        await exec_manage_cli(TYPES.LIFECYCLE, '', {continue: 'true', disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined);
+        const object_list2 = await object_sdk.list_objects({bucket: test_bucket});
+        expect(object_list2.objects.length).toBe(0);
+    });
+
+    it("continue lifecycle batching should finish the run - validate new run. don't delete already deleted items", async () => {
+        await bucketspace_fs.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all });
+        const bucket_batch_size = 6;
+        await config_fs.update_config_json_file(JSON.stringify({
+            NC_LIFECYCLE_TIMEOUT_MS: 70,
+            NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path,
+            NC_LIFECYCLE_BUCKET_BATCH_SIZE: bucket_batch_size,
+            NC_LIFECYCLE_LIST_BATCH_SIZE: 2
+        }));
+        const keys = [];
+        for (let i = 0; i < 100; i++) {
+            const new_key = `key${i}`;
+            await create_object(object_sdk, test_bucket, new_key, 100, false);
+            keys.push(new_key);
+        }
+        try {
+            await exec_manage_cli(TYPES.LIFECYCLE, '', {disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined);
+        } catch (e) {
+            //ignore error
+        }
+
+        const object_list = await object_sdk.list_objects({bucket: test_bucket});
+        const intermediate_key_list = object_list.objects.map(object => object.key);
+        //recreate deleted key. next run should skip those keys
+        for (const key of keys) {
+            if (!intermediate_key_list.includes(key)) {
+                await create_object(object_sdk, test_bucket, key, 100, false);
+            }
+        }
+        await config_fs.update_config_json_file(JSON.stringify({
+            NC_LIFECYCLE_TIMEOUT_MS: 9999,
+            NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path,
+            NC_LIFECYCLE_BUCKET_BATCH_SIZE: bucket_batch_size,
+            NC_LIFECYCLE_LIST_BATCH_SIZE: 2,
+        }));
+        await exec_manage_cli(TYPES.LIFECYCLE, '', {continue: 'true', disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined);
+        const object_list2 = await object_sdk.list_objects({bucket: test_bucket});
+        const expected_size = keys.length - intermediate_key_list.length;
+        //process can be stopped while deleting. check that size is in range of batch size
+        expect(object_list2.objects.length).toBeGreaterThanOrEqual(expected_size);
+        expect(object_list2.objects.length).toBeLessThanOrEqual(expected_size + bucket_batch_size);
+    });
 });
 
 describe('noobaa cli - lifecycle notifications', () => {
diff --git a/src/util/lifecycle_utils.js b/src/util/lifecycle_utils.js
