add support for reserved bucket tags

Signed-off-by: Utkarsh Srivastava <[email protected]>

fix put_bucket_tagging test - pass dummy objectsdk

Signed-off-by: Utkarsh Srivastava <[email protected]>

add support for events, race safe tag manipulation and --merge_tag flag

Signed-off-by: Utkarsh Srivastava <[email protected]>

fix lint issues

Signed-off-by: Utkarsh Srivastava <[email protected]>

clarify the reserved tags config comment

Signed-off-by: Utkarsh Srivastava <[email protected]>

add docs for the CLI changes

Signed-off-by: Utkarsh Srivastava <[email protected]>

address PR comments

Signed-off-by: Utkarsh Srivastava <[email protected]>

add account name to the bucket create event

Signed-off-by: Utkarsh Srivastava <[email protected]>

address PR comments

Signed-off-by: Utkarsh Srivastava <[email protected]>
(cherry picked from commit f400465)

Author: tangledbytes

config.js

@@ -951,6 +951,33 @@ config.NSFS_GLACIER_FORCE_EXPIRE_ON_GET = false;
 // interval
 config.NSFS_GLACIER_MIGRATE_LOG_THRESHOLD = 50 * 1024;
 
+/** 
+ * NSFS_GLACIER_RESERVED_BUCKET_TAGS defines an object of bucket tags which will be reserved
+ * by the system and PUT operations for them via S3 API would be limited - as in they would be
+ * mutable only if specified and only under certain conditions.
+ *
+ * @type {Record<string, {
+ *  schema: Record<any, any> & { $id: string },
+ *  immutable: true | false | 'if-data',
+ *  default: any,
+ *  event: boolean
+ * }>}
+ * 
+ * @example
+ * {
+    'deep-archive-copies': {
+        schema: {
+            $id: 'deep-archive-copies-schema-v0',
+            enum: ['1', '2']
+        }, // JSON Schema
+        immutable: 'if-data',
+        default: '1',
+        event: true
+    }
+ * }
+ */
+config.NSFS_GLACIER_RESERVED_BUCKET_TAGS = {};
+
 // anonymous account name
 config.ANONYMOUS_ACCOUNT_NAME = 'anonymous';
 

docs/NooBaaNonContainerized/Events.md

@@ -32,7 +32,10 @@ The following list includes events that indicate on a normal / successful operat
 - Description: NooBaa account was deleted successfully using NooBaa CLI.
 
 #### 4. `noobaa_bucket_created`
-- Arguments: `bucket_name`
+- Arguments:
+    - `bucket_name`
+    - `account_name`
+    - `<tag_value>` (if `event` is `true` for the reserved tag)
 - Description: NooBaa bucket was created successfully using NooBaa CLI or S3.
 
 #### 5. `noobaa_bucket_deleted`
@@ -43,6 +46,11 @@ The following list includes events that indicate on a normal / successful operat
 - Arguments: `whitelist_ips`
 - Description: Whitelist Server IPs updated successfully using NooBaa CLI.  
 
+#### 7. `noobaa_bucket_reserved_tag_modified`
+- Arguments:
+    - `bucket_name`
+    - `<tag_value>` (if `event` is `true` for the reserved tag)
+- Description: NooBaa bucket reserved tag was modified successfully using NooBaa CLI or S3.
 
 ### Error Indicating Events
 
@@ -219,4 +227,4 @@ The following list includes events that indicate on some sort of malfunction or
 - Reasons:
     - Free space in notification log dir FS is below threshold.
 - Resolutions:
-    - Free up space is FS.
+    - Free up space is FS.

docs/NooBaaNonContainerized/NooBaaCLI.md

@@ -376,6 +376,13 @@ noobaa-cli bucket update --name <bucket_name> [--new_name] [--owner]
     - Type: Boolean
     - Description: Set the bucket to force md5 ETag calculation.
 
+- `tag`
+    - Type: String
+    - Description: Set the bucket tags, type is a string of valid JSON. Behaviour is similar to `put-bucket-tagging` S3 API.
+
+- `merge_tag`
+    - Type: String
+    - Description: Merge the bucket tags with previous bucket tags, type is a string of valid JSON.
 
 ### Bucket Status
 

src/cmd/manage_nsfs.js

@@ -40,6 +40,8 @@ const { throw_cli_error, get_bucket_owner_account_by_name,
 const manage_nsfs_validations = require('../manage_nsfs/manage_nsfs_validations');
 const nc_mkm = require('../manage_nsfs/nc_master_key_manager').get_instance();
 const notifications_util = require('../util/notifications_util');
+const BucketSpaceFS = require('../sdk/bucketspace_fs');
+const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEvent;
 
 let config_fs;
 
@@ -123,7 +125,6 @@ async function fetch_bucket_data(action, user_input) {
         force_md5_etag: user_input.force_md5_etag === undefined || user_input.force_md5_etag === '' ? user_input.force_md5_etag : get_boolean_or_string_value(user_input.force_md5_etag),
         notifications: user_input.notifications
     };
-
     if (user_input.bucket_policy !== undefined) {
         if (typeof user_input.bucket_policy === 'string') {
             // bucket_policy deletion specified with empty string ''
@@ -142,6 +143,27 @@ async function fetch_bucket_data(action, user_input) {
         data = await merge_new_and_existing_config_data(data);
     }
 
+    if ((action === ACTIONS.UPDATE && user_input.tag) || (action === ACTIONS.ADD)) {
+        const tags = JSON.parse(user_input.tag || '[]');
+        data.tag = BucketSpaceFS._merge_reserved_tags(
+            data.tag || BucketSpaceFS._default_bucket_tags(),
+            tags,
+            action === ACTIONS.ADD ? true : await _is_bucket_empty(data),
+        );
+    }
+
+    if ((action === ACTIONS.UPDATE && user_input.merge_tag) || (action === ACTIONS.ADD)) {
+        const merge_tags = JSON.parse(user_input.merge_tag || '[]');
+        data.tag = _.merge(
+            data.tag,
+            BucketSpaceFS._merge_reserved_tags(
+                data.tag || BucketSpaceFS._default_bucket_tags(),
+                merge_tags,
+                action === ACTIONS.ADD ? true : await _is_bucket_empty(data),
+            )
+        );
+    }
+
     //if we're updating the owner, needs to override owner in file with the owner from user input.
     //if we're adding a bucket, need to set its owner id field
     if ((action === ACTIONS.UPDATE && user_input.owner) || (action === ACTIONS.ADD)) {
@@ -189,7 +211,14 @@ async function add_bucket(data) {
     data._id = mongo_utils.mongoObjectId();
     const parsed_bucket_data = await config_fs.create_bucket_config_file(data);
     await set_bucker_owner(parsed_bucket_data);
-    return { code: ManageCLIResponse.BucketCreated, detail: parsed_bucket_data, event_arg: { bucket: data.name }};
+
+    const [reserved_tag_event_args] = BucketSpaceFS._generate_reserved_tag_event_args({}, data.tag);
+
+    return {
+        code: ManageCLIResponse.BucketCreated,
+        detail: parsed_bucket_data,
+        event_arg: { ...(reserved_tag_event_args || {}), bucket: data.name, account: parsed_bucket_data.bucket_owner },
+    };
 }
 
 /**
@@ -245,25 +274,14 @@ async function update_bucket(data) {
  */
 async function delete_bucket(data, force) {
     try {
-        const temp_dir_name = native_fs_utils.get_bucket_tmpdir_name(data._id);
+        const bucket_empty = await _is_bucket_empty(data);
+        if (!bucket_empty && !force) {
+            throw_cli_error(ManageCLIError.BucketDeleteForbiddenHasObjects, data.name);
+        }
+
         const bucket_temp_dir_path = native_fs_utils.get_bucket_tmpdir_full_path(data.path, data._id);
-        // fs_contexts for bucket temp dir (storage path)
         const fs_context_fs_backend = native_fs_utils.get_process_fs_context(data.fs_backend);
-        let entries;
-        try {
-            entries = await nb_native().fs.readdir(fs_context_fs_backend, data.path);
-        } catch (err) {
-            dbg.warn(`delete_bucket: bucket name ${data.name},` +
-                `got an error on readdir with path: ${data.path}`, err);
-            // if the bucket's path was deleted first (encounter ENOENT error) - continue deletion
-            if (err.code !== 'ENOENT') throw err;
-        }
-        if (entries) {
-            const object_entries = entries.filter(element => !element.name.endsWith(temp_dir_name));
-            if (object_entries.length > 0 && !force) {
-                throw_cli_error(ManageCLIError.BucketDeleteForbiddenHasObjects, data.name);
-            }
-        }
+
         await native_fs_utils.folder_delete(bucket_temp_dir_path, fs_context_fs_backend, true);
         await config_fs.delete_bucket_config_file(data.name);
         return { code: ManageCLIResponse.BucketDeleted, detail: { name: data.name }, event_arg: { bucket: data.name } };
@@ -273,6 +291,33 @@ async function delete_bucket(data, force) {
     }
 }
 
+/**
+ * _is_bucket_empty returns true if the given bucket is empty
+ *
+ * @param {*} data 
+ * @returns {Promise<boolean>}
+ */
+async function _is_bucket_empty(data) {
+    const temp_dir_name = native_fs_utils.get_bucket_tmpdir_name(data._id);
+    // fs_contexts for bucket temp dir (storage path)
+    const fs_context_fs_backend = native_fs_utils.get_process_fs_context(data.fs_backend);
+    let entries;
+    try {
+        entries = await nb_native().fs.readdir(fs_context_fs_backend, data.path);
+    } catch (err) {
+        dbg.warn(`_is_bucket_empty: bucket name ${data.name},` +
+            `got an error on readdir with path: ${data.path}`, err);
+        // if the bucket's path was deleted first (encounter ENOENT error) - continue deletion
+        if (err.code !== 'ENOENT') throw err;
+    }
+    if (entries) {
+        const object_entries = entries.filter(element => !element.name.endsWith(temp_dir_name));
+        return object_entries.length === 0;
+    }
+
+    return true;
+}
+
 /**
  * bucket_management does the following - 
  * 1. fetches the bucket data if this is not a list operation
@@ -294,7 +339,24 @@ async function bucket_management(action, user_input) {
     } else if (action === ACTIONS.STATUS) {
         response = await get_bucket_status(data);
     } else if (action === ACTIONS.UPDATE) {
-        response = await update_bucket(data);
+        const bucket_path = config_fs.get_bucket_path_by_name(user_input.name);
+        const bucket_lock_file = `${bucket_path}.lock`;
+        await native_fs_utils.lock_and_run(config_fs.fs_context, bucket_lock_file, async () => {
+            const prev_bucket_info = await fetch_bucket_data(action, _.omit(user_input, ['tag', 'merge_tag']));
+            const bucket_info = await fetch_bucket_data(action, user_input);
+
+            const tagging_object = BucketSpaceFS._objectify_tagging_arr(prev_bucket_info.tag);
+            const [
+                reserved_tag_event_args,
+                reserved_tag_modified,
+            ] = BucketSpaceFS._generate_reserved_tag_event_args(tagging_object, bucket_info.tag);
+
+            response = await update_bucket(bucket_info);
+            if (reserved_tag_modified) {
+                new NoobaaEvent(NoobaaEvent.BUCKET_RESERVED_TAG_MODIFIED)
+                    .create_event(undefined, { ...reserved_tag_event_args, bucket_name: user_input.name });
+            }
+        });
     } else if (action === ACTIONS.DELETE) {
         const force = get_boolean_or_string_value(user_input.force);
         response = await delete_bucket(data, force);

src/manage_nsfs/manage_nsfs_constants.js

@@ -62,7 +62,7 @@ const VALID_OPTIONS_ANONYMOUS_ACCOUNT = {
 
 const VALID_OPTIONS_BUCKET = {
     'add': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'force_md5_etag', 'notifications', FROM_FILE, ...CLI_MUTUAL_OPTIONS]),
-    'update': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'new_name', 'force_md5_etag', 'notifications', ...CLI_MUTUAL_OPTIONS]),
+    'update': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'new_name', 'force_md5_etag', 'notifications', 'tag', 'merge_tag', ...CLI_MUTUAL_OPTIONS]),
     'delete': new Set(['name', 'force', ...CLI_MUTUAL_OPTIONS]),
     'list': new Set(['wide', 'name', ...CLI_MUTUAL_OPTIONS]),
     'status': new Set(['name', ...CLI_MUTUAL_OPTIONS]),
@@ -171,6 +171,9 @@ const OPTION_TYPE = {
     key: 'string',
     value: 'string',
     remove_key: 'boolean',
+    // bucket tagging
+    tag: 'string',
+    merge_tag: 'string',
 };
 
 const BOOLEAN_STRING_VALUES = ['true', 'false'];

src/manage_nsfs/manage_nsfs_events_utils.js

@@ -302,6 +302,16 @@ NoobaaEvent.UNAUTHORIZED = Object.freeze({
     severity: 'ERROR',
     state: 'HEALTHY',
 });
+NoobaaEvent.BUCKET_RESERVED_TAG_MODIFIED = Object.freeze({
+    event_code: 'noobaa_bucket_reserved_tag_modified',
+    message: 'Bucket reserved tag modified',
+    description: 'Noobaa bucket reserved tag modified',
+    entity_type: 'NODE',
+    event_type: 'INFO',
+    scope: 'NODE',
+    severity: 'INFO',
+    state: 'HEALTHY',
+});
 
 NoobaaEvent.IO_STREAM_ITEM_TIMEOUT = Object.freeze({
     event_code: 'bucket_io_stream_item_timeout',