3.2.0 release

Author: gys-git

_cmd.py

@@ -28,7 +28,9 @@
 from ruamel.yaml.comments import CommentedMap
 
 import _errno as err
-from tool import ConfigUtil, FileUtil, YamlLoader, OrderedDict, COMMAND_ENV
+import const
+from tool import ConfigUtil, FileUtil, YamlLoader, OrderedDict, COMMAND_ENV, aes_encrypt, aes_decrypt, \
+    string_to_md5_32bytes
 from _manager import Manager
 from _stdio import SafeStdio
 from _environ import ENV_BASE_DIR
@@ -1051,13 +1053,15 @@ def __str__(self):
 
 class DeployConfig(SafeStdio):
 
-    def __init__(self, yaml_path, yaml_loader=yaml, inner_config=None, config_parser_manager=None, stdio=None):
+    def __init__(self, yaml_path, yaml_loader=yaml, inner_config=None, config_parser_manager=None, config_encrypted=None, stdio=None):
         self._user = None
         self.unuse_lib_repository = False
         self.auto_create_tenant = False
         self._inner_config = inner_config
         self.components = OrderedDict()
         self._src_data = None
+        self._need_encrypt_dump = False
+        self._config_encrypted = config_encrypted
         self.name = os.path.split(os.path.split(yaml_path)[0])[-1]
         self.yaml_path = yaml_path
         self.yaml_loader = yaml_loader
@@ -1072,6 +1076,8 @@ def __init__(self, yaml_path, yaml_loader=yaml, inner_config=None, config_parser
         self._removed_components = set()
         self._do_not_dump = False
         self._mem_mode = False
+        self._tmp_user_password = None
+        self._auto_decrypt_password()
 
     def __deepcopy__(self, memo):
         deploy_config = self.__class__(self.yaml_path, self.yaml_loader, self._inner_config, self.config_parser_manager, self.stdio)
@@ -1091,6 +1097,15 @@ def enable_mem_mode(self):
     def disable_mem_mode(self):
         self._mem_mode = False
 
+    def set_config_unencrypted(self):
+        self._config_encrypted = False
+
+    def disable_encrypt_dump(self):
+        self._need_encrypt_dump = False
+
+    def enable_encrypt_dump(self):
+        self._need_encrypt_dump = True
+
     @property
     def sorted_components(self):
         available_depends = list(self.components.keys())
@@ -1201,6 +1216,14 @@ def update_component_info(self, repository):
         self._src_data[component] = ori_data
         return False
 
+    def _update_user_password(self):
+        try:
+            for key in self._src_data:
+                if key == 'user':
+                    self._src_data[key]['password'] = self.user.password
+        except:
+            self.stdio.exception()
+
     def _load(self):
         try:
             with open(self.yaml_path, 'rb') as f:
@@ -1225,6 +1248,8 @@ def _load(self):
                 for comp in depends:
                     conf = self.components[comp]
                     for name in depends[comp]:
+                        if depends[comp] is None:
+                            continue
                         if name == comp:
                             continue
                         if name in self.components:
@@ -1234,6 +1259,39 @@ def _load(self):
         if not self.user:
             self.set_user_conf(UserConfig())
 
+    def _auto_decrypt_password(self):
+        if not self._inner_config:
+            return True
+        if self._config_encrypted is None:
+            if COMMAND_ENV.get(const.ENCRYPT_PASSWORD) == '1':
+                self._config_encrypted = True
+            if not bool(self._inner_config.get_global_config(const.ENCRYPT_PASSWORD)):
+                self.stdio.verbose("auto_decrypt_password: deploy is not encrypted")
+                self._config_encrypted = False
+        if self._config_encrypted and bool(self._inner_config.get_global_config(const.ENCRYPT_PASSWORD)):
+            self.change_deploy_config_password(False, False)
+            self._need_encrypt_dump = True
+        elif not self._config_encrypted and COMMAND_ENV.get(const.ENCRYPT_PASSWORD) == '1':
+            self._need_encrypt_dump = True
+
+    def _encrypt_password_before_dump(self):
+        if not self._inner_config:
+            return True
+        if self._need_encrypt_dump and COMMAND_ENV.get(const.ENCRYPT_PASSWORD) == '1' and not self._config_encrypted:
+            if bool(self._inner_config.get_global_config(const.ENCRYPT_PASSWORD)):
+                change_deploy_encrypt = False
+            else:
+                change_deploy_encrypt = True
+            enable_mem_mode = False
+            if self._mem_mode:
+                enable_mem_mode = True
+                self._mem_mode = False
+            self.change_deploy_config_password(True, change_deploy_encrypt)
+            if enable_mem_mode:
+                self._mem_mode = True
+            self._config_encrypted = True
+        self._update_user_password()
+
     def scale_out(self, config_path):
         ret = True
         depends = {}
@@ -1361,6 +1419,7 @@ def _dump_inner_config(self):
     def _dump(self):
         if self._do_not_dump:
             raise Exception("Cannot dump because flag DO NOT DUMP exists.")
+        self._encrypt_password_before_dump()
         try:
             with open(self.yaml_path, 'w') as f:
                 self._dump_inner_config()
@@ -1548,6 +1607,76 @@ def change_component_config_style(self, component_name, style):
             self._src_data[component_name] = new_config['config']
         return True
 
+    def get_all_password_path(self, global_config, password_paths=[], component=None, parent_key=None):
+        pattern = r'password'
+        pattern1 = r'passwd'
+        for key, value in global_config.items():
+            if parent_key:
+                current_key = f"{parent_key}.{key}"
+            else:
+                current_key = key
+
+            if isinstance(value, dict):
+                self.get_all_password_path(value, password_paths, component, current_key)
+            else:
+                value = str(value)
+                if isinstance(key, str) and isinstance(value, str) and (re.findall(pattern, key, re.IGNORECASE) or re.findall(pattern1, key, re.IGNORECASE)):
+                    self.stdio.verbose(f"find pwd key: {current_key}")
+                    password_paths.append(current_key)
+                if component == const.COMP_PROMETHEUS and parent_key == 'basic_auth_users':
+                    self.stdio.verbose(f"find pwd key: {key}")
+                    password_paths.append(current_key)
+        return password_paths
+
+    def update_password_in_global_config(self, global_config, path, key, enable_encrypt):
+        current_key = path[0]
+        if len(path) == 1:
+            if enable_encrypt:
+                self.stdio.verbose(f"encrypt pwd-key: {current_key}")
+                global_config[current_key] = aes_encrypt(str(global_config[current_key]), key)
+            else:
+                self.stdio.verbose(f"decrypt pwd-key: {current_key}")
+                global_config[current_key] = aes_decrypt(global_config[current_key], key)
+        else:
+            if current_key in global_config and isinstance(global_config[current_key], dict):
+                self.update_password_in_global_config(global_config[current_key], path[1:], key, enable_encrypt)
+
+    def change_deploy_config_password(self, enable_encrypt, change_deploy_encrypt=True, first_encrypt=False):
+        if COMMAND_ENV.get(const.ENCRYPT_PASSWORD) != '1' and not first_encrypt:
+            return True
+        deploy_encrypt_enable = self.inner_config.get_global_config(const.ENCRYPT_PASSWORD) or False
+        if enable_encrypt == deploy_encrypt_enable == self._config_encrypted:
+            return True
+        if not enable_encrypt and not self._config_encrypted:
+            self._need_encrypt_dump = False
+        else:
+
+            cluster_id = ''
+            for component in const.COMPS_OB:
+                if component in self.components.keys():
+                    cluster_config = self.components[component]
+                    global_config = cluster_config.get_global_conf()
+                    cluster_id = str(global_config['cluster_id']) if 'cluster_id' in global_config else ''
+            key = string_to_md5_32bytes(self.name + cluster_id).encode('utf-8')
+            self.stdio.verbose(f"encrypt key: {key}")
+            if self.user.password:
+                if enable_encrypt:
+                    self.user.password = aes_encrypt(str(self.user.password), key)
+                else:
+                    self.user.password = aes_decrypt(self.user.password, key)
+            for component, cluster_config in self.components.items():
+                self.stdio.verbose(f"change {component} pwd encrypt.")
+                password_paths = []
+                global_config = cluster_config.get_global_conf()
+                self.get_all_password_path(global_config, password_paths, component)
+                change_global_config = deepcopy(global_config)
+                for password_path in password_paths:
+                    password_path_keys = password_path.split('.')
+                    self.update_password_in_global_config(change_global_config, password_path_keys, key, enable_encrypt)
+                    cluster_config.update_global_conf(password_path_keys[0], change_global_config[password_path_keys[0]], save=False)
+            self._config_encrypted = enable_encrypt
+        change_deploy_encrypt and self.inner_config.update_global_config(const.ENCRYPT_PASSWORD, enable_encrypt)
+
 
 class Deploy(object):
 
@@ -1564,6 +1693,10 @@ def __init__(self, config_dir, config_parser_manager=None, stdio=None):
         self.stdio = stdio
         self.config_parser_manager = config_parser_manager
         self._uprade_meta = None
+        self._config_encrypted = True
+
+    def set_config_decrypted(self):
+        self._config_encrypted = False
 
     def use_model(self, name, repository, dump=True):
         self.deploy_info.components[name] = {
@@ -1616,7 +1749,8 @@ def deploy_info(self):
 
     def _load_deploy_config(self, path):
         yaml_loader = YamlLoader(stdio=self.stdio)
-        deploy_config = DeployConfig(path, yaml_loader=yaml_loader, config_parser_manager=self.config_parser_manager, stdio=self.stdio)
+        inner_config = InnerConfig(self.get_inner_config_path(self.config_dir), yaml_loader=yaml_loader)
+        deploy_config = DeployConfig(path, yaml_loader=yaml_loader, inner_config=inner_config, config_parser_manager=self.config_parser_manager, config_encrypted=self._config_encrypted,stdio=self.stdio)
         deploy_info = self.deploy_info
         for component_name in deploy_info.components:
             if component_name not in deploy_config.components:
@@ -1627,7 +1761,6 @@ def _load_deploy_config(self, path):
                 cluster_config.version = config['version']
             if 'hash' in config and config['hash']:
                 cluster_config.package_hash = config['hash']
-        deploy_config.inner_config = InnerConfig(self.get_inner_config_path(self.config_dir), yaml_loader=yaml_loader)
         return deploy_config
 
     @property

_deploy.py

@@ -57,4 +57,14 @@
 ENV_OBD_INSTALL_PRE = "OBD_INSTALL_PRE"
 
 # obdeploy install path. default /usr/obd/
-ENV_OBD_INSTALL_PATH = "OBD_INSTALL_PATH"
+ENV_OBD_INSTALL_PATH = "OBD_INSTALL_PATH"
+
+# obd web idle minite time before shutdown. default 30
+# if you do not want to set idle time, you can set it to "infinity"
+ENV_IDLE_TIME_BEFORE_SHUTDOWN_MINITES = "IDLE_TIME_BEFORE_SHUTDOWN_MINITES"
+
+# obdesktop/docker mock cluster id
+ENV_CUSTOM_CLUSTER_ID = "CUSTOM_CLUSTER_ID"
+
+# obdeploy web type
+ENV_OBD_WEB_TYPE = "OBD_WEB_TYPE"

_environ.py

@@ -31,6 +31,9 @@ def __init__(self, code, msg):
     def __str__(self):
         return  self.msg
 
+    def __bool__(self):
+        return False
+
 
 class OBDErrorCodeTemplate(object):
 
@@ -142,6 +145,7 @@ class InitDirFailedErrorMessage(object):
 WC_OBSERVER_SAME_DISK = OBDErrorCodeTemplate(1012, '({ip}) clog and data use the same disk ({disk})')
 WC_FAIL_TO_RESTART_OR_RELOAD = OBDErrorCodeTemplate(1021, 'The components has been {action}, but encountered problems when reloading or restarting. Details:\n{detail}')
 WC_FAIL_TO_RESTART_OR_RELOAD_AFTER_SCALE_OUT = OBDErrorCodeTemplate(1022, 'The cluster has been scaled out, but encountered problems when reloading or restarting. Details:\n{detail}')
+WC_CHANGE_SYSTEM_PARAMETER_FAILED = OBDErrorCodeTemplate(1023, '({server}) failed to change system parameter: {key})')
 
 # error code for observer
 EC_OBSERVER_NOT_ENOUGH_MEMORY = OBDErrorCodeTemplate(2000, '({ip}) not enough memory. (Free: {free}, Need: {need})')
@@ -166,6 +170,8 @@ class InitDirFailedErrorMessage(object):
 EC_OBSERVER_GET_MEMINFO_FAIL = OBDErrorCodeTemplate(2011, "{server}: fail to get memory info.\nPlease configure 'memory_limit' manually in configuration file")
 EC_OBSERVER_FAIL_TO_START_OCS = OBDErrorCodeTemplate(2012, 'Failed to start {server} obshell')
 EC_OBSERVER_UNKONE_SCENARIO = OBDErrorCodeTemplate(2013, 'Unknown scenario: {scenario}')
+EC_CPU_NOT_SUPPORT_AVX = OBDErrorCodeTemplate(2014, "{server}'s cpu not support avx")
+EC_OBSERVER_DISABLE_AUTOSTART = OBDErrorCodeTemplate(2015, "{server}: Failed to modify the configuration of the automatic startup. Please check whether the current user has sudo permissions")
 
 WC_OBSERVER_SYS_MEM_TOO_LARGE = OBDErrorCodeTemplate(2010, '({server}): system_memory too large. system_memory should be less than {factor} * memory_limit/memory_limit_percentage.')
 
@@ -256,6 +262,9 @@ class InitDirFailedErrorMessage(object):
 EC_OBDIAG_OPTIONS_FORMAT_ERROR = OBDErrorCodeTemplate(6002, 'obdiag options {option} format error, please check the value : {value}')
 EC_OBDIAG_FUCYION_FAILED = OBDErrorCodeTemplate(6003, 'Failed to execute obdiag function {fuction}')
 
+# obshell
+EC_OBSHELL_GENERAL_ERROR = OBDErrorCodeTemplate(6100, 'OBSHELL: {msg}')
+
 # Unexpected exceptions code
 EC_UNEXPECTED_EXCEPTION = OBDErrorCodeTemplate(9999, 'Unexpected exception: need to be posted on "https://ask.oceanbase.com", and we will help you resolve them.')
 
@@ -310,6 +319,6 @@ class InitDirFailedErrorMessage(object):
 SUG_OCP_SERVER_EXIST_METADB_TENANT_NOT_ENOUGH = OBDErrorSuggestionTemplate('Please reduce the ocp meta tenant memory or ocp monitor tenant memory')
 SUG_OCP_SERVER_NOT_EXIST_METADB_TENANT_NOT_ENOUGH = OBDErrorSuggestionTemplate('Please increase the meta db memory_limit and reduce the ocp meta tenant memory or ocp monitor tenant memory')
 SUG_OB_SYS_USERNAME = OBDErrorSuggestionTemplate('Please delete the "ob_sys_username" parameter.')
-SUG_OB_SYS_PASSWORD = OBDErrorSuggestionTemplate('''Please set the "ob_sys_password" for oblogproxy by configuring the "cdcro_password" parameter in the "oceanbase" or "oceanbase-ce" component.''')
+SUG_OB_SYS_PASSWORD = OBDErrorSuggestionTemplate('''Please set the "ob_sys_password" for oblogproxy by configuring the "cdcro_password" parameter in the "oceanbase" or "oceanbase-ce" or "oceanbase-standalone" component.''')
 SUG_OBAGENT_EDIT_HTTP_BASIC_AUTH_PASSWORD = OBDErrorSuggestionTemplate('Please edit the `http_basic_auth_password`, cannot contain characters other than uppercase letters, lowercase characters, digits, special characters:~^*{{}}[]_-+', fix_eval=[FixEval(FixEval.DEL, 'http_basic_auth_password')], auto_fix=True)
 SUB_OBSERVER_UNKONE_SCENARIO = OBDErrorSuggestionTemplate('Please select a valid scenario from the options: {scenarios}')