v2025.1.0

Dockerfile

@@ -16,8 +16,8 @@ ARG USER_GID=$USER_UID
 # Whether to download Provider as part of image creation
 ARG DOWNLOAD_PROVIDER=YES
 # TF Provider version
-ARG TF_OCI_PROVIDER=6.15.0
-ARG TF_NULL_PROVIDER=3.2.1
+ARG TF_OCI_PROVIDER=6.30.0
+ARG TF_NULL_PROVIDER=3.2.3
 
 RUN microdnf install -y sudo && \
     groupadd --gid $USER_GID $USERNAME && \
@@ -62,6 +62,7 @@ sudo unzip terraform-provider-null_${TF_NULL_PROVIDER}_linux_amd64.zip -d /cd3us
 sudo cp -r /cd3user/.terraform.d/plugins/registry.terraform.io /cd3user/.terraform.d/plugins/registry.opentofu.org && \
 sudo chown -R cd3user:cd3user /cd3user/ && \
 sudo rm -rf terraform-provider-null_${TF_NULL_PROVIDER}_linux_amd64.zip terraform-provider-oci_${TF_OCI_PROVIDER}_linux_amd64.zip ;\
+
 fi
 
 ##################################### START INSTALLING JENKINS ###################################

OCIWorkVMStack/scripts/installToolkit.sh

@@ -66,7 +66,7 @@ fi
 sudo podman --version >> $logfile 2>&1
 
 echo "***Download Toolkit***" >> $logfile 2>&1
-sudo git clone https://github.com/oracle-devrel/cd3-automation-toolkit.git $toolkit_dir >> $logfile 2>&1
+sudo git clone https://github.com/oracle-devrel/cd3-automation-toolkit.git -b develop $toolkit_dir >> $logfile 2>&1
 stop_exec
 
 curl -H "Authorization: Bearer Oracle" -L http://169.254.169.254/opc/v2/instance/ -o /tmp/metadata.json

README.md

@@ -21,7 +21,7 @@ Additionally, the toolkit also supports seamless resource management using OCI D
 🚀 Click the below button to quickly launch CD3 toolkit container in Oracle Cloud and start managing your Infra as Code. 
 <br>
 
-[![Deploy_To_OCI](https://oci-resourcemanager-plugin.plugins.oci.oraclecloud.com/latest/deploy-to-oracle-cloud.svg)](https://cloud.oracle.com/resourcemanager/stacks/create?zipUrl=https://github.com/oracle-devrel/cd3-automation-toolkit/archive/refs/heads/main.zip)
+[![Deploy_To_OCI](https://oci-resourcemanager-plugin.plugins.oci.oraclecloud.com/latest/deploy-to-oracle-cloud.svg)](https://cloud.oracle.com/resourcemanager/stacks/create?zipUrl=https://github.com/oracle-devrel/cd3-automation-toolkit/archive/refs/heads/develop.zip)
 
 <br>
 

cd3_automation_toolkit/Compute/export_dedicatedvmhosts_nonGreenField.py

@@ -19,7 +19,24 @@
 oci_obj_names = {}
 
 
-def print_dedicatedvmhosts(region, dedicatedvmhost, values_for_column, ntk_compartment_name,state):
+def print_dedicatedvmhosts(region, dedicatedvmhost, values_for_column, ntk_compartment_name,export_tags, state):
+
+    # Tags filter
+    defined_tags = dedicatedvmhost.defined_tags
+    tags_list = []
+    for tkey, tval in defined_tags.items():
+        for kk, vv in tval.items():
+            tag = tkey + "." + kk + "=" + vv
+            tags_list.append(tag)
+
+    if export_tags == []:
+        check = True
+    else:
+        check = any(e in tags_list for e in export_tags)
+    # None of Tags from export_tags exist on this instance; Dont export this instance
+    if check == False:
+        return
+
     dedicatedvmhost_tf_name = commonTools.check_tf_variable(dedicatedvmhost.display_name)
     tf_resource = f'module.dedicated-hosts[\\"{dedicatedvmhost_tf_name}\\"].oci_core_dedicated_vm_host.dedicated_vm_host'
     if tf_resource not in state["resources"]:
@@ -47,7 +64,7 @@ def print_dedicatedvmhosts(region, dedicatedvmhost, values_for_column, ntk_compa
             values_for_column = commonTools.export_extra_columns(oci_objs, col_header, sheet_dict, values_for_column)
 
 # Execution of the code begins here
-def export_dedicatedvmhosts(inputfile, outdir, service_dir, config, signer, ct, export_compartments=[], export_regions=[]):
+def export_dedicatedvmhosts(inputfile, outdir, service_dir, config, signer, ct, export_compartments=[], export_regions=[],export_tags=[]):
     global tf_import_cmd
     global sheet_dict
     global importCommands
@@ -102,11 +119,12 @@ def export_dedicatedvmhosts(inputfile, outdir, service_dir, config, signer, ct,
         compute_client = oci.core.ComputeClient(config=config,retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY, signer=signer)
 
         for ntk_compartment_name in export_compartments:
+
             dedicatedvmhosts = oci.pagination.list_call_get_all_results(compute_client.list_dedicated_vm_hosts,compartment_id=ct.ntk_compartment_ids[ntk_compartment_name], lifecycle_state="ACTIVE")
 
             for dedicatedvmhost in dedicatedvmhosts.data:
                 dedicatedvmhost=compute_client.get_dedicated_vm_host(dedicatedvmhost.id).data
-                print_dedicatedvmhosts(region, dedicatedvmhost,values_for_column, ntk_compartment_name,state)
+                print_dedicatedvmhosts(region, dedicatedvmhost,values_for_column, ntk_compartment_name,export_tags,state)
 
     # writing data
     for reg in export_regions:

cd3_automation_toolkit/Compute/export_instances_nonGreenField.py

@@ -33,6 +33,11 @@ def adding_columns_values(region, ad, fd, vs, publicip, privateip, os_dname, sha
             values_for_column_instances[col_header].append(os_dname)
         elif (col_header == "Shape"):
             values_for_column_instances[col_header].append(shape)
+        elif (col_header == "Boot Volume Size In GBs"):
+            size = bdet.size_in_gbs
+            if size < 50:
+                size=""
+            values_for_column_instances[col_header].append(size)
         elif (col_header == "SSH Key Var Name"):
             values_for_column_instances[col_header].append(key_name)
         elif (col_header == "Compartment Name"):
@@ -81,7 +86,7 @@ def find_vnic(ins_id, compartment_id):
         return net
 
 
-def __get_instances_info(compartment_name, compartment_id, reg_name, display_names, ad_names, ct,state):
+def __get_instances_info(compartment_name, compartment_id, reg_name, display_names, ad_names, export_tags, ct,state):
     config.__setitem__("region", ct.region_dict[reg_name])
     compute = oci.core.ComputeClient(config=config, retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY,signer=signer)
     network = oci.core.VirtualNetworkClient(config=config, retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY,signer=signer)
@@ -102,18 +107,34 @@ def __get_instances_info(compartment_name, compartment_id, reg_name, display_nam
                 if (not any(e in AD_name for e in ad_names)):
                     continue
 
-            # Continue to next one if display names donot match the filter
+            # Continue to next one if display names do not match the filter
             if (display_names is not None):
                 if (not any(e in ins_dname for e in display_names)):
                     continue
 
+            # Tags filter
+            ins_defined_tags = ins.defined_tags
+            tags_list=[]
+            for tkey,tval in ins_defined_tags.items():
+                for kk,vv in tval.items():
+                    tag = tkey+"."+kk+"="+vv
+                    tags_list.append(tag)
+
+            if export_tags == []:
+                check = True
+            else:
+                check = any(e in tags_list for e in export_tags)
+            #None of Tags from export_tags exist on this instance; Dont export this instance
+            if check == False:
+                continue
+
             # Continue to next one if it's an OKE instance
             if 'oke-' in ins_dname:
                 ins_defined_tags = ins.defined_tags
                 if 'Oracle-Tags' in ins_defined_tags.keys():
                     if 'CreatedBy' in ins_defined_tags['Oracle-Tags'].keys():
                         created_by = ins_defined_tags['Oracle-Tags']['CreatedBy']
-                        if created_by == 'oke':
+                        if ".nodepool." in created_by or created_by == 'oke':
                             continue
 
             ins_fd = ins.fault_domain  # FD
@@ -279,7 +300,7 @@ def __get_instances_info(compartment_name, compartment_id, reg_name, display_nam
 
 
 # Execution of the code begins here
-def export_instances(inputfile, outdir, service_dir,config1, signer1, ct, export_compartments=[], export_regions=[],display_names=[],ad_names=[]):
+def export_instances(inputfile, outdir, service_dir,config1, signer1, ct, export_compartments=[], export_regions=[],export_tags=[],display_names=[],ad_names=[]):
     cd3file = inputfile
 
     if ('.xls' not in cd3file):
@@ -330,7 +351,7 @@ def export_instances(inputfile, outdir, service_dir,config1, signer1, ct, export
         except Exception as e:
             pass
         for ntk_compartment_name in export_compartments:
-           __get_instances_info(ntk_compartment_name, ct.ntk_compartment_ids[ntk_compartment_name], reg, display_names, ad_names,ct,state)
+           __get_instances_info(ntk_compartment_name, ct.ntk_compartment_ids[ntk_compartment_name], reg, display_names, ad_names,export_tags,ct,state)
 
     # writing image ocids and SSH keys into variables file
     var_data = {}

cd3_automation_toolkit/CostManagement/Budget/export_budgets_nonGreenField.py

@@ -104,7 +104,7 @@ def print_budgets(values_for_columns, region, budget,budget_name,budget_alert_ru
 
 
 # Execution of the code begins here
-def export_budgets_nongreenfield(inputfile, outdir, service_dir, config, signer, ct,export_regions=[]):
+def export_budgets_nongreenfield(inputfile, outdir, service_dir, config, signer, ct,export_regions=[],export_tags=[]):
     global importCommands
     global values_for_column_budgets
     global sheet_dict_budgets,tf_or_tofu
@@ -155,6 +155,23 @@ def export_budgets_nongreenfield(inputfile, outdir, service_dir, config, signer,
         budgets_list = oci.pagination.list_call_get_all_results(budgets_client.list_budgets,compartment_id=tenancy_id,lifecycle_state="ACTIVE",target_type="ALL")
         if budgets_list.data != []:
             for budget in budgets_list.data:
+
+                # Tags filter
+                defined_tags = budget.defined_tags
+                tags_list = []
+                for tkey, tval in defined_tags.items():
+                    for kk, vv in tval.items():
+                        tag = tkey + "." + kk + "=" + vv
+                        tags_list.append(tag)
+
+                if export_tags == []:
+                    check = True
+                else:
+                    check = any(e in tags_list for e in export_tags)
+                # None of Tags from export_tags exist on this instance; Dont export this instance
+                if check == False:
+                    continue
+
                 budget_name = str(budget.display_name)
                 budget_id = str(budget.id)
                 budget_tf_name = commonTools.check_tf_variable(budget_name)

cd3_automation_toolkit/Database/__init__.py

@@ -8,4 +8,7 @@
 from .export_exa_vmclusters_nonGreenField import export_exa_vmclusters
 from .export_exa_infra_nonGreenField import export_exa_infra
 from .export_adb_nonGreenField import export_adbs
-
+from .create_terraform_mysql_db import create_terraform_mysql_db
+from .create_terraform_mysql_configuration import create_terraform_mysql_configuration
+from .export_mysql_db_nonGreenField import export_mysql_db
+from .export_mysql_configuration_nonGreenField import export_mysql_configuration, export_mysql_configurations

cd3_automation_toolkit/Database/create_terraform_mysql_configuration.py

@@ -0,0 +1,171 @@
+#!/usr/bin/python3
+# Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
+#
+# This script will produce a Terraform file that will be used to set up OCI Database
+# MySQL Configuration
+#
+# Author: Generated by Cascade
+# Oracle Consulting
+#
+
+import os
+import re
+from jinja2 import Environment, FileSystemLoader
+from oci.config import DEFAULT_LOCATION
+from pathlib import Path
+from commonTools import *
+
+def create_terraform_mysql_configuration(inputfile, outdir, service_dir, prefix, ct):
+    filename = inputfile
+    sheetName = "MySQL-Configurations"
+    auto_tfvars_filename = prefix + '_' + sheetName.lower() + '.auto.tfvars'
+
+    # Initialize tracking variables
+    prev_values = {
+        'region': '',
+        'compartment_name': '',
+        'display_name': '',
+        'description': '',
+        'shape_name': ''
+    }
+
+    tfStr = {}
+    mysql_config_vars = {}
+
+    # Load the template file
+    file_loader = FileSystemLoader(f'{Path(__file__).parent}/templates')
+    env = Environment(loader=file_loader, keep_trailing_newline=True, trim_blocks=True, lstrip_blocks=True)
+    template = env.get_template('mysql-configuration-template')
+
+    # Add custom functions to template environment
+    def make_config_keys(config):
+        return lambda: config.keys()
+    def make_config_value(config):
+        return lambda key: config.get(key, '')
+    env.globals['config_keys'] = None
+    env.globals['config_value'] = None
+
+    # Read cd3 using pandas dataframe
+    df, col_headers = commonTools.read_cd3(filename, sheetName)
+    df = df.dropna(how='all')
+    df = df.reset_index(drop=True)
+
+    # List of column headers
+    dfcolumns = df.columns.values.tolist()
+
+    # Initialize empty TF string for each region
+    for reg in ct.all_regions:
+        tfStr[reg] = ''
+        srcdir = outdir + "/" + reg + "/" + service_dir + "/"
+        resource = sheetName.lower()
+        commonTools.backup_file(srcdir, resource, auto_tfvars_filename)
+        mysql_config_vars[reg] = {}
+
+    # Process each row
+    for i in df.index:
+        # Get values from row
+        region = str(df.loc[i, 'Region']).strip()
+        compartment_name = str(df.loc[i, 'Compartment Name']).strip()
+        display_name = str(df.loc[i, 'Display Name']).strip()
+        description = str(df.loc[i, 'Description']).strip()
+        shape_name = str(df.loc[i, 'Shape Name']).strip()
+
+        # Handle empty values using previous values
+        if region.lower() == 'nan' or region == '':
+            region = prev_values['region']
+        if compartment_name.lower() == 'nan' or compartment_name == '':
+            compartment_name = prev_values['compartment_name']
+        if display_name.lower() == 'nan' or display_name == '':
+            display_name = prev_values['display_name']
+        if description.lower() == 'nan' or description == '':
+            description = prev_values['description']
+        if shape_name.lower() == 'nan' or shape_name == '':
+            shape_name = prev_values['shape_name']
+
+        # Update previous values
+        prev_values.update({
+            'region': region,
+            'compartment_name': compartment_name,
+            'display_name': display_name,
+            'description': description,
+            'shape_name': shape_name
+        })
+
+        # Skip if essential values are missing
+        if region.lower() == 'nan' or display_name.lower() == 'nan':
+            continue
+
+        # Initialize region if needed
+        region = region.strip().lower()
+        if region not in ct.all_regions:
+            print("\nERROR!!! Invalid Region; It should be one of the regions tenancy is subscribed to..Exiting!")
+            exit(1)
+
+        # Check for variable row
+        user_var_name = str(df.loc[i, 'users_variable_name']).strip()
+        user_var_value = str(df.loc[i, 'users_variable_value']).strip()
+
+        # Initialize config if needed
+        config_tf_name = commonTools.check_tf_variable(display_name)
+        if config_tf_name not in mysql_config_vars[region]:
+            mysql_config_vars[region][config_tf_name] = {
+                'config_display_tf_name': config_tf_name,
+                'compartment_tf_name': commonTools.check_tf_variable(compartment_name),
+                'display_name': display_name,
+                'description': description,
+                'shape_name': shape_name
+            }
+
+        # Only process variable if both name and value are present and not empty
+        if (user_var_name.lower() != 'nan' and user_var_name != '' and 
+            user_var_value.lower() != 'nan' and user_var_value != ''):
+            # Add variable with mysql_configuration_ prefix
+            var_name = f"mysql_configuration_variables_{user_var_name}"
+            # Handle boolean values
+            if user_var_value.lower() in ['true', 'false']:
+                mysql_config_vars[region][config_tf_name][var_name] = user_var_value.capitalize()
+            else:
+                mysql_config_vars[region][config_tf_name][var_name] = user_var_value
+
+    # Generate terraform configuration for each region
+    for region in mysql_config_vars:
+        if not mysql_config_vars[region]:
+            continue
+
+        # Start with count = 0 to generate opening
+        env.globals['count'] = 0
+        tfStr[region] = template.render()
+
+        configs = list(mysql_config_vars[region].items())
+        for i, (config_name, config) in enumerate(configs):
+            # Update template functions for this config
+            env.globals['config_keys'] = make_config_keys(config)
+            env.globals['config_value'] = make_config_value(config)
+            config['loop'] = {'last': i == len(configs) - 1}
+            env.globals['count'] = 1
+
+            # Render configuration
+            rendered_config = template.render(**config)
+            tfStr[region] += rendered_config
+
+        # Add closing brace
+        env.globals['count'] = 2
+        tfStr[region] += template.render()
+
+    # Write files
+    for region in tfStr:
+        if tfStr[region] != '':
+            srcdir = outdir + "/" + region + "/" + service_dir + "/"
+            os.makedirs(srcdir, exist_ok=True)
+
+            outfile = srcdir + "/" + auto_tfvars_filename
+            # Clean up the output
+            tfStr[region] = tfStr[region].strip()
+            # Fix any potential formatting issues
+            tfStr[region] = re.sub(r'\s+mysql_configuration_variables_', '\n    mysql_configuration_variables_', tfStr[region])
+            tfStr[region] = re.sub(r'}\s*,\s*', '},\n', tfStr[region])
+            tfStr[region] = re.sub(r'\n\s*\n\s*\n', '\n\n', tfStr[region])
+
+            with open(outfile, 'w') as f:
+                f.write(tfStr[region])
+            print(f"Created MySQL configuration for region {region} in {outfile}")