Enhance fetching of remote parameters and usecase files

.vscode/launch.json

@@ -49,6 +49,34 @@
             "console": "integratedTerminal",
             "justMyCode": true
         },
+        {
+            "name": "Python: remote config files - no auth",
+            "type": "python",
+            "request": "launch",
+            "program": "btpsa",
+            "args": [
+                "-parameterfile",
+                "https://raw.githubusercontent.com/<USER ID>/test-btpsa-remote-pub/main/parameters.json"
+            ],
+            "console": "integratedTerminal",
+            "justMyCode": true
+        },
+        {
+            "name": "Python: remote config files - with auth",
+            "type": "python",
+            "request": "launch",
+            "program": "btpsa",
+            "args": [
+                "-parameterfile",
+                "https://raw.githubusercontent.com/<USER ID>/test-btpsa-remote-private/main/parameters.json",
+                "-externalConfigAuthMethod",
+                "token",
+                "-externalConfigToken",
+                "<YOUR TOKEN>"
+            ],
+            "console": "integratedTerminal",
+            "justMyCode": true
+        },
         {
             "name": "Python: Integrationtest Generator",
             "type": "python",

config/templates/libs/BTPSA-PARAMETERS.json

@@ -156,6 +156,47 @@
             "title": "list of environment variables on OS level to be used within commands defined in the `executeBeforeAccountSetup` and `executeAfterAccountSetup`.",
             "default": null
         },
+        "externalConfigAuthMethod": {
+            "type": [
+                "string",
+                "null"
+            ],
+            "description": "authentication method for external configuration file retrieval",
+            "title": "authentication method for external configuration",
+            "default": null,
+            "enum": [
+                "basicAuthentication",
+                "token"
+            ]
+        },
+        "externalConfigUserName": {
+            "type": [
+                "string",
+                "null"
+            ],
+            "description": "user name for external configuration file retrieval (basic authentication)",
+            "title": "user name for external configuration file retrieval ",
+            "default": null
+        },
+        "externalConfigPassword": {
+            "type": [
+                "string",
+                "null"
+            ],
+            "description": "password for external configuration file retrieval (basic authentication)",
+            "title": "password for external configuration file retrieval",
+            "default": null
+
+        },
+        "externalConfigToken": {
+            "type": [
+                "string",
+                "null"
+            ],
+            "description": "token for external configuration file retrieval (token-based authentication)",
+            "title": "token for external configuration file retrieval",
+            "default": null
+        },
         "fallbackserviceplan": {
             "type": [
                 "string",
@@ -339,6 +380,12 @@
             "title": "switch to run default tests at the beginning of the script",
             "default": true
         },
+        "subaccountenablebeta": {
+            "type": "boolean",
+            "description": "if set to true, a newly create subaccount will have beta features enabled",
+            "title": "enable beta features in new subaccount",
+            "default": false
+        },
         "subaccountid": {
             "type": "string",
             "description": "id of your sub account that should be used",

libs/btpsa-parameters.json

@@ -156,6 +156,47 @@
             "title": "list of environment variables on OS level to be used within commands defined in the `executeBeforeAccountSetup` and `executeAfterAccountSetup`.",
             "default": null
         },
+        "externalConfigAuthMethod": {
+            "type": [
+                "string",
+                "null"
+            ],
+            "description": "authentication method for external configuration file retrieval",
+            "title": "authentication method for external configuration",
+            "default": null,
+            "enum": [
+                "basicAuthentication",
+                "token"
+            ]
+        },
+        "externalConfigUserName": {
+            "type": [
+                "string",
+                "null"
+            ],
+            "description": "user name for external configuration file retrieval (basic authentication)",
+            "title": "user name for external configuration file retrieval ",
+            "default": null
+        },
+        "externalConfigPassword": {
+            "type": [
+                "string",
+                "null"
+            ],
+            "description": "password for external configuration file retrieval (basic authentication)",
+            "title": "password for external configuration file retrieval",
+            "default": null
+
+        },
+        "externalConfigToken": {
+            "type": [
+                "string",
+                "null"
+            ],
+            "description": "token for external configuration file retrieval (token-based authentication)",
+            "title": "token for external configuration file retrieval",
+            "default": null
+        },
         "fallbackserviceplan": {
             "type": [
                 "string",
@@ -339,6 +380,12 @@
             "title": "switch to run default tests at the beginning of the script",
             "default": true
         },
+        "subaccountenablebeta": {
+            "type": "boolean",
+            "description": "if set to true, a newly create subaccount will have beta features enabled",
+            "title": "enable beta features in new subaccount",
+            "default": false
+        },
         "subaccountid": {
             "type": "string",
             "description": "id of your sub account that should be used",

libs/python/btp_cli.py

@@ -66,7 +66,8 @@ def __init__(self):
         self.definedEnvironments = getEnvironmentsForUsecase(self, allServices)
         self.definedAppSubscriptions = getServiceCategoryItemsFromUsecaseFile(
             self, allServices, self.availableCategoriesApplication)
-        usecaseFileContent = getJsonFromFile(self.usecasefile)
+        # Use case file can be remote, so we need to provide authentication information    
+        usecaseFileContent = getJsonFromFile(self.usecasefile, self.externalConfigAuthMethod, self.externalConfigUserName, self.externalConfigPassword, self.externalConfigToken)
         self.definedRoleCollections = usecaseFileContent.get(
             "assignrolecollections")
 
@@ -791,6 +792,7 @@ def getEnvironmentsForUsecase(btpUsecase: BTPUSECASE, allServices):
     environments = []
 
     paramServicesFile = FOLDER_SCHEMA_LIBS + "btpsa-usecase.json"
+    # Param definition: no remote file access possible, no authentication parameters needed
     paramDefinition = getJsonFromFile(paramServicesFile)
 
     for usecaseService in allServices:
@@ -828,7 +830,8 @@ def getServiceCategoryItemsFromUsecaseFile(btpUsecase: BTPUSECASE, allServices,
 
 
 def getAdminsFromUsecaseFile(btpUsecase: BTPUSECASE):
-    usecase = getJsonFromFile(btpUsecase.usecasefile)
+    # Use case file can be remote, so we need to provide authentication information
+    usecase = getJsonFromFile(btpUsecase.usecasefile, btpUsecase.externalConfigAuthMethod, btpUsecase.externalConfigUserName, btpUsecase.externalConfigPassword, btpUsecase.externalConfigToken)
 
     items = []
     if "admins" in usecase:

libs/python/helperAccountInfo.py

@@ -1,4 +1,3 @@
-import json
 import logging
 import os
 from libs.python.helperCommandExecution import runCommandAndGetJsonResult
@@ -27,31 +26,6 @@ def getAllDatacenters(services):
     return resultCF
 
 
-def getJsonFromFile(filename):
-    data = None
-    foundError = False
-
-    try:
-        # Opening JSON file
-        f = open(filename)
-        # returns JSON object as a dictionary
-        data = json.load(f)
-    except IOError:
-        print("Can't open json file >" + filename + "<")
-        foundError = True
-    except ValueError as err:
-        print("There is an issue in the json file >" + filename +
-              "<. Issue starts on character position " + str(err.pos) + ": " + err.msg)
-        foundError = True
-    finally:
-        f.close()
-
-    if foundError is True:
-        print("Can't run the use case before the error(s) mentioned above are not fixed")
-        exit()
-    return data
-
-
 def getDataCenterFromService(service):
     result = []
     for plan in service["servicePlans"]:

libs/python/helperArgParser.py

@@ -16,6 +16,7 @@ def setupParams(myArguments):
     parser = argparse.ArgumentParser()
     if myArguments is not None and myArguments != "":
 
+        # local arguments: no remote file access possible, no authentication parameters needed
         allJsonParameters = getJsonFromFile(myArguments)
         for key, value in allJsonParameters.get("properties").items():
             argument = key
@@ -59,7 +60,13 @@ def setupParams(myArguments):
             parameterfile = getDefaultValueForParameter(allJsonParameters, "parameterfile")
 
         if parameterfile is not None and parameterfile != "":
-            myParameters = getJsonFromFile(parameterfile)
+            # Distinguish remote access when provisioning of parameters is done via CLI
+            if args.externalConfigAuthMethod == "basicAuthentication" and args.externalConfigUserName is not None and args.externalConfigPassword is not None:
+                myParameters = getJsonFromFile(filename=parameterfile, externalConfigAuthMethod=args.externalConfigAuthMethod, externalConfigUserName=args.externalConfigUserName, externalConfigPassword=args.externalConfigPassword)
+            elif args.externalConfigAuthMethod == "token" and args.externalConfigToken is not None: 
+                myParameters = getJsonFromFile(filename=parameterfile, externalConfigAuthMethod=args.externalConfigAuthMethod, externalConfigToken=args.externalConfigToken)
+            else:   
+                myParameters = getJsonFromFile(filename=parameterfile)
 
             for key in myParameters:
                 # Get the default values for the keys of the args object
@@ -86,6 +93,7 @@ def setupParams(myArguments):
 
             # in case the parameter file does not include all parameter keys, add the missing ones to the args
             btpSetupAutomatorArguments = FOLDER_SCHEMA_LIBS + "btpsa-parameters.json"
+            # Param definition: no remote file access possible, no authentication parameters needed
             allJsonParameters = getJsonFromFile(btpSetupAutomatorArguments)
             for key, value in allJsonParameters.get("properties").items():
                 default = value.get("default")
@@ -129,7 +137,7 @@ def setupParamsServices():
 
 
 def checkProvidedArguments(btpUsecase):
-    usecaseInfo = getJsonFromFile(btpUsecase.usecasefile)
+    usecaseInfo = getJsonFromFile(btpUsecase.usecasefile, externalConfigAuthMethod=btpUsecase.externalConfigAuthMethod, externalConfigUserName=btpUsecase.externalConfigUserName, externalConfigPassword=btpUsecase.externalConfigPassword, externalConfigToken=btpUsecase.externalConfigToken)
     if "aboutThisUseCase" in usecaseInfo:
         info = usecaseInfo["aboutThisUseCase"]
         log.header("Info about use case to be executed")

libs/python/helperCommandExecution.py

@@ -62,7 +62,7 @@ def login_btp(btpUsecase):
 
 def fetchEmailAddressFromBtpConfigFile(btpUsecase):
     btpConfigFile = os.environ['BTP_CLIENTCONFIG']
-    jsonResult = getJsonFromFile(btpConfigFile)
+    jsonResult = getJsonFromFile(filename=btpConfigFile, externalConfigAuthMethod=btpUsecase.externalConfigAuthMethod, externalConfigUserName=btpUsecase.externalConfigUserName, externalConfigPassword=btpUsecase.externalConfigPassword, externalConfigToken=btpUsecase.externalConfigToken)
     if "Authentication" in jsonResult and "Mail" in jsonResult["Authentication"]:
         btpUsecase.myemail = jsonResult["Authentication"]["Mail"]
         return btpUsecase.myemail
@@ -179,7 +179,7 @@ def runCommandAndGetJsonResult(btpUsecase, command, format, message):
 
 
 def executeCommandsFromUsecaseFile(btpUsecase, message, jsonSection):
-    usecaseDefinition = getJsonFromFile(btpUsecase.usecasefile)
+    usecaseDefinition = getJsonFromFile(filename=btpUsecase.usecasefile, externalConfigAuthMethod=btpUsecase.externalConfigAuthMethod, externalConfigUserName=btpUsecase.externalConfigUserName, externalConfigPassword=btpUsecase.externalConfigPassword, externalConfigToken=btpUsecase.externalConfigToken)
 
     if jsonSection in usecaseDefinition and len(usecaseDefinition[jsonSection]) > 0:
         commands = usecaseDefinition[jsonSection]

libs/python/helperJson.py

@@ -2,6 +2,7 @@
 import os
 import sys
 import re
+from requests.auth import HTTPBasicAuth
 import requests
 import logging
 from libs.python.helperDrawio import getUseCaseDataFromDrawIoFile
@@ -10,8 +11,9 @@
 log = logging.getLogger(__name__)
 
 
-def getJsonFromFile(filename):
+def getJsonFromFile(filename, externalConfigAuthMethod=None, externalConfigUserName=None, externalConfigPassword=None, externalConfigToken=None):
     data = None
+    thisRequest = None
     foundError = False
     f = None
 
@@ -22,13 +24,23 @@ def getJsonFromFile(filename):
         return result
 
     if "http://" in filename or "https://" in filename:
+        if "http://" in filename:
+            log.warning("Using http instead of https for url: " + filename + ". This is not recommended. Support will be removed in next major release!")
+
         data = None
         try:
-            thisRequest = requests.get(filename)
-            data = json.loads(thisRequest.text)
+            if externalConfigAuthMethod == "basicAuthentication" and externalConfigUserName is not None and externalConfigPassword is not None:    
+                basic = HTTPBasicAuth(externalConfigUserName, externalConfigPassword)
+                thisRequest = requests.get(filename, auth=basic)
+            elif externalConfigAuthMethod == "token" and externalConfigToken is not None:
+                thisRequest = requests.get(filename, headers={'Authorization': 'Bearer ' + externalConfigToken})
+            else:
+                thisRequest = requests.get(filename)
         except Exception as e:
             log.error("please check the json file >" + filename + "<: " + str(e))
             sys.exit(os.EX_DATAERR)
+
+        data = json.loads(thisRequest.text)
         return data
 
     try:

libs/python/helperRolesAndUsers.py

@@ -71,7 +71,8 @@ def getSubaccountAdmins(btpUsecase):
 
 
 def getRoleCollectionsOfServices(btpUsecase):
-    usecase = getJsonFromFile(btpUsecase.usecasefile)
+    # Use case file can be remote, so we need to provide authentication information
+    usecase = getJsonFromFile(btpUsecase.usecasefile, btpUsecase.externalConfigAuthMethod, btpUsecase.externalConfigUserName, btpUsecase.externalConfigPassword, btpUsecase.externalConfigToken)
     items = []
     if usecase.get("services") is not None:
         for service in usecase.get("services"):

libs/python/helperServices.py

@@ -91,9 +91,10 @@ def default(self, o):
 def readAllServicesFromUsecaseFile(btpUsecase):
     # Initiate class with configured parameters
     jsonSchema = FOLDER_SCHEMA_LIBS + "btpsa-usecase.json"
+    # local access no auth needed
     paramDefinitionServices = getJsonFromFile(jsonSchema)
 
-    usecase = getJsonFromFile(btpUsecase.usecasefile)
+    usecase = getJsonFromFile(filename=btpUsecase.usecasefile, externalConfigAuthMethod=btpUsecase.externalConfigAuthMethod, externalConfigUserName=btpUsecase.externalConfigUserName, externalConfigPassword=btpUsecase.externalConfigPassword, externalConfigToken=btpUsecase.externalConfigToken)
     items = []
     if "services" in usecase:
         for usecaseService in usecase.get("services"):

libs/python/helperYaml.py

@@ -7,6 +7,7 @@
 def build_and_store_service_instance_yaml_from_parameters(service, yamlFilePath):
 
     templatePath = FOLDER_K8S_YAML_TEMPLATES + 'K8s-SERVICE-INSTANCE.json'
+    # local access - no auth needed
     serviceInstanceTemplate = getJsonFromFile(templatePath)
 
     serviceInstanceTemplate["metadata"]["name"] = service.instancename
@@ -31,6 +32,7 @@ def build_and_store_service_instance_yaml_from_parameters(service, yamlFilePath)
 def build_and_store_service_binding_yaml_from_parameters(keyname, service, yamlFilePath, keyLabels):
 
     templatePath = FOLDER_K8S_YAML_TEMPLATES + 'K8s-SERVICE-BINDING.json'
+    # local access no auth needed
     serviceBindingTemplate = getJsonFromFile(templatePath)
 
     serviceBindingTemplate["metadata"]["name"] = keyname