Merge pull request #39 from l-d-j/submodules

Allow custom submodules

Author: vrutkovs

README.md

@@ -70,8 +70,6 @@ You can implement any of the methods that are typically used with Behave inside
 1. Clone repositories
 ```
 git clone https://github.com/Containers-Testing-Framework/ctf-cli.git
-git clone https://github.com/Containers-Testing-Framework/common-steps.git
-git clone https://github.com/Containers-Testing-Framework/common-features.git
 ```
 1. Optional: clone an example repository
 ```
@@ -90,10 +88,14 @@ cd ctf-cli
 cp ctf.conf.sample ctf.conf
 cp tests.conf.sample tests.conf
 ```
-1. Change to project directory and run tests
+1. Change to project directory and get common features
 ```
 cd example-project-postgresql
-../ctf-cli/ctf-cli.py
+../ctf-cli/ctf-cli.py remote add features https://github.com/Containers-Testing-Framework/common-features.git
+```
+1. Run tests
+```
+../ctf-cli/ctf-cli.py run
 ```
 
 ## CLI tool

circle.yml

@@ -15,6 +15,8 @@ test:
   override:
     - mkdir test_dir
     - cd test_dir && COVERAGE_FILE=../.coverage.init coverage run ../ctf-cli.py init
+    - cd test_dir && COVAREGE_FILE=../.coverage.remote_add coverage run ../ctf-cli.py remote add steps https://github.com/Containers-Testing-Framework/common-steps.git
+    - cd test_dir && COVAREGE_FILE=../.coverage.remote_add coverage run ../ctf-cli.py remote add features https://github.com/Containers-Testing-Framework/common-features.git
     - cd test_dir && COVERAGE_FILE=../.coverage.update coverage run ../ctf-cli.py update
     - cd test_dir && COVERAGE_FILE=../.coverage.run coverage run ../ctf-cli.py run
     - COVERAGE_FILE=.coverage.tests nosetests
@@ -23,4 +25,4 @@ test:
 
 notify:
   webhooks:
-    - url: https://webhooks.gitter.im/e/92b168ffacdc38a6facd
+    - url: https://webhooks.gitter.im/e/92b168ffacdc38a6facd

ctf_cli/application.py

@@ -17,6 +17,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+import shutil
 import sys
 from subprocess import check_call, CalledProcessError
 
@@ -99,23 +100,7 @@ def init(self):
             with open(steps_py_file, "w") as f:
                 f.write(common_steps_py_content)
             check_call("git add %s" % steps_py_file, shell=True)
-
-        # Add common-features and common-steps as submodules
-        # TODO:  make this generic when a different type of container is specified
-        common_features_dir = os.path.join(features_dir, "common-features")
-        if os.path.exists(common_features_dir):
-            logger.info("Directory tests/features/common-features already exists")
-        else:
-            logger.info("Adding tests/features/common-features as a submodule")
-            check_call('git submodule add https://github.com/Containers-Testing-Framework/common-features.git tests/features/common-features', shell=True)
-
-        common_steps_dir = os.path.join(steps_dir, "common_steps")
-        if os.path.exists(common_steps_dir):
-            logger.info("Directory tests/steps/common_steps already exists")
-        else:
-            logger.info("Adding tests/steps/common_steps as a submodule")
-            check_call('git submodule add https://github.com/Containers-Testing-Framework/common-steps.git tests/steps/common_steps', shell=True)
-
+            
         # Copy sample configuration
         ctf_conf_file = os.path.join(self._execution_dir_path, "ctf.conf")
         if os.path.exists(ctf_conf_file):
@@ -127,6 +112,42 @@ def init(self):
                 f.write(sample_ctl_ctf_config)
             check_call("git add %s" % ctf_conf_file, shell=True)
 
+    def add_remote(self):
+        if 'feature' in self._cli_conf.get(CTFCliConfig.GLOBAL_SECTION_NAME, CTFCliConfig.CONFIG_REMOTE_TYPE):
+            self.add_remote_feature()
+        else:
+            self.add_remote_step()
+
+    def add_remote_feature(self):
+        project = self._cli_conf.get(CTFCliConfig.GLOBAL_SECTION_NAME, CTFCliConfig.CONFIG_REMOTE_PROJECT)
+        if project is None:
+            path = "tests/features/"
+        else:
+            path = "tests/features/" + project
+        self.add_submodule(path)
+
+    def add_remote_step(self):
+        project = self._cli_conf.get(CTFCliConfig.GLOBAL_SECTION_NAME, CTFCliConfig.CONFIG_REMOTE_PROJECT)
+        if project is None:
+            path = "tests/steps/"
+        else:
+            path = "tests/steps/" + project
+        self.add_submodule(path)
+
+    def list_remotes(self):
+        check_call("git submodule foreach 'git config --get remote.origin.url'", shell=True)
+
+    def add_submodule(self, path):
+        url = self._cli_conf.get(CTFCliConfig.GLOBAL_SECTION_NAME, CTFCliConfig.CONFIG_REMOTE_URL)
+        dirname = os.path.splitext(os.path.basename(url))[0].replace('-', '_')
+        check_call('git submodule add %s %s' % (url, path + "/" + dirname), shell=True)
+
+    def remove_remote(self):
+        name = self._cli_conf.get(CTFCliConfig.GLOBAL_SECTION_NAME, CTFCliConfig.CONFIG_REMOTE_NAME)
+        check_call('git submodule deinit -f %s' %name, shell=True)
+        check_call('git config -f .gitmodules --remove-section "submodule.%s"' %name, shell=True)
+        shutil.rmtree(name)
+
     def run(self):
         """
         The main application execution method
@@ -163,15 +184,6 @@ def update(self):
         """
         Update app submodules
         """
-        logger.info("Updating Containers Testing Framework common steps and features")
-
-        common_steps_dir = os.path.join(self._execution_dir_path, "tests", "steps", "common_steps")
-        common_features_dir = os.path.join(self._execution_dir_path, "tests", "features", "common-features")
-        for directory in [common_steps_dir, common_features_dir]:
-            logger.info("Updating %s", directory)
-            check_call("git fetch origin", shell=True, cwd=directory)
-            check_call("git checkout origin/master", shell=True, cwd=directory)
-
-        # Check that steps are not contradicting with each other
-        logger.info("Checking project steps sanity")
-        check_call("behave tests -d", shell=True)
+        logger.info("Updating remote test and feautres")
+
+        check_call("git submodule foreach git pull --rebase", shell=True)

ctf_cli/arguments_parser.py

@@ -25,62 +25,101 @@ class ArgumentsParser(object):
     def __init__(self, args=None):
         """ parse arguments """
         self.parser = argparse.ArgumentParser(description='CLI for running Containers Testing Framework')
+        self.subparsers = self.parser.add_subparsers(dest="cli_action")
         self.add_args()
+        self.add_remote_subparser()
+        self.add_init_subparser()
+        self.add_run_subparser()
+        self.add_update_subparser()
         self.args = self.parser.parse_args(args)
 
-    def add_args(self):
-        self.parser.add_argument(
-            dest='cli_action',
-            choices=['init', 'run', 'update'],
-            default=['init', 'run'],
-            nargs='?',
-            help="Action to perform (default - init and run)"
-        )
-        self.parser.add_argument(
-            "-v",
-            "--verbose",
-            default=False,
-            action="store_true",
-            help="Output is more verbose (recommended)"
-        )
-        self.parser.add_argument(
+    def add_remote_add_subparser(self, subparser):
+        subparser.add_argument(
+            dest='remote_type',
+            choices=['steps', 'features'],
+            )
+
+        subparser.add_argument(
+            dest='url',
+            help='module url'
+            )
+        
+        subparser.add_argument(
+            "--project",
+            dest='project',
+            help="name of test project")
+
+    def add_remote_remove_subparser(self, subparser):
+        subparser.add_argument(
+            dest='name'
+            )
+
+    def add_remote_subparser(self):
+        remote_subparser=self.subparsers.add_parser('remote', help='addidng/removing test suites')
+        remote_oper_subparser=remote_subparser.add_subparsers(dest='remote_action')
+        self.add_remote_add_subparser(remote_oper_subparser.add_parser('add', help='add remote repository'))
+        self.add_remote_remove_subparser(remote_oper_subparser.add_parser('remove', help='remove remote repository'))
+        remote_oper_subparser.add_parser('list', help='list remote repositories')
+
+    def add_run_subparser(self):
+        run_subparser=self.subparsers.add_parser('run', help="run test suite - default")
+        run_subparser.add_argument(
             "-c",
             "--cli-config",
             default=None,
             dest='cli_config_path',
             help="Path to CLI configuration file (By default use only CLI arguments and default values)"
         )
-        self.parser.add_argument(
+        run_subparser.add_argument(
             "-t",
             "--tests-config",
             default=None,
             dest='tests_config_path',
             help="Path to tests configuration file. By default it will be searched for in test/ dir"
         )
-        self.parser.add_argument(
+        run_subparser.add_argument(
             "-f",
             "--dockerfile",
             default=None,
             dest='dockerfile',
             help="Path to Dockerfile to use. If not passed, will be searched for in the current directory"
         )
-        self.parser.add_argument(
+        run_subparser.add_argument(
             "-i",
             "--image",
             default=None,
             dest='image',
             help="Image to use for testing. If not passed, the image will be built from the Dockerfile."
         )
-        self.parser.add_argument(
+        run_subparser.add_argument(
             "-j",
             "--junit",
             default=None,
             dest='junit',
             help="Junit folder to store results. If not passed junit reports will not be generated"
         )
 
+    def add_update_subparser(self):
+        self.subparsers.add_parser('update', help="update suites")
+
+    def add_init_subparser(self):
+        self.subparsers.add_parser('init', help="update suites")
+
+    def add_args(self):
+        self.parser.add_argument(
+            "-v",
+            "--verbose",
+            default=False,
+            action="store_true",
+            help="Output is more verbose (recommended)"
+        )
+
+
     def __getattr__(self, name):
         try:
             return getattr(self.args, name)
         except AttributeError:
-            return object.__getattribute__(self, name)
+            try:
+                return object.__getattribute__(self, name)
+            except AttributeError:
+                return None

ctf_cli/behave.py

@@ -272,7 +272,7 @@ def _write_environment_py(self):
         """
         project_env_py = glob.glob(os.path.join(self._working_dir, '*environment.py'))
         if project_env_py:
-            module_name = os.path.basename(project_env_py[0]).replace('.py', '')
+            module_name = os.path.basename(project_env_py[0]).replace('.py', '').replace('-', '_')
             import_statement = 'from {0} import *\n'.format(module_name)
             import_statement += 'import {0}'.format(module_name)
         else:

ctf_cli/cli_runner.py

@@ -35,9 +35,7 @@ def run():
         try:
             # add application-wide debug log
             LoggerHelper.add_debug_log_file(os.getcwd())
-
             args = ArgumentsParser(sys.argv[1:])
-
             if args.verbose is True:
                 LoggerHelper.add_stream_handler(logger,
                                                 logging.Formatter('%(levelname)s:\t%(message)s'),
@@ -54,6 +52,14 @@ def run():
                 app.run()
             if 'update' in args.cli_action:
                 app.update()
+            if 'remote' in args.cli_action:
+                if 'add' in args.remote_action:
+                    app.add_remote()
+                if 'remove' in args.remote_action:
+                    app.remove_remote()
+                if 'list' in args.remote_action:
+                    app.list_remotes()
+
         except KeyboardInterrupt:
             logger.info('Interrupted by user')
         except CTFCliError as e:

ctf_cli/config.py

@@ -38,6 +38,10 @@ class CTFCliConfig(object):
     CONFIG_IMAGE = 'Image'
     CONFIG_JUNIT = 'Junit'
     CONFIG_EXEC_TYPE = 'ExecType'
+    CONFIG_REMOTE_TYPE = 'remote_type'
+    CONFIG_REMOTE_URL = 'url'
+    CONFIG_REMOTE_PROJECT = 'project'
+    CONFIG_REMOTE_NAME = 'name'
 
     ANSIBLE_SECTION_NAME = 'ansible'
     CONFIG_ANSIBLE_HOST = 'Host'
@@ -99,8 +103,11 @@ def _add_commandline_arguments(self, cli_conf):
             self.CONFIG_IMAGE: cli_conf.image,
             self.CONFIG_JUNIT: cli_conf.junit,
             self.CONFIG_EXEC_TYPE: 'ansible',
+            self.CONFIG_REMOTE_TYPE: cli_conf.remote_type,
+            self.CONFIG_REMOTE_URL: cli_conf.url,
+            self.CONFIG_REMOTE_PROJECT: cli_conf.project,
+            self.CONFIG_REMOTE_NAME: cli_conf.name,
         }}
-
         self.config_parser_read_dict(self._config, cli_settings)
 
     def __getattr__(self, name):
@@ -133,4 +140,4 @@ def config_parser_read_dict(config_parser_obj, dictionary):
             # go through all key: value in the section
             for option, value in six.iteritems(conf):
                 if config_parser_obj.has_option(section, option) is False:
-                    config_parser_obj.set(section, option, value)
+                    config_parser_obj.set(section, option, value)