Add support for unstable unit tests (#382)

* Added pytest marks for unstable tests

* Added new build function to call unstable tests

Author: Elliot Boschwitz

build.py

@@ -126,10 +126,23 @@ def unit_test():
     """
     runid = str(uuid.uuid1())
     python_version = platform.python_version()
-    utility.exec_command(
-        'pytest -l --cov mssqlcli --doctest-modules --junitxml=junit/test-{}-results.xml \
-            --cov-report=xml --cov-report=html --cov-append '
-        '-o junit_suite_name=pytest-{} '
+    utility.exec_command(('pytest -l --cov mssqlcli --doctest-modules '
+                          '--junitxml=junit/test-{}-results.xml --cov-report=xml '
+                          '--cov-report=html --cov-append -o junit_suite_name=pytest-{} '
+                          '-s -m "not unstable" {}').format(runid, python_version,
+                                                            get_active_test_filepaths()),
+                         utility.ROOT_DIR, continue_on_error=False)
+
+def unstable_unit_test():
+    """
+    Run all unstable unit tests.
+    """
+    utility.exec_command('pytest -s -v -m unstable {}'.format(get_active_test_filepaths()),
+                         utility.ROOT_DIR, continue_on_error=False)
+
+
+def get_active_test_filepaths():
+    return (
         'tests/test_mssqlcliclient.py '
         'tests/test_completion_refresher.py '
         'tests/test_config.py '
@@ -143,10 +156,9 @@ def unit_test():
         'tests/test_telemetry.py '
         'tests/test_localization.py '
         'tests/test_globalization.py '
-        '-s tests/test_noninteractive_mode.py '# -s disables capturing--test doesn't work without it
-        'tests/test_special.py'.format(runid, python_version),
-        utility.ROOT_DIR,
-        continue_on_error=False)
+        'tests/test_noninteractive_mode.py '
+        'tests/test_special.py'
+    )
 
 
 def integration_test():
@@ -223,6 +235,7 @@ def create_dir(path):
         'build': build,
         'validate_package': validate_package,
         'unit_test': unit_test,
+        'unstable_unit_test': unstable_unit_test,
         'integration_test': integration_test,
         'test': test
     }

tests/jsonrpc/test_jsonrpcclient.py

@@ -2,7 +2,7 @@
 import time
 import io
 import threading
-
+import pytest
 import mssqlcli.jsonrpc.jsonrpcclient as json_rpc_client
 
 
@@ -170,6 +170,7 @@ def test_send_invalid_request(self):
         with self.assertRaises(ValueError):
             test_client.submit_request(None, None)
 
+    @pytest.mark.unstable
     def test_receive_invalid_response_exception(self):
         """
             Verify invalid response has exception queued and response thread dies.

tests/test_globalization.py

@@ -172,6 +172,7 @@ def test_db():
         yield db
         clean_up_test_db(db)
 
+    @pytest.mark.unstable
     def test_schema_metadata_double(self, test_db):
         self.run_schema_metadata_validation('double', test_db)
 

tests/test_mssqlcliclient.py

@@ -119,6 +119,7 @@ def test_get_query_results(client):
             assert query == test_query
 
     @staticmethod
+    @pytest.mark.unstable
     def test_schema_table_views_and_columns_query(client):
         """
             Verify mssqlcliclient's tables, views, columns, and schema are populated.
@@ -160,6 +161,7 @@ def drop_entities(mssqlcli_client):
             drop_entities(client)
 
     @staticmethod
+    @pytest.mark.unstable
     def test_stored_proc_multiple_result_sets(client):
         """
             Verify the results of running a stored proc with multiple result sets

utility.py

@@ -2,6 +2,7 @@
 from subprocess import check_call, CalledProcessError
 import os
 import platform
+import shlex
 import shutil
 import sys
 import string
@@ -21,7 +22,12 @@ def exec_command(command, directory, continue_on_error=True):
         Execute command.
     """
     try:
-        check_call(command.split(), cwd=directory)
+        command_split = [token.strip('"') for token in shlex.split(command, posix=False)]
+        # The logic above is used to preserve multiple token arguments with pytest. It is
+        # specifically needed when calling "not unstable" for running all tests not marked
+        # as unstable.
+
+        check_call(command_split, cwd=directory)
     except CalledProcessError as err:
         # Continue execution in scenarios where we may be bulk command execution.
         print(err, file=sys.stderr)