diff --git a/.github/workflows/build-test-linux.yml b/.github/workflows/build-test-linux.yml
index 5718e96642..a6aaac4132 100644
--- a/.github/workflows/build-test-linux.yml
+++ b/.github/workflows/build-test-linux.yml
@@ -140,6 +140,8 @@ jobs:
         python -m pip install -r requirements.txt
         cd dynamo
         python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml -n 4 conversion/
+        python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml automatic_plugin/test_automatic_plugin.py
+        python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml automatic_plugin/test_automatic_plugin_with_attrs.py
         popd
 
   tests-py-dynamo-fe:
diff --git a/tests/py/dynamo/automatic_plugin/__init__.py b/tests/py/dynamo/automatic_plugin/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tests/py/dynamo/conversion/test_automatic_plugin.py b/tests/py/dynamo/automatic_plugin/test_automatic_plugin.py
similarity index 98%
rename from tests/py/dynamo/conversion/test_automatic_plugin.py
rename to tests/py/dynamo/automatic_plugin/test_automatic_plugin.py
index e843686f9f..ae60f8cda7 100644
--- a/tests/py/dynamo/conversion/test_automatic_plugin.py
+++ b/tests/py/dynamo/automatic_plugin/test_automatic_plugin.py
@@ -8,7 +8,7 @@
 from parameterized import parameterized
 from torch.testing._internal.common_utils import run_tests
 
-from .harness import DispatchTestCase
+from ..conversion.harness import DispatchTestCase
 
 
 @triton.jit
diff --git a/tests/py/dynamo/conversion/test_automatic_plugin_with_attrs.py b/tests/py/dynamo/automatic_plugin/test_automatic_plugin_with_attrs.py
similarity index 98%
rename from tests/py/dynamo/conversion/test_automatic_plugin_with_attrs.py
rename to tests/py/dynamo/automatic_plugin/test_automatic_plugin_with_attrs.py
index da0d6bfdfb..6d82c15ea2 100644
--- a/tests/py/dynamo/conversion/test_automatic_plugin_with_attrs.py
+++ b/tests/py/dynamo/automatic_plugin/test_automatic_plugin_with_attrs.py
@@ -8,7 +8,7 @@
 from parameterized import parameterized
 from torch.testing._internal.common_utils import run_tests
 
-from .harness import DispatchTestCase
+from ..conversion.harness import DispatchTestCase
 
 
 @triton.jit