fix include functionality

When using --include on files, the directories get excluded.
removing those ressource end up in removing also the
nested files

The include exclude algorithm makes it quite difficult to support
ignoring a whole directory by just saying ignore=path/to/dir

In this version, we support that usecase with ignore=path/to/dir/*

Signed-off-by: Pierre Tardy <[email protected]>

Author: tardyp

src/scancode/plugin_ignore.py

@@ -112,6 +112,8 @@ def process_codebase(self, codebase, ignore=(), include=(), **kwargs):
             resource_rid = resource.rid
             if resource.is_root:
                 continue
+            if resource.is_dir:  # removing dirs will also remove its files
+                continue
             if resource_rid in rids_to_remove:
                 rids_to_remove_discard(resource_rid)
                 remove_resource(resource)

tests/scancode/test_plugin_ignore.py

@@ -47,10 +47,10 @@ def test_is_included_glob_file(self):
         excludes = {'*.txt': 'test ignore'}
         assert not is_included(location, excludes=excludes)
 
-    def check_ProcessIgnore(self, test_dir, expected, ignore):
+    def check_ProcessIgnore(self, test_dir, expected, ignore, include=()):
         codebase = Codebase(test_dir, strip_root=True)
         test_plugin = ProcessIgnore()
-        test_plugin.process_codebase(codebase, ignore=ignore)
+        test_plugin.process_codebase(codebase, ignore=ignore, include=include)
         resources = [res.path for res in codebase.walk(skip_root=True)]
         assert sorted(resources) == expected
 
@@ -91,12 +91,13 @@ def test_ProcessIgnore_with_glob_for_extension(self):
 
     def test_ProcessIgnore_with_glob_for_path(self):
         test_dir = self.extract_test_tar('plugin_ignore/user.tgz')
-        ignore = ('*/src/test',)
+        ignore = ('*/src/test/*',)
         expected = [
             'user',
             'user/ignore.doc',
             'user/src',
-            'user/src/ignore.doc'
+            'user/src/ignore.doc',
+            'user/src/test' # test is included, but no file inside
         ]
         self.check_ProcessIgnore(test_dir, expected, ignore)
 
@@ -110,6 +111,19 @@ def test_ProcessIgnore_with_multiple_ignores(self):
         ]
         self.check_ProcessIgnore(test_dir, expected, ignore)
 
+    def test_ProcessIgnore_include_with_glob_for_extension(self):
+        test_dir = self.extract_test_tar('plugin_ignore/user.tgz')
+        include = ('*.doc',)
+        expected = [
+            'user',
+            'user/ignore.doc',
+            'user/src',
+            'user/src/ignore.doc',
+            'user/src/test',
+            'user/src/test/sample.doc',
+        ]
+        self.check_ProcessIgnore(test_dir, expected, ignore=(), include=include)
+
     def test_ProcessIgnore_process_codebase_does_not_fail_to_access_an_ignored_resourced_cached_to_disk(self):
         test_dir = self.extract_test_tar('plugin_ignore/user.tgz')
         codebase = Codebase(test_dir, max_in_memory=1)
@@ -217,12 +231,12 @@ def test_scancode_ignore_glob_path(self):
     def test_scancode_multiple_ignores(self):
         test_dir = self.extract_test_tar('plugin_ignore/user.tgz')
         result_file = self.get_temp_file('json')
-        args = ['--copyright', '--strip-root', '--ignore', '*/src/test', '--ignore', '*.doc', test_dir, '--json', result_file]
+        args = ['--copyright', '--strip-root', '--ignore', '*/src/test/*', '--ignore', '*.doc', test_dir, '--json', result_file]
         run_scan_click(args)
         scan_result = load_json_result(result_file)
         assert scan_result['headers'][0]['extra_data']['files_count'] == 0
         scan_locs = [x['path'] for x in scan_result['files']]
-        assert scan_locs == [u'user', u'user/src']
+        assert scan_locs == [u'user', u'user/src', u'user/src/test']
 
     def test_scancode_codebase_attempt_to_access_an_ignored_resourced_cached_to_disk(self):
         test_dir = self.extract_test_tar('plugin_ignore/user.tgz')
@@ -237,5 +251,6 @@ def test_scancode_codebase_attempt_to_access_an_ignored_resourced_cached_to_disk
             u'user/ignore.doc',
             u'user/src',
             u'user/src/ignore.doc',
+            u'user/src/test',
         ]
         assert scan_locs == expected