diff --git a/src/PowerShellEditorServices/Language/LanguageService.cs b/src/PowerShellEditorServices/Language/LanguageService.cs
index 70979cd2c..e30c00b8a 100644
--- a/src/PowerShellEditorServices/Language/LanguageService.cs
+++ b/src/PowerShellEditorServices/Language/LanguageService.cs
@@ -13,6 +13,7 @@
 using System.Management.Automation;
 using System.Management.Automation.Language;
 using System.Runtime.InteropServices;
+using System.Security;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -346,9 +347,14 @@ public async Task<FindReferencesResult> FindReferencesOfSymbol(
                     {
                         scriptFile = workspace.GetFile(file);
                     }
-                    catch (IOException)
+                    catch (Exception e) when (e is IOException
+                                           || e is SecurityException
+                                           || e is FileNotFoundException
+                                           || e is DirectoryNotFoundException
+                                           || e is PathTooLongException
+                                           || e is UnauthorizedAccessException)
                     {
-                        // If the file has ceased to exist for some reason, we just skip it
+                        // If we can't access the file for some reason, just ignore it
                         continue;
                     }