Apply changes from bundled appdirs to vendored

* Convert Windows app data directory values to bytes on Python 2, so the
  output type is consistent across platforms (#4000)
* Also look in ~/.config for user config on macOS (#4100)
* Remove pywin32 dependency, only use ctypes and winreg for directory
  lookup on Windows (#2467)
* Always use os.path.join() instead of os.sep.join() so cross-platform
  tests work as expected (#3275)

Author: uranusjr

src/pip/_internal/utils/appdirs.py

@@ -1,19 +1,17 @@
 """
-This code was taken from https://github.com/ActiveState/appdirs and modified
-to suit our purposes.
-"""
+This code wraps the vendored appdirs module to so the return values are
+compatible for the current pip code base.
 
-# The following comment should be removed at some point in the future.
-# mypy: disallow-untyped-defs=False
+The intention is to rewrite current usages guradually, keeping the tests pass,
+and eventually drop this after all usages are changed.
+"""
 
 from __future__ import absolute_import
 
 import os
-import sys
 
-from pip._vendor.six import PY2, text_type
+from pip._vendor import appdirs as _appdirs
 
-from pip._internal.utils.compat import WINDOWS, expanduser
 from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 
 if MYPY_CHECK_RUNNING:
@@ -22,255 +20,22 @@
 
 def user_cache_dir(appname):
     # type: (str) -> str
-    r"""
-    Return full path to the user-specific cache dir for this application.
-
-        "appname" is the name of application.
-
-    Typical user cache directories are:
-        macOS:      ~/Library/Caches/<AppName>
-        Unix:       ~/.cache/<AppName> (XDG default)
-        Windows:    C:\Users\<username>\AppData\Local\<AppName>\Cache
-
-    On Windows the only suggestion in the MSDN docs is that local settings go
-    in the `CSIDL_LOCAL_APPDATA` directory. This is identical to the
-    non-roaming app data dir (the default returned by `user_data_dir`). Apps
-    typically put cache data somewhere *under* the given dir here. Some
-    examples:
-        ...\Mozilla\Firefox\Profiles\<ProfileName>\Cache
-        ...\Acme\SuperApp\Cache\1.0
-
-    OPINION: This function appends "Cache" to the `CSIDL_LOCAL_APPDATA` value.
-    """
-    if WINDOWS:
-        # Get the base path
-        path = os.path.normpath(_get_win_folder("CSIDL_LOCAL_APPDATA"))
-
-        # When using Python 2, return paths as bytes on Windows like we do on
-        # other operating systems. See helper function docs for more details.
-        if PY2 and isinstance(path, text_type):
-            path = _win_path_to_bytes(path)
-
-        # Add our app name and Cache directory to it
-        path = os.path.join(path, appname, "Cache")
-    elif sys.platform == "darwin":
-        # Get the base path
-        path = expanduser("~/Library/Caches")
-
-        # Add our app name to it
-        path = os.path.join(path, appname)
-    else:
-        # Get the base path
-        path = os.getenv("XDG_CACHE_HOME", expanduser("~/.cache"))
-
-        # Add our app name to it
-        path = os.path.join(path, appname)
-
-    return path
-
-
-def user_data_dir(appname, roaming=False):
-    # type: (str, bool) -> str
-    r"""
-    Return full path to the user-specific data dir for this application.
-
-        "appname" is the name of application.
-            If None, just the system directory is returned.
-        "roaming" (boolean, default False) can be set True to use the Windows
-            roaming appdata directory. That means that for users on a Windows
-            network setup for roaming profiles, this user data will be
-            sync'd on login. See
-            <http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx>
-            for a discussion of issues.
-
-    Typical user data directories are:
-        macOS:                  ~/Library/Application Support/<AppName>
-                                if it exists, else ~/.config/<AppName>
-        Unix:                   ~/.local/share/<AppName>    # or in
-                                $XDG_DATA_HOME, if defined
-        Win XP (not roaming):   C:\Documents and Settings\<username>\ ...
-                                ...Application Data\<AppName>
-        Win XP (roaming):       C:\Documents and Settings\<username>\Local ...
-                                ...Settings\Application Data\<AppName>
-        Win 7  (not roaming):   C:\\Users\<username>\AppData\Local\<AppName>
-        Win 7  (roaming):       C:\\Users\<username>\AppData\Roaming\<AppName>
-
-    For Unix, we follow the XDG spec and support $XDG_DATA_HOME.
-    That means, by default "~/.local/share/<AppName>".
-    """
-    if WINDOWS:
-        const = roaming and "CSIDL_APPDATA" or "CSIDL_LOCAL_APPDATA"
-        path = os.path.join(os.path.normpath(_get_win_folder(const)), appname)
-    elif sys.platform == "darwin":
-        path = os.path.join(
-            expanduser('~/Library/Application Support/'),
-            appname,
-        ) if os.path.isdir(os.path.join(
-            expanduser('~/Library/Application Support/'),
-            appname,
-        )
-        ) else os.path.join(
-            expanduser('~/.config/'),
-            appname,
-        )
-    else:
-        path = os.path.join(
-            os.getenv('XDG_DATA_HOME', expanduser("~/.local/share")),
-            appname,
-        )
-
-    return path
+    return _appdirs.user_cache_dir(appname, appauthor=False)
 
 
 def user_config_dir(appname, roaming=True):
     # type: (str, bool) -> str
-    """Return full path to the user-specific config dir for this application.
-
-        "appname" is the name of application.
-            If None, just the system directory is returned.
-        "roaming" (boolean, default True) can be set False to not use the
-            Windows roaming appdata directory. That means that for users on a
-            Windows network setup for roaming profiles, this user data will be
-            sync'd on login. See
-            <http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx>
-            for a discussion of issues.
-
-    Typical user data directories are:
-        macOS:                  same as user_data_dir
-        Unix:                   ~/.config/<AppName>
-        Win *:                  same as user_data_dir
+    return _appdirs.user_config_dir(appname, appauthor=False, roaming=roaming)
 
-    For Unix, we follow the XDG spec and support $XDG_CONFIG_HOME.
-    That means, by default "~/.config/<AppName>".
-    """
-    if WINDOWS:
-        path = user_data_dir(appname, roaming=roaming)
-    elif sys.platform == "darwin":
-        path = user_data_dir(appname)
-    else:
-        path = os.getenv('XDG_CONFIG_HOME', expanduser("~/.config"))
-        path = os.path.join(path, appname)
 
-    return path
+def user_data_dir(appname, roaming=False):
+    # type: (str, bool) -> str
+    return _appdirs.user_data_dir(appname, appauthor=False, roaming=roaming)
 
 
-# for the discussion regarding site_config_dirs locations
-# see <https://github.com/pypa/pip/issues/1733>
 def site_config_dirs(appname):
     # type: (str) -> List[str]
-    r"""Return a list of potential user-shared config dirs for this application.
-
-        "appname" is the name of application.
-
-    Typical user config directories are:
-        macOS:      /Library/Application Support/<AppName>/
-        Unix:       /etc or $XDG_CONFIG_DIRS[i]/<AppName>/ for each value in
-                    $XDG_CONFIG_DIRS
-        Win XP:     C:\Documents and Settings\All Users\Application ...
-                    ...Data\<AppName>\
-        Vista:      (Fail! "C:\ProgramData" is a hidden *system* directory
-                    on Vista.)
-        Win 7:      Hidden, but writeable on Win 7:
-                    C:\ProgramData\<AppName>\
-    """
-    if WINDOWS:
-        path = os.path.normpath(_get_win_folder("CSIDL_COMMON_APPDATA"))
-        pathlist = [os.path.join(path, appname)]
-    elif sys.platform == 'darwin':
-        pathlist = [os.path.join('/Library/Application Support', appname)]
-    else:
-        # try looking in $XDG_CONFIG_DIRS
-        xdg_config_dirs = os.getenv('XDG_CONFIG_DIRS', '/etc/xdg')
-        if xdg_config_dirs:
-            pathlist = [
-                os.path.join(expanduser(x), appname)
-                for x in xdg_config_dirs.split(os.pathsep)
-            ]
-        else:
-            pathlist = []
-
-        # always look in /etc directly as well
-        pathlist.append('/etc')
-
-    return pathlist
-
-
-# -- Windows support functions --
-
-def _get_win_folder_from_registry(csidl_name):
-    # type: (str) -> str
-    """
-    This is a fallback technique at best. I'm not sure if using the
-    registry for this guarantees us the correct answer for all CSIDL_*
-    names.
-    """
-    import _winreg
-
-    shell_folder_name = {
-        "CSIDL_APPDATA": "AppData",
-        "CSIDL_COMMON_APPDATA": "Common AppData",
-        "CSIDL_LOCAL_APPDATA": "Local AppData",
-    }[csidl_name]
-
-    key = _winreg.OpenKey(
-        _winreg.HKEY_CURRENT_USER,
-        r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
-    )
-    directory, _type = _winreg.QueryValueEx(key, shell_folder_name)
-    return directory
-
-
-def _get_win_folder_with_ctypes(csidl_name):
-    # type: (str) -> str
-    # On Python 2, ctypes.create_unicode_buffer().value returns "unicode",
-    # which isn't the same as str in the annotation above.
-    csidl_const = {
-        "CSIDL_APPDATA": 26,
-        "CSIDL_COMMON_APPDATA": 35,
-        "CSIDL_LOCAL_APPDATA": 28,
-    }[csidl_name]
-
-    buf = ctypes.create_unicode_buffer(1024)
-    windll = ctypes.windll  # type: ignore
-    windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf)
-
-    # Downgrade to short path name if have highbit chars. See
-    # <http://bugs.activestate.com/show_bug.cgi?id=85099>.
-    has_high_char = False
-    for c in buf:
-        if ord(c) > 255:
-            has_high_char = True
-            break
-    if has_high_char:
-        buf2 = ctypes.create_unicode_buffer(1024)
-        if windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024):
-            buf = buf2
-
-    # The type: ignore is explained under the type annotation for this function
-    return buf.value  # type: ignore
-
-
-if WINDOWS:
-    try:
-        import ctypes
-        _get_win_folder = _get_win_folder_with_ctypes
-    except ImportError:
-        _get_win_folder = _get_win_folder_from_registry
-
-
-def _win_path_to_bytes(path):
-    """Encode Windows paths to bytes. Only used on Python 2.
-
-    Motivation is to be consistent with other operating systems where paths
-    are also returned as bytes. This avoids problems mixing bytes and Unicode
-    elsewhere in the codebase. For more details and discussion see
-    <https://github.com/pypa/pip/issues/3463>.
-
-    If encoding using ASCII and MBCS fails, return the original Unicode path.
-    """
-    for encoding in ('ASCII', 'MBCS'):
-        try:
-            return path.encode(encoding)
-        except (UnicodeEncodeError, LookupError):
-            pass
-    return path
+    dirval = _appdirs.site_config_dir(appname, appauthor=False, multipath=True)
+    if _appdirs.system == "linux2":
+        return dirval.split(os.pathsep)
+    return [dirval]

src/pip/_vendor/appdirs.py

@@ -64,7 +64,7 @@ def user_data_dir(appname=None, appauthor=None, version=None, roaming=False):
             for a discussion of issues.
 
     Typical user data directories are:
-        Mac OS X:               ~/Library/Application Support/<AppName>
+        Mac OS X:               ~/Library/Application Support/<AppName>  # or ~/.config/<AppName>, if the other does not exist
         Unix:                   ~/.local/share/<AppName>    # or in $XDG_DATA_HOME, if defined
         Win XP (not roaming):   C:\Documents and Settings\<username>\Application Data\<AppAuthor>\<AppName>
         Win XP (roaming):       C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName>
@@ -88,6 +88,10 @@ def user_data_dir(appname=None, appauthor=None, version=None, roaming=False):
         path = os.path.expanduser('~/Library/Application Support/')
         if appname:
             path = os.path.join(path, appname)
+        if not os.path.isdir(path):
+            path = os.path.expanduser('~/.config/')
+            if appname:
+                path = os.path.join(path, appname)
     else:
         path = os.getenv('XDG_DATA_HOME', os.path.expanduser("~/.local/share"))
         if appname:
@@ -150,7 +154,7 @@ def site_data_dir(appname=None, appauthor=None, version=None, multipath=False):
         if appname:
             if version:
                 appname = os.path.join(appname, version)
-            pathlist = [os.sep.join([x, appname]) for x in pathlist]
+            pathlist = [os.path.join(x, appname) for x in pathlist]
 
         if multipath:
             path = os.pathsep.join(pathlist)
@@ -203,6 +207,8 @@ def user_config_dir(appname=None, appauthor=None, version=None, roaming=False):
     return path
 
 
+# for the discussion regarding site_config_dir locations
+# see <https://github.com/pypa/pip/issues/1733>
 def site_config_dir(appname=None, appauthor=None, version=None, multipath=False):
     r"""Return full path to the user-shared data dir for this application.
 
@@ -245,7 +251,9 @@ def site_config_dir(appname=None, appauthor=None, version=None, multipath=False)
         if appname:
             if version:
                 appname = os.path.join(appname, version)
-            pathlist = [os.sep.join([x, appname]) for x in pathlist]
+            pathlist = [os.path.join(x, appname) for x in pathlist]
+        # always look in /etc directly as well
+        pathlist.append('/etc')
 
         if multipath:
             path = os.pathsep.join(pathlist)
@@ -291,6 +299,10 @@ def user_cache_dir(appname=None, appauthor=None, version=None, opinion=True):
         if appauthor is None:
             appauthor = appname
         path = os.path.normpath(_get_win_folder("CSIDL_LOCAL_APPDATA"))
+        # When using Python 2, return paths as bytes on Windows like we do on
+        # other operating systems. See helper function docs for more details.
+        if not PY3 and isinstance(path, unicode):
+            path = _win_path_to_bytes(path)
         if appname:
             if appauthor is not False:
                 path = os.path.join(path, appauthor, appname)
@@ -567,6 +579,24 @@ def _get_win_folder_with_jna(csidl_name):
             _get_win_folder = _get_win_folder_from_registry
 
 
+def _win_path_to_bytes(path):
+    """Encode Windows paths to bytes. Only used on Python 2.
+
+    Motivation is to be consistent with other operating systems where paths
+    are also returned as bytes. This avoids problems mixing bytes and Unicode
+    elsewhere in the codebase. For more details and discussion see
+    <https://github.com/pypa/pip/issues/3463>.
+
+    If encoding using ASCII and MBCS fails, return the original Unicode path.
+    """
+    for encoding in ('ASCII', 'MBCS'):
+        try:
+            return path.encode(encoding)
+        except (UnicodeEncodeError, LookupError):
+            pass
+    return path
+
+
 #---- self test code
 
 if __name__ == "__main__":