MacOSX: remove dependency on Carbon package for urllib

This patch removes the dependency on the Carbon package from urllib.
The mac-specific code for getting proxy configuration is now writting in
Python using ctypes and uses the SystemConfiguration framework instead of
InternetConfig. Also provides a mac-specific implementation of proxy_bypass.

Author: ronaldoussoren

Lib/urllib.py

@@ -1321,44 +1321,214 @@ def proxy_bypass_environment(host):
 
 
 if sys.platform == 'darwin':
-    def getproxies_internetconfig():
-        """Return a dictionary of scheme -> proxy server URL mappings.
-
-        By convention the mac uses Internet Config to store
-        proxies.  An HTTP proxy, for instance, is stored under
-        the HttpProxy key.
+    def _CStringFromCFString(sc, value):
+        from ctypes import create_string_buffer
+        length = sc.CFStringGetLength(value) + 1
+        buff = create_string_buffer(length)
+        sc.CFStringGetCString(value, buff, length, 0)
+        return buff.value
+
+    def _CFNumberToInt32(sc, cfnum):
+        from ctypes import byref, c_int
+        val = c_int()
+        kCFNumberSInt32Type = 3
+        sc.CFNumberGetValue(cfnum, kCFNumberSInt32Type, byref(val))
+        return val.value
+
+
+    def proxy_bypass_macosx_sysconf(host):
+        """
+        Return True iff this host shouldn't be accessed using a proxy
 
+        This function uses the MacOSX framework SystemConfiguration
+        to fetch the proxy information.
         """
-        try:
-            import ic
-        except ImportError:
-            return {}
+        from ctypes import cdll
+        from ctypes.util import find_library
+        import re
+        import socket
+        from fnmatch import fnmatch
+
+        def ip2num(ipAddr):
+            parts = ipAddr.split('.')
+            parts = map(int, parts)
+            if len(parts) != 4:
+                parts = (parts + [0, 0, 0, 0])[:4]
+            return (parts[0] << 24) | (parts[1] << 16) | (parts[2] << 8) | parts[3]
+
+        sc = cdll.LoadLibrary(find_library("SystemConfiguration"))
+
+        hostIP = None
+
+        if not sc:
+            return False
+
+        kSCPropNetProxiesExceptionsList = sc.CFStringCreateWithCString(0, "ExceptionsList", 0)
+        kSCPropNetProxiesExcludeSimpleHostnames = sc.CFStringCreateWithCString(0,
+                "ExcludeSimpleHostnames", 0)
+
+
+        proxyDict = sc.SCDynamicStoreCopyProxies(None)
 
         try:
-            config = ic.IC()
-        except ic.error:
+            # Check for simple host names:
+            if '.' not in host:
+                exclude_simple = sc.CFDictionaryGetValue(proxyDict,
+                        kSCPropNetProxiesExcludeSimpleHostnames)
+                if exclude_simple and _CFNumberToInt32(sc, exclude_simple):
+                    return True
+
+
+            # Check the exceptions list:
+            exceptions = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesExceptionsList)
+            if exceptions:
+                # Items in the list are strings like these: *.local, 169.254/16
+                for index in xrange(sc.CFArrayGetCount(exceptions)):
+                    value = sc.CFArrayGetValueAtIndex(exceptions, index)
+                    if not value: continue
+                    value = _CStringFromCFString(sc, value)
+
+                    m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value)
+                    if m is not None:
+                        if hostIP is None:
+                            hostIP = socket.gethostbyname(host)
+                            hostIP = ip2num(hostIP)
+
+                        base = ip2num(m.group(1))
+                        mask = int(m.group(2)[1:])
+                        mask = 32 - mask
+
+                        if (hostIP >> mask) == (base >> mask):
+                            return True
+
+                    elif fnmatch(host, value):
+                        return True
+
+            return False
+
+        finally:
+            sc.CFRelease(kSCPropNetProxiesExceptionsList)
+            sc.CFRelease(kSCPropNetProxiesExcludeSimpleHostnames)
+
+
+
+    def getproxies_macosx_sysconf():
+        """Return a dictionary of scheme -> proxy server URL mappings.
+
+        This function uses the MacOSX framework SystemConfiguration
+        to fetch the proxy information.
+        """
+        from ctypes import cdll
+        from ctypes.util import find_library
+
+        sc = cdll.LoadLibrary(find_library("SystemConfiguration"))
+
+        if not sc:
             return {}
+
+
+        kSCPropNetProxiesHTTPEnable = sc.CFStringCreateWithCString(0, "HTTPEnable", 0)
+        kSCPropNetProxiesHTTPProxy = sc.CFStringCreateWithCString(0, "HTTPProxy", 0)
+        kSCPropNetProxiesHTTPPort = sc.CFStringCreateWithCString(0, "HTTPPort", 0)
+
+        kSCPropNetProxiesHTTPSEnable = sc.CFStringCreateWithCString(0, "HTTPSEnable", 0)
+        kSCPropNetProxiesHTTPSProxy = sc.CFStringCreateWithCString(0, "HTTPSProxy", 0)
+        kSCPropNetProxiesHTTPSPort = sc.CFStringCreateWithCString(0, "HTTPSPort", 0)
+
+        kSCPropNetProxiesFTPEnable = sc.CFStringCreateWithCString(0, "FTPEnable", 0)
+        kSCPropNetProxiesFTPPassive = sc.CFStringCreateWithCString(0, "FTPPassive", 0)
+        kSCPropNetProxiesFTPPort = sc.CFStringCreateWithCString(0, "FTPPort", 0)
+        kSCPropNetProxiesFTPProxy = sc.CFStringCreateWithCString(0, "FTPProxy", 0)
+
+        kSCPropNetProxiesGopherEnable = sc.CFStringCreateWithCString(0, "GopherEnable", 0)
+        kSCPropNetProxiesGopherPort = sc.CFStringCreateWithCString(0, "GopherPort", 0)
+        kSCPropNetProxiesGopherProxy = sc.CFStringCreateWithCString(0, "GopherProxy", 0)
+
         proxies = {}
-        # HTTP:
-        if 'UseHTTPProxy' in config and config['UseHTTPProxy']:
-            try:
-                value = config['HTTPProxyHost']
-            except ic.error:
-                pass
-            else:
-                proxies['http'] = 'http://%s' % value
-        # FTP: XXX To be done.
-        # Gopher: XXX To be done.
+        proxyDict = sc.SCDynamicStoreCopyProxies(None)
+
+        try:
+            # HTTP:
+            enabled = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPEnable)
+            if enabled and _CFNumberToInt32(sc, enabled):
+                proxy = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPProxy)
+                port = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPPort)
+
+                if proxy:
+                    proxy = _CStringFromCFString(sc, proxy)
+                    if port:
+                        port = _CFNumberToInt32(sc, port)
+                        proxies["http"] = "http://%s:%i" % (proxy, port)
+                    else:
+                        proxies["http"] = "http://%s" % (proxy, )
+
+            # HTTPS:
+            enabled = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPSEnable)
+            if enabled and _CFNumberToInt32(sc, enabled):
+                proxy = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPSProxy)
+                port = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesHTTPSPort)
+
+                if proxy:
+                    proxy = _CStringFromCFString(sc, proxy)
+                    if port:
+                        port = _CFNumberToInt32(sc, port)
+                        proxies["https"] = "http://%s:%i" % (proxy, port)
+                    else:
+                        proxies["https"] = "http://%s" % (proxy, )
+
+            # FTP:
+            enabled = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesFTPEnable)
+            if enabled and _CFNumberToInt32(sc, enabled):
+                proxy = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesFTPProxy)
+                port = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesFTPPort)
+
+                if proxy:
+                    proxy = _CStringFromCFString(sc, proxy)
+                    if port:
+                        port = _CFNumberToInt32(sc, port)
+                        proxies["ftp"] = "http://%s:%i" % (proxy, port)
+                    else:
+                        proxies["ftp"] = "http://%s" % (proxy, )
+
+            # Gopher:
+            enabled = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesGopherEnable)
+            if enabled and _CFNumberToInt32(sc, enabled):
+                proxy = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesGopherProxy)
+                port = sc.CFDictionaryGetValue(proxyDict, kSCPropNetProxiesGopherPort)
+
+                if proxy:
+                    proxy = _CStringFromCFString(sc, proxy)
+                    if port:
+                        port = _CFNumberToInt32(sc, port)
+                        proxies["gopher"] = "http://%s:%i" % (proxy, port)
+                    else:
+                        proxies["gopher"] = "http://%s" % (proxy, )
+        finally:
+            sc.CFRelease(proxyDict)
+
+        sc.CFRelease(kSCPropNetProxiesHTTPEnable)
+        sc.CFRelease(kSCPropNetProxiesHTTPProxy)
+        sc.CFRelease(kSCPropNetProxiesHTTPPort)
+        sc.CFRelease(kSCPropNetProxiesFTPEnable)
+        sc.CFRelease(kSCPropNetProxiesFTPPassive)
+        sc.CFRelease(kSCPropNetProxiesFTPPort)
+        sc.CFRelease(kSCPropNetProxiesFTPProxy)
+        sc.CFRelease(kSCPropNetProxiesGopherEnable)
+        sc.CFRelease(kSCPropNetProxiesGopherPort)
+        sc.CFRelease(kSCPropNetProxiesGopherProxy)
+
         return proxies
 
+
+
     def proxy_bypass(host):
         if getproxies_environment():
             return proxy_bypass_environment(host)
         else:
-            return 0
+            return proxy_bypass_macosx_sysconf(host)
 
     def getproxies():
-        return getproxies_environment() or getproxies_internetconfig()
+        return getproxies_environment() or getproxies_macosx_sysconf()
 
 elif os.name == 'nt':
     def getproxies_registry():