Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion Doc/library/urllib.request.rst
Original file line number Diff line number Diff line change
Expand Up @@ -358,10 +358,11 @@

To disable autodetected proxy pass an empty dictionary.

The :envvar:`no_proxy` environment variable can be used to specify hosts

Check warning on line 361 in Doc/library/urllib.request.rst

View workflow job for this annotation

GitHub Actions / Docs / Docs

'envvar' reference target not found: no_proxy [ref.envvar]
which shouldn't be reached via proxy; if set, it should be a comma-separated
list of hostname suffixes, optionally with ``:port`` appended, for example
``cern.ch,ncsa.uiuc.edu,some.host:8080``.
``cern.ch,ncsa.uiuc.edu,some.host:8080``. IP CIDR notation is also
supported.

.. note::

Expand Down
23 changes: 23 additions & 0 deletions Lib/test/test_urllib.py
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,29 @@ def test_proxy_bypass_environment_newline(self):
self.assertFalse(bypass('anotherdomain.com:8888\n'))
self.assertFalse(bypass('newdomain.com:1234\n'))

def test_proxy_bypass_environment_cidrs(self):
bypass = lambda a, b: urllib.request.proxy_bypass_environment(a, {'no': b})

# IPv4 CIDRs
self.assertTrue(bypass('192.168.0.5', 'asdf.com,192.168.0.0/24'))
self.assertTrue(bypass('192.168.0.5', 'asdf.com,192.168.0.10/24'))
self.assertTrue(bypass('192.168.0.5:8443', 'asdf.com,192.168.0.0/24'))
self.assertTrue(bypass('192.168.0.5', 'asdf.com,192.168.0.5'))
self.assertTrue(bypass('192.168.0.5', 'asdf.com,192.168.0.5/32'))
self.assertFalse(bypass('10.1.2.3', 'asdf.com,192.168.0.0/24'))

# IPv6 CIDRs
self.assertTrue(bypass('2001:db8:85a3:1::10', 'asdf.com,2001:db8:85a3:1::/64'))
self.assertTrue(bypass('[2001:db8:85a3:1::10]', 'asdf.com,2001:db8:85a3:1::/64'))
self.assertTrue(bypass('[2001:db8:85a3:1::10]:443', 'asdf.com,2001:db8:85a3:1::/64'))
self.assertTrue(bypass('2001:db8:85a3:1::10', 'asdf.com,2001:db8:85a3:1::10'))
self.assertTrue(bypass('2001:db8:85a3:1::10', 'asdf.com,2001:db8:85a3:1::10/128'))
self.assertFalse(bypass('1001:db8:85a3:1::10', 'asdf.com,2001:db8:85a3:1::10/128'))

# Invalid CIDRs should be ignored
self.assertFalse(bypass('192.168.0.5', 'asdf.com,192.168.0.5/40'))
self.assertFalse(bypass('2001:db8:85a3:1::10', 'asdf.com,2001:db8:85a3:1::/150'))


class ProxyTests_withOrderedEnv(unittest.TestCase):

Expand Down
1 change: 1 addition & 0 deletions Lib/test/test_urlparse.py
Original file line number Diff line number Diff line change
Expand Up @@ -1843,6 +1843,7 @@ def test_splitport(self):
self.assertEqual(splitport('parrot:cheese'), ('parrot:cheese', None))
self.assertEqual(splitport('[::1]:88'), ('[::1]', '88'))
self.assertEqual(splitport('[::1]'), ('[::1]', None))
self.assertEqual(splitport('::1'), ('::1', None))
self.assertEqual(splitport(':88'), ('', '88'))

def test_splitnport(self):
Expand Down
9 changes: 9 additions & 0 deletions Lib/urllib/parse.py
Original file line number Diff line number Diff line change
Expand Up @@ -1314,6 +1314,15 @@ def splitport(host):
_portprog = None
def _splitport(host):
"""splitport('host:port') --> 'host', 'port'."""
# Handle bare IPv6 addresses with : (e.g. ::1)
if host.count(':') > 1 and not host.startswith('['):
try:
ipaddress.ip_address(host)
except ValueError:
pass
else:
return host, None

global _portprog
if _portprog is None:
_portprog = re.compile('(.*):([0-9]*)', re.DOTALL)
Expand Down
72 changes: 57 additions & 15 deletions Lib/urllib/request.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,8 @@
import time
import tempfile

from functools import lru_cache
from ipaddress import ip_address, ip_network

from urllib.error import URLError, HTTPError, ContentTooShortError
from urllib.parse import (
Expand Down Expand Up @@ -1924,7 +1926,8 @@ def proxy_bypass_environment(host, proxies=None):
"""Test if proxies should not be used for a particular host.

Checks the proxy dict for the value of no_proxy, which should
be a list of comma separated DNS suffixes, or '*' for all hosts.
be a list of comma separated DNS suffixes and IP CIDRs, or
'*' for all hosts.

"""
if proxies is None:
Expand All @@ -1937,22 +1940,61 @@ def proxy_bypass_environment(host, proxies=None):
# '*' is special case for always bypass
if no_proxy == '*':
return True
host = host.lower()
# strip port off host
hostonly, port = _splitport(host)
# check if the host ends with any of the DNS suffixes
for name in no_proxy.split(','):
name = name.strip()
if name:
name = name.lstrip('.') # ignore leading dots
name = name.lower()
if hostonly == name or host == name:

return _ProxyBypassEnvSettings.cached(no_proxy).should_bypass(host)


class _ProxyBypassEnvSettings:
def __init__(self, data):
self.matches = set()
self.cidrs = []

for elem in data.split(','):
elem = elem.strip()

if not elem:
continue

try:
block = ip_network(elem, strict=False)
except ValueError:
self.matches.add(elem.lstrip('.').lower())
else:
self.cidrs.append(block)

def should_bypass(self, host):
host = host.lower()
hostonly, _ = _splitport(host)

try:
ip = ip_address(hostonly.strip('[]'))
except ValueError:
pass
else:
if any(ip in cidr for cidr in self.cidrs):
return True
name = '.' + name
if hostonly.endswith(name) or host.endswith(name):

if host in self.matches or hostonly in self.matches:
return True

for match in self.matches:
dot_match = '.' + match
if host.endswith(dot_match) or hostonly.endswith(dot_match):
return True
# otherwise, don't bypass
return False

return False

@classmethod
@lru_cache
def cached(cls, data):
"""Initializer with cache.

The vast majority of the time, NO_PROXY won't change.

This function caches the string manipulations so they don't repeat on
every single proxy check.
"""
return cls(data)


# This code tests an OSX specific data structure but is testable on all
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
urllib now allows IP CIDRs to be excluded from proxying by including the
CIDR in the ``no_proxy`` env var. Contributed by Neal Turett.
Loading