Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-10-31 13:41:24 +00:00
Code Issues Projects Releases Packages Wiki Activity

fixes 813986

Browse source
This commit is contained in:
Skip Montanaro 2007-08-28 23:22:52 +00:00
parent bf138333ce
commit 1a41313684
2 changed files with 16 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

5
Lib/robotparser.py
View file

@ -230,6 +230,11 @@ def __init__(self, *args):
urllib.FancyURLopener.__init__(self, *args)
self.errcode = 200
def prompt_user_passwd(self, host, realm):
## If robots.txt file is accessible only with a password,
## we act as if the file wasn't there.
return None, None
def http_error_default(self, url, fp, errcode, errmsg, headers):
self.errcode = errcode
return urllib.FancyURLopener.http_error_default(self, url, fp, errcode,

11
Lib/test/test_robotparser.py
View file

@ -134,8 +134,19 @@ def RobotTest(index, robots_txt, good_urls, bad_urls,
RobotTest(7, doc, good, bad)
class TestCase(unittest.TestCase):
def runTest(self):
test_support.requires('network')
# whole site is password-protected.
url = 'http://mueblesmoraleda.com'
parser = robotparser.RobotFileParser()
parser.set_url(url)
parser.read()
self.assertEqual(parser.can_fetch("*", url+"/robots.txt"), False)
def test_main():
test_support.run_unittest(tests)
TestCase().run()
if __name__=='__main__':
test_support.Verbose = 1