diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 75198b70ad4ff5..bee8d238be6b2f 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -246,6 +246,32 @@ class DefaultEntryTest(BaseRequestRateTest, unittest.TestCase): bad = ['/cyberworld/map/index.html'] +class StringFormattingTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Crawl-delay: 1 +Request-rate: 3/15 +Disallow: /cyberworld/map/ # This is an infinite virtual URL space + +# Cybermapper knows where to go. +User-agent: cybermapper +Disallow: /some/path + """ + + expected_output = """\ +User-agent: cybermapper +Disallow: /some/path + +User-agent: * +Crawl-delay: 1 +Request-rate: 3/15 +Disallow: /cyberworld/map/\ +""" + + def test_string_formatting(self): + self.assertEqual(str(self.parser), self.expected_output) + + class RobotHandler(BaseHTTPRequestHandler): def do_GET(self): diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index daac29c68dc36d..92e4efe6865e1f 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -190,7 +190,10 @@ def request_rate(self, useragent): return self.default_entry.req_rate def __str__(self): - return ''.join([str(entry) + "\n" for entry in self.entries]) + entries = self.entries + if self.default_entry is not None: + entries = entries + [self.default_entry] + return '\n\n'.join(map(str, entries)) class RuleLine: @@ -222,10 +225,14 @@ def __init__(self): def __str__(self): ret = [] for agent in self.useragents: - ret.extend(["User-agent: ", agent, "\n"]) - for line in self.rulelines: - ret.extend([str(line), "\n"]) - return ''.join(ret) + ret.append(f"User-agent: {agent}") + if self.delay is not None: + ret.append(f"Crawl-delay: {self.delay}") + if self.req_rate is not None: + rate = self.req_rate + ret.append(f"Request-rate: {rate.requests}/{rate.seconds}") + ret.extend(map(str, self.rulelines)) + return '\n'.join(ret) def applies_to(self, useragent): """check if this entry applies to the specified agent""" diff --git a/Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst b/Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst new file mode 100644 index 00000000000000..2bb24d7edab54d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst @@ -0,0 +1,4 @@ +The urllib.robotparser's ``__str__`` representation now includes wildcard +entries and the "Crawl-delay" and "Request-rate" fields. Also removes extra +newlines that were being appended to the end of the string. Patch by +Michael Lazar.