From f8c35b09e375cfd0161b78a5d5dd1259a220463f Mon Sep 17 00:00:00 2001 From: jichaowang02-lang Date: Mon, 29 Jun 2026 13:59:31 +0100 Subject: [PATCH] Fix URLMatcher eating leading host chars (unescaped dots in prefix regex) (#2808) URLMatcher._HTTP_URL_RE_STR was "^https?://(www.|m.)?(.+)$". The dots in the optional (www.|m.)? subdomain-prefix group were unescaped, so each `.` matched ANY character. For a host that simply starts with `m` or `www` (not the literal `m.`/`www.` prefix), the group greedily ate the first characters: extract_main_part('https://medium.com/alice') -> 'dium.com/alice' extract_main_part('https://myspace.com/bob') -> 'space.com/bob' extract_main_part feeds make_profile_url_regexp / get_url_template, so the generated site-detection regexp was corrupted too (medium.com -> dium.com), and the loose prefix could match bogus hosts like `xmedium.com`. 170+ hosts in resources/data.json start with `m`. Fix: escape the dots with a raw string, r"^https?://(www\.|m\.)?(.+)$". Genuine `m.`/`www.` prefixes are still stripped (m.wikipedia.org -> wikipedia.org). The two existing tests that pinned the unescaped pattern string are updated to the corrected form, and a regression test covers hosts beginning with m/www. --- maigret/utils.py | 2 +- tests/test_sites.py | 2 +- tests/test_utils.py | 13 ++++++++++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/maigret/utils.py b/maigret/utils.py index e980a6c..7b215ee 100644 --- a/maigret/utils.py +++ b/maigret/utils.py @@ -48,7 +48,7 @@ def enrich_link_str(link: str) -> str: class URLMatcher: - _HTTP_URL_RE_STR = "^https?://(www.|m.)?(.+)$" + _HTTP_URL_RE_STR = r"^https?://(www\.|m\.)?(.+)$" HTTP_URL_RE = re.compile(_HTTP_URL_RE_STR) UNSAFE_SYMBOLS = ".?" diff --git a/tests/test_sites.py b/tests/test_sites.py index a3bce6a..78b07ba 100644 --- a/tests/test_sites.py +++ b/tests/test_sites.py @@ -160,7 +160,7 @@ def test_site_url_detector(): assert ( db.sites[0].url_regexp.pattern - == r'^https?://(www.|m.)?forum\.amperka\.ru/members/\?username=(.+?)$' + == r'^https?://(www\.|m\.)?forum\.amperka\.ru/members/\?username=(.+?)$' ) assert ( db.sites[0].detect_username('http://forum.amperka.ru/members/?username=test') diff --git a/tests/test_utils.py b/tests/test_utils.py index 26ca1cd..b606f03 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -123,6 +123,17 @@ def test_url_extract_main_part(): assert not url_regexp.match(url) is None +def test_url_extract_main_part_keeps_host_starting_with_prefix_letters(): + # The optional (www.|m.)? group must only strip the literal 'www.'/'m.' + # subdomain prefixes. With unescaped dots it instead ate the first + # character(s) of any host starting with 'm'/'www' (e.g. medium.com). + assert URLMatcher.extract_main_part('https://medium.com/alice') == 'medium.com/alice' + assert URLMatcher.extract_main_part('https://myspace.com/bob') == 'myspace.com/bob' + # genuine mobile/web subdomain prefixes are still stripped + assert URLMatcher.extract_main_part('https://m.wikipedia.org/wiki/Foo') == 'wikipedia.org/wiki/Foo' + assert URLMatcher.extract_main_part('https://www.flickr.com/photos/x') == 'flickr.com/photos/x' + + def test_url_make_profile_url_regexp(): url_main_part = 'flickr.com/photos/{username}' @@ -139,7 +150,7 @@ def test_url_make_profile_url_regexp(): # ensure all combinations match pattern assert ( URLMatcher.make_profile_url_regexp(url).pattern - == r'^https?://(www.|m.)?flickr\.com/photos/(.+?)$' + == r'^https?://(www\.|m\.)?flickr\.com/photos/(.+?)$' )