diff --git a/maigret/utils.py b/maigret/utils.py index e980a6c..7b215ee 100644 --- a/maigret/utils.py +++ b/maigret/utils.py @@ -48,7 +48,7 @@ def enrich_link_str(link: str) -> str: class URLMatcher: - _HTTP_URL_RE_STR = "^https?://(www.|m.)?(.+)$" + _HTTP_URL_RE_STR = r"^https?://(www\.|m\.)?(.+)$" HTTP_URL_RE = re.compile(_HTTP_URL_RE_STR) UNSAFE_SYMBOLS = ".?" diff --git a/tests/test_sites.py b/tests/test_sites.py index a3bce6a..78b07ba 100644 --- a/tests/test_sites.py +++ b/tests/test_sites.py @@ -160,7 +160,7 @@ def test_site_url_detector(): assert ( db.sites[0].url_regexp.pattern - == r'^https?://(www.|m.)?forum\.amperka\.ru/members/\?username=(.+?)$' + == r'^https?://(www\.|m\.)?forum\.amperka\.ru/members/\?username=(.+?)$' ) assert ( db.sites[0].detect_username('http://forum.amperka.ru/members/?username=test') diff --git a/tests/test_utils.py b/tests/test_utils.py index 26ca1cd..b606f03 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -123,6 +123,17 @@ def test_url_extract_main_part(): assert not url_regexp.match(url) is None +def test_url_extract_main_part_keeps_host_starting_with_prefix_letters(): + # The optional (www.|m.)? group must only strip the literal 'www.'/'m.' + # subdomain prefixes. With unescaped dots it instead ate the first + # character(s) of any host starting with 'm'/'www' (e.g. medium.com). + assert URLMatcher.extract_main_part('https://medium.com/alice') == 'medium.com/alice' + assert URLMatcher.extract_main_part('https://myspace.com/bob') == 'myspace.com/bob' + # genuine mobile/web subdomain prefixes are still stripped + assert URLMatcher.extract_main_part('https://m.wikipedia.org/wiki/Foo') == 'wikipedia.org/wiki/Foo' + assert URLMatcher.extract_main_part('https://www.flickr.com/photos/x') == 'flickr.com/photos/x' + + def test_url_make_profile_url_regexp(): url_main_part = 'flickr.com/photos/{username}' @@ -139,7 +150,7 @@ def test_url_make_profile_url_regexp(): # ensure all combinations match pattern assert ( URLMatcher.make_profile_url_regexp(url).pattern - == r'^https?://(www.|m.)?flickr\.com/photos/(.+?)$' + == r'^https?://(www\.|m\.)?flickr\.com/photos/(.+?)$' )