mirror of
https://github.com/soxoj/maigret.git
synced 2026-08-17 19:25:41 +02:00
Fix URLMatcher eating leading host chars (unescaped dots in prefix regex) (#2808)
URLMatcher._HTTP_URL_RE_STR was "^https?://(www.|m.)?(.+)$". The dots in
the optional (www.|m.)? subdomain-prefix group were unescaped, so each `.`
matched ANY character. For a host that simply starts with `m` or `www`
(not the literal `m.`/`www.` prefix), the group greedily ate the first
characters:
extract_main_part('https://medium.com/alice') -> 'dium.com/alice'
extract_main_part('https://myspace.com/bob') -> 'space.com/bob'
extract_main_part feeds make_profile_url_regexp / get_url_template, so the
generated site-detection regexp was corrupted too (medium.com -> dium.com),
and the loose prefix could match bogus hosts like `xmedium.com`. 170+ hosts
in resources/data.json start with `m`.
Fix: escape the dots with a raw string, r"^https?://(www\.|m\.)?(.+)$".
Genuine `m.`/`www.` prefixes are still stripped
(m.wikipedia.org -> wikipedia.org). The two existing tests that pinned the
unescaped pattern string are updated to the corrected form, and a
regression test covers hosts beginning with m/www.
This commit is contained in:
+1
-1
@@ -48,7 +48,7 @@ def enrich_link_str(link: str) -> str:
|
||||
|
||||
|
||||
class URLMatcher:
|
||||
_HTTP_URL_RE_STR = "^https?://(www.|m.)?(.+)$"
|
||||
_HTTP_URL_RE_STR = r"^https?://(www\.|m\.)?(.+)$"
|
||||
HTTP_URL_RE = re.compile(_HTTP_URL_RE_STR)
|
||||
UNSAFE_SYMBOLS = ".?"
|
||||
|
||||
|
||||
+1
-1
@@ -160,7 +160,7 @@ def test_site_url_detector():
|
||||
|
||||
assert (
|
||||
db.sites[0].url_regexp.pattern
|
||||
== r'^https?://(www.|m.)?forum\.amperka\.ru/members/\?username=(.+?)$'
|
||||
== r'^https?://(www\.|m\.)?forum\.amperka\.ru/members/\?username=(.+?)$'
|
||||
)
|
||||
assert (
|
||||
db.sites[0].detect_username('http://forum.amperka.ru/members/?username=test')
|
||||
|
||||
+12
-1
@@ -123,6 +123,17 @@ def test_url_extract_main_part():
|
||||
assert not url_regexp.match(url) is None
|
||||
|
||||
|
||||
def test_url_extract_main_part_keeps_host_starting_with_prefix_letters():
|
||||
# The optional (www.|m.)? group must only strip the literal 'www.'/'m.'
|
||||
# subdomain prefixes. With unescaped dots it instead ate the first
|
||||
# character(s) of any host starting with 'm'/'www' (e.g. medium.com).
|
||||
assert URLMatcher.extract_main_part('https://medium.com/alice') == 'medium.com/alice'
|
||||
assert URLMatcher.extract_main_part('https://myspace.com/bob') == 'myspace.com/bob'
|
||||
# genuine mobile/web subdomain prefixes are still stripped
|
||||
assert URLMatcher.extract_main_part('https://m.wikipedia.org/wiki/Foo') == 'wikipedia.org/wiki/Foo'
|
||||
assert URLMatcher.extract_main_part('https://www.flickr.com/photos/x') == 'flickr.com/photos/x'
|
||||
|
||||
|
||||
def test_url_make_profile_url_regexp():
|
||||
url_main_part = 'flickr.com/photos/{username}'
|
||||
|
||||
@@ -139,7 +150,7 @@ def test_url_make_profile_url_regexp():
|
||||
# ensure all combinations match pattern
|
||||
assert (
|
||||
URLMatcher.make_profile_url_regexp(url).pattern
|
||||
== r'^https?://(www.|m.)?flickr\.com/photos/(.+?)$'
|
||||
== r'^https?://(www\.|m\.)?flickr\.com/photos/(.+?)$'
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user