WEBDIENSTE/maigret
1
0
Fork 0
mirror of https://github.com/soxoj/maigret.git synced 2026-08-17 19:25:41 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add QQ search via qq_id id type; per-site encoding config (#2890)

Browse Source
This commit is contained in:
Soxoj
2026-07-20 19:47:33 +02:00
committed by GitHub
parent 1d3805d9ce
commit d5cf4b15ef
11 changed files with 141 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/source/command-line-options.rst
+4 -3
View File
@@ -97,9 +97,10 @@ recursive search by them.
false positives).
``--id-type`` - Specify identifier(s) type (default: username).
Supported types: gaia_id, vk_id, yandex_public_id, ok_id, wikimapia_uid.
Currently, you must add ``-a`` flag to run a scan on sites with custom
id types, sites will be filtered automatically.
Supported types: gaia_id, steam_id, vk_id, yandex_public_id, ok_id,
wikimapia_uid, uidme_uguid, yelp_userid, orcid, qq_id.
Sites whose type does not match are filtered out automatically. See
:ref:`supported-identifier-types` for details and an example.
``--ignore-ids`` - Do not make search by the specified username or other
ids. Useful for repeated scanning with found known irrelevant usernames.
docs/source/locale/zh_CN/LC_MESSAGES/command-line-options.po
+10 -8
View File
@@ -210,16 +210,18 @@ msgid ""
"false positives)."
msgstr "``--use-disabled-sites`` —— 把已禁用的站点也纳入搜索范围(可能产生大量误报)。"
#: ../../source/command-line-options.rst:99 20ce94c212df49338e550f7ecf71cc28
#: ../../source/command-line-options.rst:99
msgid ""
"``--id-type`` - Specify identifier(s) type (default: username). Supported"
" types: gaia_id, vk_id, yandex_public_id, ok_id, wikimapia_uid. "
"Currently, you must add ``-a`` flag to run a scan on sites with custom id"
" types, sites will be filtered automatically."
"``--id-type`` - Specify identifier(s) type (default: username). Supported "
"types: gaia_id, steam_id, vk_id, yandex_public_id, ok_id, wikimapia_uid, "
"uidme_uguid, yelp_userid, orcid, qq_id. Sites whose type does not match are "
"filtered out automatically. See :ref:`supported-identifier-types` for "
"details and an example."
msgstr ""
"``--id-type`` —— "
"指定标识符类型(默认:username)。支持的类型有:gaia_id、vk_id、yandex_public_id、ok_id、wikimapia_uid。目前必须同时加"
" ``-a``,才能扫描支持自定义 ID 类型的站点 —— 站点会被自动筛选。"
"``--id-type`` —— 指定标识符类型(默认:username)。支持的类型有:gaia_id、"
"steam_id、vk_id、yandex_public_id、ok_id、wikimapia_uid、uidme_uguid、"
"yelp_userid、orcid、qq_id。类型不匹配的站点会被自动筛除。详情及示例见 "
":ref:`supported-identifier-types`。"
#: ../../source/command-line-options.rst:104 ef2bfbd405b1471abfed99b2b82974d7
msgid ""
docs/source/locale/zh_CN/LC_MESSAGES/supported-identifier-types.po
+29
View File
@@ -78,3 +78,32 @@ msgstr "**ok_id** —— OK.ru 内部的数字用户标识符。"
#: 5eac68e6daa548d4a0559b3c70fc5de8
msgid "**yelp_userid** - Yelp inner user identifier."
msgstr "**yelp_userid** —— Yelp 内部的用户标识符。"
#: ../../source/supported-identifier-types.rst:16
msgid ""
"**qq_id** - QQ (Tencent) numeric account number, also used as the Qzone "
"profile id."
msgstr "**qq_id** —— QQ(腾讯)的数字账号,同时用作 Qzone 空间的用户 id。"
#: ../../source/supported-identifier-types.rst:18
msgid "Example"
msgstr "示例"
#: ../../source/supported-identifier-types.rst:21
msgid ""
"Pass the identifier as the target and select its type with ``--id-type``. "
"For example, to look up the QQ number ``10001``:"
msgstr ""
"将标识符作为搜索目标传入,并通过 ``--id-type`` 指定其类型。例如,查询 QQ 号 "
"``10001``:"
#: ../../source/supported-identifier-types.rst:27
msgid ""
"Maigret resolves the account via the Qzone portrait API and, when found, "
"extracts the nickname and avatar. Any site whose ``type`` does not match the "
"given ``--id-type`` is skipped, so a plain ``maigret 10001`` (default "
"``username``) will not query QQ."
msgstr ""
"Maigret 通过 Qzone 头像 API 解析该账号;找到后会提取昵称和头像。凡是 ``type`` "
"与所给 ``--id-type`` 不匹配的站点都会被跳过,因此直接运行 ``maigret 10001``"
"(默认为 ``username``)不会查询 QQ。"
docs/source/supported-identifier-types.rst
+12
View File
@@ -13,3 +13,15 @@ Maigret can search against not only ordinary usernames, but also through certain
- **vk_id** - VK.com inner numeric user identifier.
- **ok_id** - OK.ru inner numeric user identifier.
- **yelp_userid** - Yelp inner user identifier.
- **qq_id** - QQ (Tencent) numeric account number, also used as the Qzone profile id.
Example
-------
Pass the identifier as the target and select its type with ``--id-type``. For example, to look up the QQ number ``10001``:
.. code-block:: console
$ maigret 10001 --id-type qq_id
Maigret resolves the account via the Qzone portrait API and, when found, extracts the nickname and avatar. Any site whose ``type`` does not match the given ``--id-type`` is skipped, so a plain ``maigret 10001`` (default ``username``) will not query QQ.
maigret/checking.py
+16 -6
View File
@@ -69,6 +69,7 @@ SUPPORTED_IDS = (
"uidme_uguid",
"yelp_userid",
"orcid",
"qq_id",
)
BAD_CHARS = "#"
@@ -133,6 +134,7 @@ class CheckerBase:
self.timeout = 0
self.method = 'get'
self.payload = None
self.encoding = None
class SimpleAiohttpChecker(CheckerBase):
@@ -150,13 +152,14 @@ class SimpleAiohttpChecker(CheckerBase):
# "Could not contact DNS servers" for every site.
self.dns_resolver = kwargs.get('dns_resolver', 'async')
def prepare(self, url, headers=None, allow_redirects=True, timeout=0, method='get', payload=None):
def prepare(self, url, headers=None, allow_redirects=True, timeout=0, method='get', payload=None, encoding=None):
self.url = url
self.headers = headers
self.allow_redirects = allow_redirects
self.timeout = timeout
self.method = method
self.payload = payload
self.encoding = encoding
return None
async def close(self):
@@ -190,7 +193,7 @@ class SimpleAiohttpChecker(CheckerBase):
async with request_method(**kwargs) as response:
status_code = response.status
response_content = await response.content.read()
charset = response.charset or "utf-8"
charset = self.encoding or response.charset or "utf-8"
decoded_content = response_content.decode(charset, "ignore")
error = CheckError("Connection lost") if status_code == 0 else None
@@ -275,7 +278,7 @@ class AiodnsDomainResolver(CheckerBase):
loop = asyncio.get_event_loop()
self.resolver = aiodns.DNSResolver(loop=loop)
def prepare(self, url, headers=None, allow_redirects=True, timeout=0, method='get', payload=None):
def prepare(self, url, headers=None, allow_redirects=True, timeout=0, method='get', payload=None, encoding=None):
self.url = url
return None
@@ -308,13 +311,14 @@ class CurlCffiChecker(CheckerBase):
self.browser_emulate = kwargs.get('browser_emulate', 'chrome')
self.proxy = kwargs.get('proxy')
def prepare(self, url, headers=None, allow_redirects=True, timeout=0, method='get', payload=None):
def prepare(self, url, headers=None, allow_redirects=True, timeout=0, method='get', payload=None, encoding=None):
self.url = url
self.headers = headers
self.allow_redirects = allow_redirects
self.timeout = timeout
self.method = method
self.payload = payload
self.encoding = encoding
return None
async def close(self):
@@ -350,6 +354,8 @@ class CurlCffiChecker(CheckerBase):
response = await session.get(**kwargs)
status_code = response.status_code
if self.encoding:
response.encoding = self.encoding
decoded_content = response.text
self.logger.debug(decoded_content)
@@ -416,13 +422,14 @@ class CloudflareWebgateChecker(CheckerBase):
host_safe = re.sub(r"[^a-zA-Z0-9.-]", "_", host)
return f"{self._session_prefix}-{host_safe}"
def prepare(self, url, headers=None, allow_redirects=True, timeout=0, method='get', payload=None):
def prepare(self, url, headers=None, allow_redirects=True, timeout=0, method='get', payload=None, encoding=None):
self.url = url
self.headers = headers or {}
self.allow_redirects = allow_redirects
self.timeout = timeout
self.method = method
self.payload = payload
self.encoding = encoding
return None
async def close(self):
@@ -599,7 +606,7 @@ class CheckerMock:
def __init__(self, *args, **kwargs):
pass
def prepare(self, url, headers=None, allow_redirects=True, timeout=0, method='get', payload=None):
def prepare(self, url, headers=None, allow_redirects=True, timeout=0, method='get', payload=None, encoding=None):
return None
async def check(self) -> Tuple[Optional[str], int, Optional[CheckError]]:
@@ -965,6 +972,7 @@ def make_site_result(
allow_redirects=allow_redirects,
timeout=options['timeout'],
payload=payload,
encoding=site.encoding,
)
# Store future request object in the results object
@@ -1033,6 +1041,7 @@ async def check_site_for_username(
timeout=checker.timeout,
method=checker.method,
payload=getattr(checker, 'payload', None),
encoding=site.encoding,
)
response = await checker.check()
@@ -1711,6 +1720,7 @@ async def run_url_mutations(checker, site, results_info, options, logger, query_
allow_redirects=True,
timeout=checker.timeout,
method='get',
encoding=site.encoding,
)
options["enrich_requests"] = options.get("enrich_requests", 0) + 1
html_text, status_code, err = await checker.check()
maigret/resources/data.json
+22
View File
@@ -1,5 +1,27 @@
{
"sites": {
"QQ": {
"type": "qq_id",
"encoding": "gbk",
"regexCheck": "^[1-9][0-9]{4,8}$|^[1-3][0-9]{9}$",
"checkType": "message",
"presenseStrs": [
"portraitCallBack("
],
"absenceStrs": [
"\"\",0]",
"\"error\""
],
"urlProbe": "https://users.qzone.qq.com/fcg-bin/cgi_get_portrait.fcg?uins={username}",
"url": "https://user.qzone.qq.com/{username}",
"urlMain": "https://qq.com",
"usernameClaimed": "10001",
"usernameUnclaimed": "3999999999",
"tags": [
"cn",
"social"
]
},
"Facebook": {
"regexCheck": "^[a-zA-Z0-9_\\.]{3,49}(?<!\\.com|\\.org|\\.net)$",
"checkType": "message",
maigret/resources/db_meta.json
+3 -3
View File
@@ -1,8 +1,8 @@
{
"version": 1,
"updated_at": "2026-07-20T14:30:45Z",
"sites_count": 3210,
"updated_at": "2026-07-20T17:18:36Z",
"sites_count": 3211,
"min_maigret_version": "0.5.0",
"data_sha256": "adf974a01e838201afe830ee4ec1278fd0a41b16b447a81321986b476658d6a7",
"data_sha256": "3dac4e972128b0d0839d725326eebce5363252d000ac78420534a92624ecbe06",
"data_url": "https://raw.githubusercontent.com/soxoj/maigret/main/maigret/resources/data.json"
}
maigret/sites.py
+2
View File
@@ -57,6 +57,8 @@ class MaigretSite:
# Type of identifier (username, gaia_id etc); see SUPPORTED_IDS in checking.py
type = "username"
# Forced response encoding, overriding a wrong/lying Content-Type charset
encoding = None
# Custom HTTP headers
headers: Dict[str, str] = {}
# Error message substrings
sites.md
+9 -8
View File
@@ -1,5 +1,5 @@
## List of supported sites (search methods): total 3210
## List of supported sites (search methods): total 3211
Rank data fetched from Majestic Million by domains.
@@ -1177,6 +1177,7 @@ Rank data fetched from Majestic Million by domains.
1. ![](https://www.google.com/s2/favicons?domain=https://www.parkrocker.net) [Parkrocker (https://www.parkrocker.net)](https://www.parkrocker.net)*: top 10M, de, forum*
1. ![](https://www.google.com/s2/favicons?domain=http://dogster.ru/) [Dogster (http://dogster.ru/)](http://dogster.ru/)*: top 10M, ru*
1. ![](https://www.google.com/s2/favicons?domain=https://www.dalnoboi.ru) [Dalnoboi (https://www.dalnoboi.ru)](https://www.dalnoboi.ru)*: top 10M, ru*
1. ![](https://www.google.com/s2/favicons?domain=https://qq.com) [QQ (https://qq.com)](https://qq.com)*: top 100M, cn, social*
1. ![](https://www.google.com/s2/favicons?domain=https://social.technet.microsoft.com) [MicrosoftTechNet (https://social.technet.microsoft.com)](https://social.technet.microsoft.com)*: top 100M*, search is disabled
1. ![](https://www.google.com/s2/favicons?domain=https://learn.microsoft.com) [MicrosoftLearn (https://learn.microsoft.com)](https://learn.microsoft.com)*: top 100M, tech*
1. ![](https://www.google.com/s2/favicons?domain=https://social.msdn.microsoft.com) [social.msdn.microsoft.com (https://social.msdn.microsoft.com)](https://social.msdn.microsoft.com)*: top 100M*, search is disabled
@@ -3217,21 +3218,21 @@ Rank data fetched from Majestic Million by domains.
The list was updated at (2026-07-20)
## Statistics
Enabled/total sites: 2537/3210 = 79.03%
Enabled/total sites: 2538/3211 = 79.04%
Incomplete message checks: 325/2537 = 12.81% (false positive risks)
Incomplete message checks: 325/2538 = 12.81% (false positive risks)
Status code checks: 691/2537 = 27.24% (false positive risks)
Status code checks: 691/2538 = 27.23% (false positive risks)
False positive risk (total): 40.05%
False positive risk (total): 40.04%
Sites with probing: 500px, Armchairgm, BinarySearch (disabled), BitBucket, BleachFandom, Bluesky, BongaCams, Boosty, Bunpro, BuyMeACoffee, Calendly, Cent, Chess, Code Sandbox (disabled), Code Snippet Wiki, DailyMotion, Discord, Diskusjon.no, Disqus, Docker Hub, Duolingo, F-droid, Faceit, FandomCommunityCentral, GitHub, GitLab, Golangbridge, Google Plus (archived), Gravatar, HackTheBox, HackerNews, HackerNoon, Hackerrank, Hashnode, Hey, Holopin, ITVDN Forum, Imgur, Instagram, Keybase, Kick, Kvinneguiden, LeetCode, Lemmy World, Lesswrong, Livejasmin, LocalCryptos (disabled), Mapillary Forum, Matrix, Medium, MetaDiscourse, MicrosoftLearn, Minds, MixCloud, Monkeytype, NPM, Niftygateway, ORCID, Omg.lol, OnlyFans, Paragraph, Picsart, Polarsteps, Rarible, Reddit, Reddit Search (Pushshift) (disabled), Revolut.me, RoyalCams, Scratch, Silver-collector, Soop, SportsTracker, Spotify, StackOverflow, Substack, TAP'D, Topcoder, Trello, Twitch, Twitter, Twitter Shadowban (disabled), UnstoppableDomains, Vimeo, Vivino, Warframe Market, Warpcast, Weibo, Wikipedia, Yapisal (disabled), Ybox, YouNow, community.endlessos.com, community.getpostman.com, community.icons8.com, community.p2pu.org, discourse.haskell.org, discourse.jupyter.org, discuss.inventables.com, en.brickimedia.org, forum.audacityteam.org, forum.garudalinux.org, forum.ghost.org, forum.languagelearningwithnetflix.com, forum.shotcut.org, forum.zorin.com, forums.docker.com, forums.grandstream.com, forums.steinberg.net, habbo.com.br, habbo.com.tr, hiveos.farm, iNaturalist, nightbot, notabug.org, openframeworks, programming.dev, qiwi.me (disabled), sourceruns, support.ilovegrowingmarijuana.com
Sites with probing: 500px, Armchairgm, BinarySearch (disabled), BitBucket, BleachFandom, Bluesky, BongaCams, Boosty, Bunpro, BuyMeACoffee, Calendly, Cent, Chess, Code Sandbox (disabled), Code Snippet Wiki, DailyMotion, Discord, Diskusjon.no, Disqus, Docker Hub, Duolingo, F-droid, Faceit, FandomCommunityCentral, GitHub, GitLab, Golangbridge, Google Plus (archived), Gravatar, HackTheBox, HackerNews, HackerNoon, Hackerrank, Hashnode, Hey, Holopin, ITVDN Forum, Imgur, Instagram, Keybase, Kick, Kvinneguiden, LeetCode, Lemmy World, Lesswrong, Livejasmin, LocalCryptos (disabled), Mapillary Forum, Matrix, Medium, MetaDiscourse, MicrosoftLearn, Minds, MixCloud, Monkeytype, NPM, Niftygateway, ORCID, Omg.lol, OnlyFans, Paragraph, Picsart, Polarsteps, QQ, Rarible, Reddit, Reddit Search (Pushshift) (disabled), Revolut.me, RoyalCams, Scratch, Silver-collector, Soop, SportsTracker, Spotify, StackOverflow, Substack, TAP'D, Topcoder, Trello, Twitch, Twitter, Twitter Shadowban (disabled), UnstoppableDomains, Vimeo, Vivino, Warframe Market, Warpcast, Weibo, Wikipedia, Yapisal (disabled), Ybox, YouNow, community.endlessos.com, community.getpostman.com, community.icons8.com, community.p2pu.org, discourse.haskell.org, discourse.jupyter.org, discuss.inventables.com, en.brickimedia.org, forum.audacityteam.org, forum.garudalinux.org, forum.ghost.org, forum.languagelearningwithnetflix.com, forum.shotcut.org, forum.zorin.com, forums.docker.com, forums.grandstream.com, forums.steinberg.net, habbo.com.br, habbo.com.tr, hiveos.farm, iNaturalist, nightbot, notabug.org, openframeworks, programming.dev, qiwi.me (disabled), sourceruns, support.ilovegrowingmarijuana.com
Sites with activation: OnlyFans, ProtonMail, Twitter, Vimeo, Weibo, WikimapiaSearch
Top 20 profile URLs:
- (709) `{urlMain}/index/8-0-{username} (uCoz)`
- (325) `/{username}`
- (326) `/{username}`
- (222) `{urlMain}{urlSubpath}/members/?username={username} (XenForo)`
- (176) `/user/{username}`
- (142) `/profile/{username}`
@@ -3275,8 +3276,8 @@ Top 20 tags:
- (134) `gaming`
- (99) `coding`
- (58) `photo`
- (49) `social`
- (49) `tech`
- (48) `social`
- (46) `news`
- (42) `blog`
- (36) `shopping`
tests/local.json
+9
View File
@@ -16,6 +16,15 @@
"absenseStrs": ["not found", "404"],
"usernameClaimed": "claimed",
"usernameUnclaimed": "unclaimed"
},
"GbkMessage": {
"checkType": "message",
"encoding": "gbk",
"url": "http://localhost:8989/url?id={username}",
"urlMain": "http://localhost:8989/",
"presenseStrs": ["个人资料"],
"usernameClaimed": "claimed",
"usernameUnclaimed": "unclaimed"
}
}
}
tests/test_checking.py
+25 -1
View File
@@ -92,6 +92,28 @@ async def test_checking_by_message_negative(httpserver, local_test_db):
assert result['Message']['status'].is_found() is True
@pytest.mark.slow
@pytest.mark.asyncio
async def test_checking_respects_site_encoding_override(httpserver, local_test_db):
# Endpoint lies about the charset (says utf-8) but serves GBK bytes.
# The site pins encoding=gbk, so the Chinese presenseStr must decode and match;
# without the override the GBK bytes decode as utf-8 mojibake and never match.
sites_dict = local_test_db.sites_dict
ct = 'text/html; charset=utf-8'
site_result_except(
httpserver, 'claimed', response_data='个人资料'.encode('gbk'), content_type=ct
)
site_result_except(
httpserver, 'unclaimed', response_data='页面不存在'.encode('gbk'), content_type=ct
)
result = await search('claimed', site_dict=sites_dict, logger=Mock())
assert result['GbkMessage']['status'].is_found() is True
result = await search('unclaimed', site_dict=sites_dict, logger=Mock())
assert result['GbkMessage']['status'].is_found() is False
# ---- Pure-function unit tests (no network) ----
@@ -585,6 +607,7 @@ async def test_check_site_for_username_awaits_activation_before_retry(monkeypatc
timeout=0,
method="get",
payload=None,
encoding=None,
):
self.url = url
self.headers = headers
@@ -660,6 +683,7 @@ async def test_concurrent_activation_uses_independent_checkers(monkeypatch):
timeout=0,
method="get",
payload=None,
encoding=None,
):
self.url = url
self.headers = headers
@@ -1154,7 +1178,7 @@ class _MutationChecker(CheckerMock):
self.headers = {}
self.timeout = 3
def prepare(self, url, headers=None, allow_redirects=True, timeout=0, method='get', payload=None):
def prepare(self, url, headers=None, allow_redirects=True, timeout=0, method='get', payload=None, encoding=None):
self.urls_seen.append(url)
self.headers = headers or {}
self.timeout = timeout or self.timeout