diff --git a/docs/source/command-line-options.rst b/docs/source/command-line-options.rst index e08f063..e758674 100644 --- a/docs/source/command-line-options.rst +++ b/docs/source/command-line-options.rst @@ -11,6 +11,67 @@ Usernames You can specify several usernames separated by space. Usernames are **not** mandatory as there are other operations modes (see below). +.. _identifiers-from-a-file: + +Identifiers from a file +----------------------- + +``maigret --input-file ids.txt`` + +Reads identifiers from a file, one per line, and searches them exactly like +positional ones. A single ``-`` as the path reads standard input instead, so a +generator can be piped straight in. Blank lines and lines starting with ``#`` +are skipped. + +Every line is searched as the type given by ``--id-type``, which is +``username`` unless you change it. A line can also carry its own type as an +``id_type:value`` prefix, and that is how one run can mix usernames with social +network ids. + +For example, ``ids.txt``: + +.. code-block:: text + + # usernames from a generator + john + jsmith + john.smith + + # ids of a known type + vk_id:12345 + gaia_id:109876543210 + +Then run Maigret against it: + +.. code-block:: bash + + maigret --input-file ids.txt --html + +Every line is searched with the type Maigret picked for it, and the type is +printed as it goes: + +.. code-block:: text + + [*] Checking username john on: + [*] Checking username jsmith on: + [*] Checking username john.smith on: + [*] Checking vk_id 12345 on: + [*] Checking gaia_id 109876543210 on: + +A generator can also be piped in directly, without a file in between: + +.. code-block:: bash + + ./generate-usernames.py john.smith | maigret --input-file - --html + +Mixing types in one run is worth it because everything found lands in a single +report and a single connections graph, while separate runs give you separate +ones. + +Note that ``--permute`` applies to positional usernames only. Names coming from +a file are searched as they are written, because a file can hold thousands of +lines and permuting those is rarely what you want. + Parsing of account pages and online documents --------------------------------------------- @@ -106,6 +167,9 @@ wikimapia_uid, uidme_uguid, yelp_userid, orcid, qq_id, bilibili_id. Sites whose type does not match are filtered out automatically. See :ref:`supported-identifier-types` for details and an example. +``--input-file`` - Read identifiers from a file, one per line. See +:ref:`identifiers-from-a-file` above. + ``--ignore-ids`` - Do not make search by the specified username or other ids. Useful for repeated scanning with found known irrelevant usernames. diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/command-line-options.po b/docs/source/locale/zh_CN/LC_MESSAGES/command-line-options.po index af8c0a7..21b7dcf 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/command-line-options.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/command-line-options.po @@ -735,4 +735,63 @@ msgid "" "The ``CREATE CONSTRAINT ... IF NOT EXISTS FOR ... REQUIRE`` syntax " "requires Neo4j 4.4+; the ``MERGE`` statements themselves work on any " "version." -msgstr "``CREATE CONSTRAINT ... IF NOT EXISTS FOR ... REQUIRE``\\ 语法需要 Neo4j 4.4+;而 ``MERGE``\\ 语句本身在任何版本上都可用。" \ No newline at end of file +msgstr "``CREATE CONSTRAINT ... IF NOT EXISTS FOR ... REQUIRE``\\ 语法需要 Neo4j 4.4+;而 ``MERGE``\\ 语句本身在任何版本上都可用。" + +msgid "Identifiers from a file" +msgstr "从文件读取标识符" + +msgid "``maigret --input-file ids.txt``" +msgstr "``maigret --input-file ids.txt``" + +msgid "" +"Reads identifiers from a file, one per line, and searches them exactly " +"like positional ones. A single ``-`` as the path reads standard input " +"instead, so a generator can be piped straight in. Blank lines and lines " +"starting with ``#`` are skipped." +msgstr "" +"从文件中读取标识符,每行一个,搜索方式与命令行位置参数完全相同。把路径写成单个 ``-`` " +"则改为读取标准输入,因此可以直接用管道接上生成器。空行以及以 ``#`` 开头的行会被跳过。" + +msgid "" +"Every line is searched as the type given by ``--id-type``, which is " +"``username`` unless you change it. A line can also carry its own type as " +"an ``id_type:value`` prefix, and that is how one run can mix usernames " +"with social network ids." +msgstr "" +"默认情况下,每一行都按 ``--id-type`` 指定的类型搜索,该选项不改时为 ``username``。行内也可以用 " +"``id_type:value`` 前缀单独指定类型,这样一次运行就能把用户名和社交网络 ID 混在一起。" + +msgid "" +"Mixing types in one run is worth it because everything found lands in a " +"single report and a single connections graph, while separate runs give you " +"separate ones." +msgstr "把不同类型混在一次运行里是值得的:所有结果会进入同一份报告和同一张关系图,而分开运行只会得到彼此独立的结果。" + +msgid "" +"Note that ``--permute`` applies to positional usernames only. Names coming " +"from a file are searched as they are written, because a file can hold " +"thousands of lines and permuting those is rarely what you want." +msgstr "" +"注意 ``--permute`` " +"只作用于命令行位置参数中的用户名。来自文件的名字按原样搜索,因为文件里可能有成千上万行,对它们做排列组合通常并非你想要的结果。" + +msgid "" +"``--input-file`` - Read identifiers from a file, one per line. See " +":ref:`identifiers-from-a-file` above." +msgstr "" +"``--input-file`` —— 从文件读取标识符,每行一个。参见上文的 :ref:`identifiers-from-a-file`。" + +msgid "For example, ``ids.txt``:" +msgstr "例如,``ids.txt``:" + +msgid "Then run Maigret against it:" +msgstr "然后针对该文件运行 Maigret:" + +msgid "" +"Every line is searched with the type Maigret picked for it, and the type " +"is printed as it goes:" +msgstr "每一行都会按 Maigret 为它选定的类型进行搜索,运行过程中会打印所用的类型:" + +msgid "A generator can also be piped in directly, without a file in between:" +msgstr "也可以直接用管道接上生成器,中间不需要文件:" + diff --git a/maigret/maigret.py b/maigret/maigret.py index 083192b..ba35735 100755 --- a/maigret/maigret.py +++ b/maigret/maigret.py @@ -9,9 +9,9 @@ import sys import platform import re from argparse import ArgumentParser, RawDescriptionHelpFormatter -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, List, Optional, Tuple import os.path as path -from maigret.utils import extract_usernames +from maigret.utils import extract_usernames, read_input_file try: from socid_extractor import extract, parse @@ -138,6 +138,13 @@ def setup_arguments_parser(settings: Settings): metavar="USERNAMES", help="One or more usernames to search by.", ) + parser.add_argument( + "--input-file", + dest="input_file", + metavar="PATH", + help="Read identifiers from a file, one per line ('-' for stdin). " + "A line can carry its own id type, e.g. vk_id:12345.", + ) parser.add_argument( "--version", action="version", @@ -608,6 +615,13 @@ async def main(): arg_parser = setup_arguments_parser(settings) args = arg_parser.parse_args() + input_entries: List[Tuple[str, Optional[str]]] = [] + if args.input_file: + try: + input_entries = read_input_file(args.input_file, SUPPORTED_IDS) + except OSError as e: + arg_parser.error(f"can't read input file: {e}") + # Resolve Cloudflare webgate config (CLI flag OR settings.cloudflare_bypass.enabled) cf_bypass_config = build_cloudflare_bypass_config( settings, force_enable=args.cloudflare_bypass @@ -642,6 +656,12 @@ async def main(): original_usernames = " ".join(usernames.keys()) usernames = Permute(usernames).gather(method='strict') + # Added after permutation on purpose: a file can hold thousands of names and + # ids of several types, and permuting those makes no sense. + for value, id_type in input_entries: + if value not in args.ignore_ids_list: + usernames[value] = id_type or args.id_type + parsing_enabled = not args.disable_extracting recursive_search_enabled = not args.disable_recursive_search diff --git a/maigret/utils.py b/maigret/utils.py index 416ba4d..bf4a9a6 100644 --- a/maigret/utils.py +++ b/maigret/utils.py @@ -4,11 +4,56 @@ import difflib import re import random import string -from typing import Any +import sys +from typing import Any, List, Optional, Tuple from markupsafe import Markup, escape +def read_input_file( + path: str, supported_ids: Tuple[str, ...] +) -> List[Tuple[str, Optional[str]]]: + """Read identifiers from a file, one per line; '-' means stdin. + + A line is either a bare value, or ``id_type:value`` where the prefix is one + of the supported id types. Blank lines and lines starting with # are + skipped. Bare values get their type from --id-type, resolved by the caller, + and duplicates are left alone: they collapse in the caller's dict anyway. + """ + if path == '-': + text = sys.stdin.read() + else: + with open(path, encoding='utf-8') as f: + text = f.read() + + entries: List[Tuple[str, Optional[str]]] = [] + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith('#'): + continue + + prefix, separator, value = line.partition(':') + value = value.strip() + if separator and value and prefix in supported_ids: + entries.append((value, prefix)) + continue + + # A prefix that almost spells an id type is a typo, not a value. Values + # with a colon in them (URLs and the like) are far enough from every + # supported type that they stay silent. + if separator and difflib.get_close_matches( + prefix, supported_ids, n=1, cutoff=0.8 + ): + print( + f"Unknown id type '{prefix}', taking the whole line as a value: {line}", + file=sys.stderr, + ) + + entries.append((line, None)) + + return entries + + DEFAULT_USER_AGENTS = [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36", ] diff --git a/tests/test_cli.py b/tests/test_cli.py index a58cfbf..5178a1b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -21,6 +21,7 @@ DEFAULT_ARGS: Dict[str, Any] = { 'graph': False, 'neo4j': False, 'id_type': 'username', + 'input_file': None, 'ignore_ids_list': [], 'info': False, 'json': '', @@ -86,6 +87,19 @@ def test_args_search_mode_several_usernames(argparser): assert getattr(args, arg) == want_args[arg] +def test_args_input_file(argparser): + args = argparser.parse_args('--input-file ids.txt'.split()) + + assert args.username == [] + assert args.input_file == 'ids.txt' + + want_args = dict(DEFAULT_ARGS) + want_args.update({'input_file': 'ids.txt'}) + + for arg in vars(args): + assert getattr(args, arg) == want_args[arg] + + def test_args_self_check_mode(argparser): args = argparser.parse_args('--self-check --site GitHub'.split()) diff --git a/tests/test_utils.py b/tests/test_utils.py index b2f0528..79cfe5d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,5 +1,6 @@ """Maigret utils test functions""" +import io import itertools import re @@ -7,6 +8,7 @@ from markupsafe import Markup from maigret.utils import ( CaseConverter, + read_input_file, is_country_tag, enrich_link_str, URLMatcher, @@ -261,3 +263,54 @@ def test_get_dict_ascii_tree_new_line_false_strips_leading_newline(): assert not without_nl.startswith('\n') # Only the leading newline differs; the tree content is identical. assert with_nl[1:] == without_nl + + +SUPPORTED_IDS = ("username", "vk_id", "gaia_id", "steam_id") + + +def test_read_input_file(tmp_path): + names = tmp_path / "names.txt" + names.write_text( + "john\n\n# a comment\n jsmith \n \njohn.smith\n", encoding="utf-8" + ) + + assert read_input_file(str(names), SUPPORTED_IDS) == [ + ("john", None), + ("jsmith", None), + ("john.smith", None), + ] + + +def test_read_input_file_with_id_types(tmp_path): + names = tmp_path / "ids.txt" + names.write_text("john\nvk_id:12345\ngaia_id: 109876 \n", encoding="utf-8") + + assert read_input_file(str(names), SUPPORTED_IDS) == [ + ("john", None), + ("12345", "vk_id"), + ("109876", "gaia_id"), + ] + + +def test_read_input_file_keeps_values_with_colons(tmp_path): + names = tmp_path / "ids.txt" + names.write_text("https://vk.com/id1\nnick:name\n", encoding="utf-8") + + assert read_input_file(str(names), SUPPORTED_IDS) == [ + ("https://vk.com/id1", None), + ("nick:name", None), + ] + + +def test_read_input_file_warns_on_a_misspelled_id_type(tmp_path, capsys): + names = tmp_path / "ids.txt" + names.write_text("vkid:12345\n", encoding="utf-8") + + assert read_input_file(str(names), SUPPORTED_IDS) == [("vkid:12345", None)] + assert "Unknown id type 'vkid'" in capsys.readouterr().err + + +def test_read_input_file_stdin(monkeypatch): + monkeypatch.setattr("sys.stdin", io.StringIO("john\n# skip\nvk_id:1\n")) + + assert read_input_file("-", SUPPORTED_IDS) == [("john", None), ("1", "vk_id")]