WEBDIENSTE/maigret
1
0
Fork 0
mirror of https://github.com/soxoj/maigret.git synced 2026-08-17 19:25:41 +02:00
Code Issues Packages Projects Releases Wiki Activity

Web interface update: live scan, settings perseistence, history, nagivation (#2925)

Browse Source
This commit is contained in:
Soxoj
2026-08-03 08:41:07 +09:00
committed by GitHub
parent 9b6c5381fa
commit d1b8b9fa5c
12 changed files with 1769 additions and 628 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -39,6 +39,7 @@ htmlcov/
# Maigret files
settings.json
web_settings.json
# other
*.egg-info
maigret/executors.py
+40 -3
View File
@@ -12,6 +12,18 @@ class AsyncioQueueGeneratorExecutor:
self._results: asyncio.Queue = asyncio.Queue()
self._stop_signal = object()
def _log_late_task_result(self, task):
"""Done-callback for a task we stopped waiting on (timed out or the
worker itself was cancelled). Nothing else will ever retrieve this
task's outcome, so without this callback asyncio logs a noisy
"exception was never retrieved" once it finally finishes unwinding.
"""
if task.cancelled():
return
exc = task.exception()
if exc is not None:
self.logger.debug(f"Timed-out/cancelled check task raised: {exc}")
async def worker(self):
"""Process tasks from the queue and put results into the results queue."""
while True:
@@ -20,16 +32,41 @@ class AsyncioQueueGeneratorExecutor:
self.queue.task_done()
break
query_task = None
try:
f, args, kwargs = task
query_future = f(*args, **kwargs)
query_task = asyncio.create_task(query_future)
try:
result = await asyncio.wait_for(query_task, timeout=self.timeout)
except asyncio.TimeoutError:
# Deliberately asyncio.wait(), not wait_for(): wait_for's
# cancel-on-timeout path awaits the cancelled task's own
# cleanup (e.g. closing an aiohttp/curl_cffi session) before
# returning. A site that holds its connection open without
# completing (bot protection doing exactly this) can make
# that cleanup itself take far longer than `timeout`, which
# blocks this worker — and the whole scan's progress — well
# past the configured timeout. asyncio.wait() returns the
# moment the deadline hits regardless of how long the task
# takes to actually unwind; we let that happen in the
# background instead of waiting on it here.
done, _ = await asyncio.wait({query_task}, timeout=self.timeout)
if query_task in done:
result = query_task.result()
else:
query_task.cancel()
query_task.add_done_callback(self._log_late_task_result)
result = kwargs.get('default')
await self._results.put(result)
except asyncio.CancelledError:
# The worker itself was cancelled (Ctrl+C / Stop button —
# see run()'s finally). Request cancellation of whatever
# query was in flight, but don't wait on it for the same
# reason as above, then let the cancellation propagate so
# this worker actually stops.
if query_task is not None and not query_task.done():
query_task.cancel()
query_task.add_done_callback(self._log_late_task_result)
raise
except Exception as e:
self.logger.error(f"Error in worker: {e}", exc_info=True)
finally:
maigret/report.py
+4 -1
View File
@@ -329,7 +329,10 @@ def save_graph_report(filename: str, username_results: list, db: MaigretDatabase
# Generate interactive visualization
from pyvis.network import Network # type: ignore[import-untyped]
nt = Network(notebook=True, height="100vh", width="100%")
# cdn_resources="in_line": self-contained HTML, no lib/ folder written
# relative to the process cwd (pyvis's default "local" mode does that,
# which breaks when the report is served from a different directory).
nt = Network(notebook=True, height="100vh", width="100%", cdn_resources="in_line")
nt.from_nx(G)
nt.show(filename)
maigret/resources/db_meta.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"version": 1,
"updated_at": "2026-08-02T03:15:46Z",
"updated_at": "2026-08-02T16:00:54Z",
"sites_count": 3221,
"min_maigret_version": "0.5.0",
"data_sha256": "7fcd1658d7a40560042ecdb819f6cc6b15b3dbe14d689c5c6c7b03e2ef146bfa",
maigret/web/app.py
+456 -120
View File
@@ -12,12 +12,16 @@ from werkzeug.exceptions import NotFound
import logging
import os
import asyncio
import json
import queue
import uuid
from datetime import datetime
from threading import Thread
from typing import Any, Dict
import maigret
import maigret.settings
from maigret.checking import build_cloudflare_bypass_config
from maigret.result import MaigretCheckStatus
from maigret.sites import MaigretDatabase
from maigret.report import generate_report_context
@@ -29,11 +33,167 @@ app.secret_key = os.getenv('FLASK_SECRET_KEY', os.urandom(24).hex())
background_jobs: Dict[str, Any] = {}
job_results = {}
# Live (streaming) scan jobs, keyed by job_id. Each entry:
# {'queue': Queue, 'cancelled': bool, 'loop': event loop, 'task': asyncio task}
# ponytail: in-memory, single-process. Entries are dropped when their SSE stream
# ends; a stream that's never consumed lingers — fine for a local tool.
live_jobs: Dict[str, Any] = {}
class StreamNotify:
"""query_notify shim: pushes each per-site check into a queue as an SSE event.
maigret's search loop calls update() once per finished site check, which is
exactly the granularity we want to stream to the browser.
"""
def __init__(self, event_queue, username):
self.q = event_queue
self.username = username
self.total = 0
self.checked = 0
self.sites = {}
# Per-site results collected so far, in the shape build_reports()
# expects. If the scan gets cancelled mid-way (Stop button), this is
# what's left to report on — otherwise every already-streamed
# 'found' event is silently discarded because the search() task
# never returns to hand back its own results dict.
self.results = {}
def set_total(self, total):
self.total = total
self.q.put({'type': 'start', 'username': self.username, 'total': total})
def set_sites(self, sites):
self.sites = sites
def update(self, result, is_similar=False):
self.checked += 1
if not is_similar:
entry = {'status': result, 'url_user': result.site_url_user}
site = self.sites.get(result.site_name)
if site is not None:
entry['site'] = site
entry['url_main'] = site.url_main
self.results[result.site_name] = entry
if result.status == MaigretCheckStatus.CLAIMED and not is_similar:
ids = {
k: v
for k, v in (result.ids_data or {}).items()
if k != '_extractor' and isinstance(v, (str, int, float))
}
self.q.put(
{
'type': 'found',
'username': result.username or self.username,
'site': result.site_name,
'url': result.site_url_user,
'ids': ids,
}
)
self.q.put(
{
'type': 'progress',
'checked': self.checked,
'total': self.total,
'site': result.site_name,
}
)
# No-op sinks for the rest of the notifier surface the search loop touches.
def start(self, message=None, id_type="username"):
pass
def finish(self, message=None):
pass
def warning(self, *a, **k):
pass
def info(self, *a, **k):
pass
def success(self, *a, **k):
pass
def enrich(self, *a, **k):
pass
# Configuration
app.config["MAIGRET_DB_FILE"] = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'resources', 'data.json')
app.config["MAIGRET_DB_FILE"] = os.path.join(
os.path.dirname(os.path.dirname(__file__)), 'resources', 'data.json'
)
app.config["COOKIES_FILE"] = "cookies.txt"
app.config["UPLOAD_FOLDER"] = 'uploads'
app.config["REPORTS_FOLDER"] = os.path.abspath('/tmp/maigret_reports')
app.config["SETTINGS_FILE"] = "web_settings.json"
# Search-wide defaults, editable from the Settings modal (base.html). Persisted
# to app.config["SETTINGS_FILE"] so they survive a process restart.
DEFAULT_SETTINGS = {
'timeout': 10,
'top_sites': 500,
'tags': [],
'excluded_tags': [],
'site_list': [],
'proxy': '',
'tor_proxy': '',
'i2p_proxy': '',
'permute': False,
'disable_recursive_search': False,
'disable_extracting': False,
'with_domains': False,
}
def load_settings():
settings = dict(DEFAULT_SETTINGS)
path = app.config["SETTINGS_FILE"]
if os.path.exists(path):
try:
with open(path, encoding='utf-8') as f:
settings.update(json.load(f))
except (json.JSONDecodeError, OSError) as e:
logging.error(f"Failed to load settings from {path}: {e}")
return settings
def save_settings(settings):
with open(app.config["SETTINGS_FILE"], 'w', encoding='utf-8') as f:
json.dump(settings, f, indent=2)
def parse_settings_form(form):
try:
timeout = int(form.get('timeout'))
except (TypeError, ValueError):
timeout = DEFAULT_SETTINGS['timeout']
try:
top_sites = int(form.get('top_sites'))
except (TypeError, ValueError):
top_sites = DEFAULT_SETTINGS['top_sites']
return {
'timeout': timeout,
'top_sites': top_sites,
'tags': form.getlist('tags'),
'excluded_tags': form.getlist('excluded_tags'),
'site_list': [s.strip() for s in form.get('site', '').split(',') if s.strip()],
'proxy': form.get('proxy', '').strip(),
'tor_proxy': form.get('tor_proxy', '').strip(),
'i2p_proxy': form.get('i2p_proxy', '').strip(),
'permute': 'permute' in form,
'disable_recursive_search': 'disable_recursive_search' in form,
'disable_extracting': 'disable_extracting' in form,
'with_domains': 'with_domains' in form,
}
@app.context_processor
def inject_settings():
return {'web_settings': load_settings()}
def setup_logger(log_level, name):
@@ -42,7 +202,7 @@ def setup_logger(log_level, name):
return logger
async def maigret_search(username, options):
async def maigret_search(username, options, query_notify=None):
logger = setup_logger(logging.WARNING, 'maigret')
try:
settings = maigret.settings.Settings()
@@ -80,12 +240,19 @@ async def maigret_search(username, options):
logger.info(f"Found {len(sites)} sites matching the tag criteria")
if query_notify is not None and hasattr(query_notify, 'set_total'):
query_notify.set_total(len(sites))
if query_notify is not None and hasattr(query_notify, 'set_sites'):
query_notify.set_sites(sites)
results = await maigret.search(
username=username,
site_dict=sites,
timeout=int(options.get('timeout', 30)),
logger=logger,
id_type='username',
query_notify=query_notify,
no_progressbar=bool(query_notify),
cookies=app.config["COOKIES_FILE"] if options.get('use_cookies') else None,
is_parsing_enabled=(not options.get('disable_extracting', False)),
recursive_search_enabled=(
@@ -124,7 +291,95 @@ def sanitize_username_for_path(username: str) -> str:
return sanitized or '_'
def build_reports(general_results, usernames, session_key):
"""Write per-username CSV/JSON/PDF/HTML reports + combined graph to disk.
Shared by the background /search job and the live SSE /api/scan job, so
both flows land on the same results.html (report buttons + profile list).
"""
os.makedirs(app.config["REPORTS_FOLDER"], exist_ok=True)
session_folder = os.path.join(
app.config["REPORTS_FOLDER"], f"search_{session_key}"
)
os.makedirs(session_folder, exist_ok=True)
graph_path = os.path.join(session_folder, "combined_graph.html")
maigret.report.save_graph_report(
graph_path,
general_results,
MaigretDatabase().load_from_path(app.config["MAIGRET_DB_FILE"]),
)
individual_reports = []
found_count = 0
for username, id_type, results in general_results:
safe_username = sanitize_username_for_path(username)
report_base = os.path.join(session_folder, f"report_{safe_username}")
csv_path = f"{report_base}.csv"
json_path = f"{report_base}.json"
pdf_path = f"{report_base}.pdf"
html_path = f"{report_base}.html"
context = generate_report_context(general_results)
maigret.report.save_csv_report(csv_path, username, results)
maigret.report.save_json_report(
json_path, username, results, report_type='ndjson'
)
maigret.report.save_pdf_report(pdf_path, context)
maigret.report.save_html_report(html_path, context)
claimed_profiles = []
for site_name, site_data in results.items():
if (
site_data.get('status')
and site_data['status'].status == MaigretCheckStatus.CLAIMED
):
claimed_profiles.append(
{
'site_name': site_name,
'url': site_data.get('url_user', ''),
'tags': (
site_data.get('status').tags
if site_data.get('status')
else []
),
}
)
found_count += len(claimed_profiles)
individual_reports.append(
{
'username': username,
'csv_file': os.path.join(
f"search_{session_key}", f"report_{safe_username}.csv"
),
'json_file': os.path.join(
f"search_{session_key}", f"report_{safe_username}.json"
),
'pdf_file': os.path.join(
f"search_{session_key}", f"report_{safe_username}.pdf"
),
'html_file': os.path.join(
f"search_{session_key}", f"report_{safe_username}.html"
),
'claimed_profiles': claimed_profiles,
}
)
return {
'status': 'completed',
'session_folder': f"search_{session_key}",
'graph_file': os.path.join(f"search_{session_key}", "combined_graph.html"),
'usernames': usernames,
'individual_reports': individual_reports,
'found_count': found_count,
}
def process_search_task(usernames, options, timestamp):
started_at = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
try:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
@@ -132,153 +387,234 @@ def process_search_task(usernames, options, timestamp):
general_results = loop.run_until_complete(
search_multiple_usernames(usernames, options)
)
os.makedirs(app.config["REPORTS_FOLDER"], exist_ok=True)
session_folder = os.path.join(
app.config["REPORTS_FOLDER"], f"search_{timestamp}"
)
os.makedirs(session_folder, exist_ok=True)
graph_path = os.path.join(session_folder, "combined_graph.html")
maigret.report.save_graph_report(
graph_path,
general_results,
MaigretDatabase().load_from_path(app.config["MAIGRET_DB_FILE"]),
)
individual_reports = []
for username, id_type, results in general_results:
safe_username = sanitize_username_for_path(username)
report_base = os.path.join(session_folder, f"report_{safe_username}")
csv_path = f"{report_base}.csv"
json_path = f"{report_base}.json"
pdf_path = f"{report_base}.pdf"
html_path = f"{report_base}.html"
context = generate_report_context(general_results)
maigret.report.save_csv_report(csv_path, username, results)
maigret.report.save_json_report(
json_path, username, results, report_type='ndjson'
)
maigret.report.save_pdf_report(pdf_path, context)
maigret.report.save_html_report(html_path, context)
claimed_profiles = []
for site_name, site_data in results.items():
if (
site_data.get('status')
and site_data['status'].status
== maigret.result.MaigretCheckStatus.CLAIMED
):
claimed_profiles.append(
{
'site_name': site_name,
'url': site_data.get('url_user', ''),
'tags': (
site_data.get('status').tags
if site_data.get('status')
else []
),
}
)
individual_reports.append(
{
'username': username,
'csv_file': os.path.join(
f"search_{timestamp}", f"report_{safe_username}.csv"
),
'json_file': os.path.join(
f"search_{timestamp}", f"report_{safe_username}.json"
),
'pdf_file': os.path.join(
f"search_{timestamp}", f"report_{safe_username}.pdf"
),
'html_file': os.path.join(
f"search_{timestamp}", f"report_{safe_username}.html"
),
'claimed_profiles': claimed_profiles,
}
)
# save results and mark job as complete using timestamp as key
job_results[timestamp] = {
'status': 'completed',
'session_folder': f"search_{timestamp}",
'graph_file': os.path.join(f"search_{timestamp}", "combined_graph.html"),
'usernames': usernames,
'individual_reports': individual_reports,
}
job_results[timestamp] = build_reports(general_results, usernames, timestamp)
except Exception as e:
logging.error(f"Error in search task for timestamp {timestamp}: {str(e)}")
job_results[timestamp] = {'status': 'failed', 'error': str(e)}
job_results[timestamp] = {
'status': 'failed',
'error': str(e),
'usernames': usernames,
}
finally:
job_results[timestamp]['started_at'] = started_at
background_jobs[timestamp]['completed'] = True
def parse_usernames(form):
usernames_input = form.get('usernames', '').strip()
return [u.strip() for u in usernames_input.replace(',', ' ').split() if u.strip()]
def parse_search_options(form):
settings = load_settings()
return {
'top_sites': settings['top_sites'],
'timeout': settings['timeout'],
'use_cookies': 'use_cookies' in form,
'all_sites': form.get('mode') == 'full',
'disable_recursive_search': settings['disable_recursive_search'],
'disable_extracting': settings['disable_extracting'],
'with_domains': settings['with_domains'],
'proxy': settings['proxy'] or None,
'tor_proxy': settings['tor_proxy'] or None,
'i2p_proxy': settings['i2p_proxy'] or None,
'permute': settings['permute'],
'tags': settings['tags'],
'excluded_tags': settings['excluded_tags'],
'site_list': settings['site_list'],
}
async def _stream_search(job, usernames, options):
q = job['queue']
general_results = []
for username in usernames:
if job['cancelled']:
break
notify = StreamNotify(q, username.strip())
task = asyncio.ensure_future(
maigret_search(username.strip(), options, query_notify=notify)
)
job['task'] = task
try:
results = await task
general_results.append((username.strip(), 'username', results))
except asyncio.CancelledError:
# The task never got to return its own results dict, but every
# site checked before cancellation already streamed a 'found' /
# 'progress' event and was captured by the notifier — report on
# that instead of throwing it away.
if notify.results:
general_results.append((username.strip(), 'username', notify.results))
q.put({'type': 'stopped', 'username': username.strip()})
break
except Exception as e:
if notify.results:
general_results.append((username.strip(), 'username', notify.results))
q.put({'type': 'error', 'message': str(e), 'username': username.strip()})
return general_results
def run_stream_job(job_id, usernames, options):
started_at = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
job = live_jobs[job_id]
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
job['loop'] = loop
general_results = []
try:
general_results = loop.run_until_complete(
_stream_search(job, usernames, options)
)
except Exception as e:
job['queue'].put({'type': 'error', 'message': str(e)})
finally:
loop.close()
# Same report files + results page as the classic /search flow, so the
# live graph is a progress view, not a replacement for the report.
done_event = {'type': 'done'}
if general_results:
try:
job_results[job_id] = build_reports(general_results, usernames, job_id)
job_results[job_id]['started_at'] = started_at
done_event['redirect'] = f"/results/search_{job_id}"
except Exception as e:
logging.error(f"Error building reports for live scan {job_id}: {str(e)}")
job['queue'].put(done_event)
def start_live_job(usernames, options):
job_id = uuid.uuid4().hex
live_jobs[job_id] = {
'queue': queue.Queue(),
'cancelled': False,
'loop': None,
'task': None,
}
Thread(target=run_stream_job, args=(job_id, usernames, options)).start()
return job_id
@app.route('/api/scan', methods=['POST'])
def scan_start():
usernames = parse_usernames(request.form)
if not usernames:
return {'error': 'At least one username is required'}, 400
options = parse_search_options(request.form)
job_id = start_live_job(usernames, options)
return {'job_id': job_id}
@app.route('/api/scan/<job_id>/stream')
def scan_stream(job_id):
job = live_jobs.get(job_id)
if not job:
return "Unknown job", 404
def gen():
try:
while True:
event = job['queue'].get()
yield f"data: {json.dumps(event)}\n\n"
if event.get('type') == 'done':
break
finally:
live_jobs.pop(job_id, None)
return Response(gen(), mimetype='text/event-stream')
@app.route('/api/scan/<job_id>/stop', methods=['POST'])
def scan_stop(job_id):
job = live_jobs.get(job_id)
if not job:
return {'error': 'unknown job'}, 404
job['cancelled'] = True
loop = job.get('loop')
task = job.get('task')
if loop and task:
loop.call_soon_threadsafe(task.cancel)
return {'ok': True}
@app.route('/')
def index():
# load site data for autocomplete
return render_template('index.html')
@app.route('/api/sites')
def api_sites():
"""Site names/URLs for the Filters site-picker datalist, fetched lazily
from the Settings modal instead of loading the DB on every page render."""
db = MaigretDatabase().load_from_path(app.config["MAIGRET_DB_FILE"])
site_options = []
for site in db.sites:
# add main site name
site_options.append(site.name)
# add URL if different from name
if site.url_main and site.url_main not in site_options:
site_options.append(site.url_main)
return {'sites': sorted(set(site_options))}
# sort and deduplicate
site_options = sorted(set(site_options))
return render_template('index.html', site_options=site_options)
@app.route('/settings', methods=['POST'])
def settings_update():
save_settings(parse_settings_form(request.form))
flash('Settings saved.', 'success')
return redirect(request.referrer or url_for('index'))
@app.route('/history')
def history():
entries = sorted(
job_results.values(), key=lambda r: r.get('started_at', ''), reverse=True
)
return render_template('history.html', entries=entries)
@app.route('/live', methods=['POST'])
def live_start():
usernames = parse_usernames(request.form)
if not usernames:
flash('At least one username is required', 'danger')
return redirect(url_for('index'))
options = parse_search_options(request.form)
job_id = start_live_job(usernames, options)
return redirect(url_for('live_results', job_id=job_id))
@app.route('/live/<job_id>')
def live_results(job_id):
result = job_results.get(job_id)
if job_id not in live_jobs and not result:
flash('Unknown or expired scan session.', 'danger')
return redirect(url_for('index'))
done_redirect = None
if result and result.get('status') == 'completed':
done_redirect = url_for('results', session_id=result['session_folder'])
return render_template('live.html', job_id=job_id, done_redirect=done_redirect)
# Modified search route
@app.route('/search', methods=['POST'])
def search():
usernames_input = request.form.get('usernames', '').strip()
if not usernames_input:
usernames = parse_usernames(request.form)
if not usernames:
flash('At least one username is required', 'danger')
return redirect(url_for('index'))
usernames = [
u.strip() for u in usernames_input.replace(',', ' ').split() if u.strip()
]
# Create timestamp for this search session
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
# Get selected tags - ensure it's a list
selected_tags = request.form.getlist('tags')
excluded_tags = request.form.getlist('excluded_tags')
logging.info(f"Selected tags: {selected_tags}, Excluded tags: {excluded_tags}")
options = {
'top_sites': request.form.get('top_sites') or '500',
'timeout': request.form.get('timeout') or '30',
'use_cookies': 'use_cookies' in request.form,
'all_sites': 'all_sites' in request.form,
'disable_recursive_search': 'disable_recursive_search' in request.form,
'disable_extracting': 'disable_extracting' in request.form,
'with_domains': 'with_domains' in request.form,
'proxy': request.form.get('proxy', None) or None,
'tor_proxy': request.form.get('tor_proxy', None) or None,
'i2p_proxy': request.form.get('i2p_proxy', None) or None,
'permute': 'permute' in request.form,
'tags': selected_tags, # Pass selected tags as a list
'excluded_tags': excluded_tags, # Pass excluded tags as a list
'site_list': [
s.strip() for s in request.form.get('site', '').split(',') if s.strip()
],
}
options = parse_search_options(request.form)
logging.info(
f"Starting search for usernames: {usernames} with tags: {selected_tags}, excluded: {excluded_tags}"
f"Starting search for usernames: {usernames} with tags: {options['tags']}, "
f"excluded: {options['excluded_tags']}"
)
# Start background job
maigret/web/templates/base.html
+476 -3
View File
@@ -68,6 +68,89 @@
.footer a:hover {
text-decoration: underline;
}
.tag-cloud {
display: flex;
flex-wrap: wrap;
gap: 8px;
padding: 15px;
border-radius: 8px;
background: rgba(0, 0, 0, 0.05);
margin-bottom: 20px;
}
.tag {
display: inline-block;
padding: 5px 10px;
border-radius: 15px;
background-color: #dc3545;
color: white;
cursor: pointer;
font-size: 14px;
transition: all 0.3s ease;
user-select: none;
}
.tag.selected {
background-color: #28a745;
}
.tag.excluded {
background-color: #343a40;
text-decoration: line-through;
}
.tag:hover {
transform: translateY(-2px);
box-shadow: 0 2px 5px rgba(0, 0, 0, 0.2);
}
.hidden-select {
display: none !important;
}
.site-input-container {
position: relative;
}
.selected-sites {
display: flex;
flex-wrap: wrap;
gap: 8px;
padding: 10px 0;
}
.selected-site {
background-color: #214e7b;
padding: 2px 8px;
border-radius: 12px;
font-size: 14px;
display: inline-flex;
align-items: center;
gap: 5px;
}
.remove-site {
cursor: pointer;
color: #dc3545;
font-weight: bold;
}
/* The header/body/footer live inside a <form>, which breaks the flex
chain .modal-dialog-scrollable relies on (.modal-content's direct
children stop being .modal-header/.modal-body/.modal-footer) — redo
that flex layout one level down, on the form itself. */
#settingsModal .modal-content>form {
display: flex;
flex-direction: column;
min-height: 0;
}
#settingsModal .modal-body {
flex: 1 1 auto;
min-height: 0;
overflow-y: auto;
}
</style>
</head>
@@ -79,19 +162,270 @@
<img src="{{ url_for('static', filename='maigret.png') }}" alt="Maigret Logo" class="logo">
<h1 class="h4 mb-0">Maigret Web Interface</h1>
</div>
<button class="btn btn-outline-secondary" id="theme-toggle">
Toggle Dark/Light Mode
</button>
<div class="d-flex align-items-center gap-2">
<a href="{{ url_for('index') }}" class="btn btn-outline-secondary">New Search</a>
<a href="{{ url_for('history') }}" class="btn btn-outline-secondary">History</a>
<button class="btn btn-outline-secondary" id="theme-toggle">
Toggle Dark/Light Mode
</button>
<button class="btn btn-outline-secondary" id="settings-toggle" title="Settings"
data-bs-toggle="modal" data-bs-target="#settingsModal">
&#9881;
</button>
</div>
</div>
</div>
</div>
<div class="main-container">
<div class="container">
{% with messages = get_flashed_messages() %}
{% if messages %}
{% for message in messages %}
<div class="alert alert-info">{{ message }}</div>
{% endfor %}
{% endif %}
{% endwith %}
{% block content %}{% endblock %}
</div>
</div>
<div class="modal fade" id="settingsModal" tabindex="-1" aria-labelledby="settingsModalLabel" aria-hidden="true">
<div class="modal-dialog modal-lg modal-dialog-scrollable">
<div class="modal-content">
<form method="POST" action="{{ url_for('settings_update') }}">
<div class="modal-header">
<h5 class="modal-title" id="settingsModalLabel">Settings</h5>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<div class="modal-body">
<div class="mb-3">
<label for="settings-timeout" class="form-label">Timeout (seconds)</label>
<input type="number" class="form-control" id="settings-timeout" name="timeout" min="1"
value="{{ web_settings.timeout }}">
</div>
<div class="mb-3">
<label for="settings-top_sites" class="form-label">Number of Sites (Fast check)</label>
<input type="number" class="form-control" id="settings-top_sites" name="top_sites" min="1"
max="10000" value="{{ web_settings.top_sites }}">
</div>
<h6>Advanced Options</h6>
<div class="mb-3 form-check">
<input type="checkbox" class="form-check-input" id="settings-permute" name="permute"
{{ 'checked' if web_settings.permute }}>
<label class="form-check-label" for="settings-permute">Enable Username Permutations</label>
</div>
<div class="mb-3 form-check">
<input type="checkbox" class="form-check-input" id="settings-disable_recursive_search"
name="disable_recursive_search" {{ 'checked' if web_settings.disable_recursive_search }}>
<label class="form-check-label" for="settings-disable_recursive_search">Disable Recursive Search</label>
</div>
<div class="mb-3 form-check">
<input type="checkbox" class="form-check-input" id="settings-disable_extracting"
name="disable_extracting" {{ 'checked' if web_settings.disable_extracting }}>
<label class="form-check-label" for="settings-disable_extracting">Disable Information Extraction</label>
</div>
<div class="mb-3 form-check">
<input type="checkbox" class="form-check-input" id="settings-with_domains"
name="with_domains" {{ 'checked' if web_settings.with_domains }}>
<label class="form-check-label" for="settings-with_domains">Check Domains</label>
</div>
<h6>Proxy URLs</h6>
<div class="mb-3">
<label for="settings-proxy" class="form-label">Proxy URL</label>
<input type="text" class="form-control" id="settings-proxy" name="proxy"
placeholder="e.g., 127.0.0.1:1080" value="{{ web_settings.proxy }}">
</div>
<div class="mb-3">
<label for="settings-tor_proxy" class="form-label">TOR Proxy URL</label>
<input type="text" class="form-control" id="settings-tor_proxy" name="tor_proxy"
placeholder="Default: 127.0.0.1:9050" value="{{ web_settings.tor_proxy }}">
</div>
<div class="mb-3">
<label for="settings-i2p_proxy" class="form-label">I2P Proxy URL</label>
<input type="text" class="form-control" id="settings-i2p_proxy" name="i2p_proxy"
placeholder="Default: 127.0.0.1:4444" value="{{ web_settings.i2p_proxy }}">
</div>
<h6>Filters</h6>
<div class="mb-3 site-input-container">
<label for="settingsSiteInput" class="form-label">Specify Sites (Optional)</label>
<input type="text" class="form-control" id="settingsSiteInput"
placeholder="Type to search for sites..." list="settingsSiteOptions">
<input type="hidden" id="settings-site" name="site" value="{{ web_settings.site_list | join(',') }}">
<datalist id="settingsSiteOptions"></datalist>
<div class="selected-sites" id="settingsSelectedSites"></div>
</div>
<div class="mb-3">
<label class="form-label">Tags (click to cycle: include &rarr; exclude &rarr; neutral)</label>
<div class="mb-2">
<small class="text-muted">
<span style="display:inline-block;width:12px;height:12px;background:#28a745;border-radius:50%;"></span> Included (whitelist)
&nbsp;&nbsp;
<span style="display:inline-block;width:12px;height:12px;background:#343a40;border-radius:50%;"></span> Excluded (blacklist)
&nbsp;&nbsp;
<span style="display:inline-block;width:12px;height:12px;background:#dc3545;border-radius:50%;"></span> Neutral
</small>
</div>
<div class="tag-cloud" id="settingsTagCloud"></div>
<select multiple class="hidden-select" id="settingsTags" name="tags">
<option value="gaming">Gaming</option>
<option value="coding">Coding</option>
<option value="photo">Photo</option>
<option value="music">Music</option>
<option value="blog">Blog</option>
<option value="finance">Finance</option>
<option value="freelance">Freelance</option>
<option value="dating">Dating</option>
<option value="tech">Tech</option>
<option value="forum">Forum</option>
<option value="porn">Porn</option>
<option value="erotic">Erotic</option>
<option value="webcam">Webcam</option>
<option value="video">Video</option>
<option value="movies">Movies</option>
<option value="hacking">Hacking</option>
<option value="art">Art</option>
<option value="discussion">Discussion</option>
<option value="sharing">Sharing</option>
<option value="writing">Writing</option>
<option value="wiki">Wiki</option>
<option value="business">Business</option>
<option value="shopping">Shopping</option>
<option value="sport">Sport</option>
<option value="books">Books</option>
<option value="news">News</option>
<option value="documents">Documents</option>
<option value="travel">Travel</option>
<option value="maps">Maps</option>
<option value="hobby">Hobby</option>
<option value="apps">Apps</option>
<option value="classified">Classified</option>
<option value="career">Career</option>
<option value="geosocial">Geosocial</option>
<option value="streaming">Streaming</option>
<option value="education">Education</option>
<option value="networking">Networking</option>
<option value="torrent">Torrent</option>
<option value="science">Science</option>
<option value="medicine">Medicine</option>
<option value="reading">Reading</option>
<option value="stock">Stock</option>
<option value="messaging">Messaging</option>
<option value="trading">Trading</option>
<option value="links">Links</option>
<option value="fashion">Fashion</option>
<option value="tasks">Tasks</option>
<option value="military">Military</option>
<option value="auto">Auto</option>
<option value="gambling">Gambling</option>
<option value="cybercriminal">Cybercriminal</option>
<option value="review">Review</option>
<option value="bookmarks">Bookmarks</option>
<option value="design">Design</option>
<option value="tor">Tor</option>
<option value="i2p">I2P</option>
<option value="q&a">Q&A</option>
<option value="crypto">Crypto</option>
<option value="ai">AI</option>
<!-- Country tags -->
<option value="ae" data-group="country">AE - United Arab Emirates</option>
<option value="ao" data-group="country">AO - Angola</option>
<option value="ar" data-group="country">AR - Argentina</option>
<option value="at" data-group="country">AT - Austria</option>
<option value="au" data-group="country">AU - Australia</option>
<option value="az" data-group="country">AZ - Azerbaijan</option>
<option value="bd" data-group="country">BD - Bangladesh</option>
<option value="be" data-group="country">BE - Belgium</option>
<option value="bg" data-group="country">BG - Bulgaria</option>
<option value="br" data-group="country">BR - Brazil</option>
<option value="by" data-group="country">BY - Belarus</option>
<option value="ca" data-group="country">CA - Canada</option>
<option value="ch" data-group="country">CH - Switzerland</option>
<option value="cl" data-group="country">CL - Chile</option>
<option value="cn" data-group="country">CN - China</option>
<option value="co" data-group="country">CO - Colombia</option>
<option value="cr" data-group="country">CR - Costa Rica</option>
<option value="cz" data-group="country">CZ - Czechia</option>
<option value="de" data-group="country">DE - Germany</option>
<option value="dk" data-group="country">DK - Denmark</option>
<option value="dz" data-group="country">DZ - Algeria</option>
<option value="ee" data-group="country">EE - Estonia</option>
<option value="eg" data-group="country">EG - Egypt</option>
<option value="es" data-group="country">ES - Spain</option>
<option value="eu" data-group="country">EU - European Union</option>
<option value="fi" data-group="country">FI - Finland</option>
<option value="fr" data-group="country">FR - France</option>
<option value="gb" data-group="country">GB - United Kingdom</option>
<option value="global" data-group="country">&#127757; Global</option>
<option value="gr" data-group="country">GR - Greece</option>
<option value="hk" data-group="country">HK - Hong Kong</option>
<option value="hr" data-group="country">HR - Croatia</option>
<option value="hu" data-group="country">HU - Hungary</option>
<option value="id" data-group="country">ID - Indonesia</option>
<option value="ie" data-group="country">IE - Ireland</option>
<option value="il" data-group="country">IL - Israel</option>
<option value="in" data-group="country">IN - India</option>
<option value="ir" data-group="country">IR - Iran</option>
<option value="it" data-group="country">IT - Italy</option>
<option value="jp" data-group="country">JP - Japan</option>
<option value="kg" data-group="country">KG - Kyrgyzstan</option>
<option value="kr" data-group="country">KR - Korea</option>
<option value="kz" data-group="country">KZ - Kazakhstan</option>
<option value="la" data-group="country">LA - Laos</option>
<option value="lk" data-group="country">LK - Sri Lanka</option>
<option value="lt" data-group="country">LT - Lithuania</option>
<option value="ma" data-group="country">MA - Morocco</option>
<option value="md" data-group="country">MD - Moldova</option>
<option value="mg" data-group="country">MG - Madagascar</option>
<option value="mk" data-group="country">MK - North Macedonia</option>
<option value="mx" data-group="country">MX - Mexico</option>
<option value="ng" data-group="country">NG - Nigeria</option>
<option value="nl" data-group="country">NL - Netherlands</option>
<option value="no" data-group="country">NO - Norway</option>
<option value="ph" data-group="country">PH - Philippines</option>
<option value="pk" data-group="country">PK - Pakistan</option>
<option value="pl" data-group="country">PL - Poland</option>
<option value="pt" data-group="country">PT - Portugal</option>
<option value="re" data-group="country">RE - Réunion</option>
<option value="ro" data-group="country">RO - Romania</option>
<option value="rs" data-group="country">RS - Serbia</option>
<option value="ru" data-group="country">RU - Russia</option>
<option value="sa" data-group="country">SA - Saudi Arabia</option>
<option value="sd" data-group="country">SD - Sudan</option>
<option value="se" data-group="country">SE - Sweden</option>
<option value="sg" data-group="country">SG - Singapore</option>
<option value="sk" data-group="country">SK - Slovakia</option>
<option value="sv" data-group="country">SV - El Salvador</option>
<option value="th" data-group="country">TH - Thailand</option>
<option value="tn" data-group="country">TN - Tunisia</option>
<option value="tr" data-group="country">TR - Türkiye</option>
<option value="tw" data-group="country">TW - Taiwan</option>
<option value="ua" data-group="country">UA - Ukraine</option>
<option value="uk" data-group="country">UK - United Kingdom</option>
<option value="us" data-group="country">US - United States</option>
<option value="uz" data-group="country">UZ - Uzbekistan</option>
<option value="ve" data-group="country">VE - Venezuela</option>
<option value="vi" data-group="country">VI - Virgin Islands</option>
<option value="vn" data-group="country">VN - Viet Nam</option>
<option value="za" data-group="country">ZA - South Africa</option>
</select>
<select multiple class="hidden-select" id="settingsExcludedTags" name="excluded_tags"></select>
</div>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-outline-secondary" data-bs-dismiss="modal">Cancel</button>
<button type="submit" class="btn" style="background-color: rgb(249, 207, 0); color: black;">Save Settings</button>
</div>
</form>
</div>
</div>
</div>
<footer class="footer">
<div class="container">
<p class="mb-0">
@@ -112,6 +446,145 @@
html.setAttribute('data-bs-theme', 'dark');
}
});
document.addEventListener('DOMContentLoaded', function () {
const initialSelected = {{ web_settings.tags | tojson }};
const initialExcluded = {{ web_settings.excluded_tags | tojson }};
const initialSites = {{ web_settings.site_list | tojson }};
// Tag cloud: include/exclude (whitelist/blacklist) cycling
const tagCloud = document.getElementById('settingsTagCloud');
const hiddenSelect = document.getElementById('settingsTags');
const excludedSelect = document.getElementById('settingsExcludedTags');
const allTags = Array.from(hiddenSelect.options).map(opt => ({
value: opt.value,
label: opt.text,
group: opt.dataset.group || 'category'
}));
function updateTagSelects() {
Array.from(hiddenSelect.options).forEach(opt => opt.selected = false);
excludedSelect.innerHTML = '';
document.querySelectorAll('#settingsTagCloud .tag').forEach(tagEl => {
const val = tagEl.dataset.value;
if (tagEl.classList.contains('selected')) {
const option = Array.from(hiddenSelect.options).find(opt => opt.value === val);
if (option) option.selected = true;
} else if (tagEl.classList.contains('excluded')) {
const opt = document.createElement('option');
opt.value = val;
opt.selected = true;
excludedSelect.appendChild(opt);
}
});
}
let lastGroup = '';
allTags.forEach(tag => {
if (tag.group !== lastGroup && tag.group === 'country') {
const separator = document.createElement('div');
separator.style.cssText = 'width:100%;margin:8px 0 4px;padding:4px 0;border-top:1px solid rgba(0,0,0,0.15);font-size:13px;color:#666;';
separator.textContent = 'Countries';
tagCloud.appendChild(separator);
}
lastGroup = tag.group;
const tagElement = document.createElement('span');
tagElement.className = 'tag';
tagElement.textContent = tag.label;
tagElement.dataset.value = tag.value;
if (initialSelected.includes(tag.value)) tagElement.classList.add('selected');
if (initialExcluded.includes(tag.value)) tagElement.classList.add('excluded');
tagElement.addEventListener('click', function (e) {
e.preventDefault();
if (this.classList.contains('selected')) {
this.classList.remove('selected');
this.classList.add('excluded');
} else if (this.classList.contains('excluded')) {
this.classList.remove('excluded');
} else {
this.classList.add('selected');
}
updateTagSelects();
});
tagCloud.appendChild(tagElement);
});
updateTagSelects();
// Site picker: chips + lazily-loaded datalist (avoids a DB read on every page)
const siteInput = document.getElementById('settingsSiteInput');
const siteHiddenInput = document.getElementById('settings-site');
const selectedSitesContainer = document.getElementById('settingsSelectedSites');
const siteDatalist = document.getElementById('settingsSiteOptions');
let selectedSites = new Set(initialSites);
let sitesLoaded = false;
function updateSiteHiddenInput() {
siteHiddenInput.value = Array.from(selectedSites).join(',');
}
function addSite(site) {
if (site && !selectedSites.has(site)) {
selectedSites.add(site);
updateSiteHiddenInput();
const siteElement = document.createElement('span');
siteElement.className = 'selected-site';
siteElement.innerHTML = `${site}<span class="remove-site" data-site="${site}">&times;</span>`;
selectedSitesContainer.appendChild(siteElement);
}
}
function removeSite(site) {
selectedSites.delete(site);
updateSiteHiddenInput();
selectedSitesContainer.querySelectorAll('.selected-site').forEach(el => {
if (el.querySelector('.remove-site').dataset.site === site) el.remove();
});
}
initialSites.forEach(site => {
const siteElement = document.createElement('span');
siteElement.className = 'selected-site';
siteElement.innerHTML = `${site}<span class="remove-site" data-site="${site}">&times;</span>`;
selectedSitesContainer.appendChild(siteElement);
});
document.getElementById('settingsModal').addEventListener('show.bs.modal', function () {
if (sitesLoaded) return;
sitesLoaded = true;
fetch("{{ url_for('api_sites') }}")
.then(r => r.json())
.then(data => {
(data.sites || []).forEach(site => {
const option = document.createElement('option');
option.value = site;
siteDatalist.appendChild(option);
});
});
});
siteInput.addEventListener('change', function () {
const value = this.value.trim();
if (value) {
addSite(value);
this.value = '';
}
});
selectedSitesContainer.addEventListener('click', function (e) {
if (e.target.classList.contains('remove-site')) {
removeSite(e.target.dataset.site);
}
});
siteInput.addEventListener('paste', function (e) {
e.preventDefault();
const paste = (e.clipboardData || window.clipboardData).getData('text');
paste.split(',').map(s => s.trim()).filter(Boolean).forEach(addSite);
});
});
</script>
</body>
maigret/web/templates/history.html
+43
View File
@@ -0,0 +1,43 @@
{% extends "base.html" %}
{% block content %}
<div class="form-container">
<h1 class="mb-4">Search History</h1>
{% if not entries %}
<p>No searches have been run yet.</p>
{% else %}
<table class="table table-striped align-middle">
<thead>
<tr>
<th>Started</th>
<th>Usernames</th>
<th>Found</th>
<th>Status</th>
<th></th>
</tr>
</thead>
<tbody>
{% for result in entries %}
<tr>
<td>{{ result.started_at or '—' }}</td>
<td>{{ result.usernames | join(', ') if result.usernames else '—' }}</td>
<td>{{ result.found_count if result.status == 'completed' else '—' }}</td>
<td>
{% if result.status == 'completed' %}
<span class="badge bg-success">completed</span>
{% else %}
<span class="badge bg-danger" title="{{ result.error }}">failed</span>
{% endif %}
</td>
<td>
{% if result.status == 'completed' %}
<a href="{{ url_for('results', session_id=result.session_folder) }}" class="btn btn-sm btn-outline-warning">View</a>
{% endif %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endif %}
</div>
{% endblock %}
maigret/web/templates/index.html
+12 -481
View File
@@ -2,106 +2,6 @@
{% block content %}
<style>
.tag-cloud {
display: flex;
flex-wrap: wrap;
gap: 8px;
padding: 15px;
border-radius: 8px;
background: rgba(0, 0, 0, 0.05);
margin-bottom: 20px;
}
.tag {
display: inline-block;
padding: 5px 10px;
border-radius: 15px;
background-color: #dc3545;
color: white;
cursor: pointer;
font-size: 14px;
transition: all 0.3s ease;
user-select: none;
}
.tag.selected {
background-color: #28a745;
}
.tag.excluded {
background-color: #343a40;
text-decoration: line-through;
}
.tag:hover {
transform: translateY(-2px);
box-shadow: 0 2px 5px rgba(0, 0, 0, 0.2);
}
.hidden-select {
display: none !important;
}
.site-input-container {
position: relative;
}
.site-input {
width: 100%;
}
.selected-sites {
display: flex;
flex-wrap: wrap;
gap: 8px;
padding: 10px 0;
}
.selected-site {
background-color: #214e7b;
padding: 2px 8px;
border-radius: 12px;
font-size: 14px;
display: inline-flex;
align-items: center;
gap: 5px;
}
.remove-site {
cursor: pointer;
color: #dc3545;
font-weight: bold;
}
.section-header {
cursor: pointer;
padding: 1rem;
background: rgba(255, 255, 255, 0.05);
border-radius: 4px;
margin-bottom: 0.5rem;
display: flex;
justify-content: space-between;
align-items: center;
}
.section-content {
padding: 1rem;
display: none;
}
.section-content.show {
display: block;
}
.chevron::after {
content: '▼';
transition: transform 0.2s;
}
.chevron.collapsed::after {
transform: rotate(-90deg);
}
.main-search-section {
background: rgba(255, 255, 255, 0.03);
padding: 2rem;
@@ -122,8 +22,7 @@
<div class="alert alert-danger">{{ error }}</div>
{% endif %}
<form method="POST" action="{{ url_for('search') }}" class="mb-4">
<!-- Main Search Section -->
<form method="POST" action="{{ url_for('live_start') }}" class="mb-4">
<div class="main-search-section">
<div class="mb-4">
<label for="usernames" class="form-label h5">Usernames to Search</label>
@@ -131,390 +30,22 @@
placeholder="Enter one or more usernames (separated by spaces or commas)..."></textarea>
</div>
<div class="row align-items-center">
<div class="col-md-6">
<label for="top_sites" class="form-label">Number of Sites</label>
<input type="number" class="form-control" id="top_sites" name="top_sites" min="1" max="10000"
placeholder="Default: 500">
</div>
<div class="col-md-6">
<label for="timeout" class="form-label">Timeout (seconds)</label>
<input type="number" class="form-control" id="timeout" name="timeout" min="1"
placeholder="Default: 30">
</div>
<div class="col-12 mt-3">
<div class="form-check">
<input type="checkbox" class="form-check-input" id="all_sites" name="all_sites"
onchange="document.getElementById('top_sites').disabled = this.checked;">
<label class="form-check-label" for="all_sites">Search All Sites</label>
</div>
</div>
<div class="btn-group" role="group" aria-label="Search mode">
<input type="radio" class="btn-check" name="mode" id="mode-fast" value="fast" checked>
<label class="btn btn-outline-secondary" for="mode-fast">Fast check ({{ web_settings.top_sites }} sites)</label>
<input type="radio" class="btn-check" name="mode" id="mode-full" value="full">
<label class="btn btn-outline-secondary" for="mode-full">Full check</label>
</div>
</div>
<!-- Filters Section -->
<div class="mb-4">
<div class="section-header" onclick="toggleSection('filters')">
<h5 class="mb-0">Filters</h5>
<span class="chevron"></span>
</div>
<div id="filters" class="section-content">
<div class="mb-3 site-input-container">
<label for="site" class="form-label">Specify Sites (Optional)</label>
<input type="text" class="form-control site-input" id="siteInput"
placeholder="Type to search for sites..." list="siteOptions">
<input type="hidden" id="site" name="site">
<datalist id="siteOptions">
{% for site in site_options %}
<option value="{{ site }}">
{% endfor %}
</datalist>
<div class="selected-sites" id="selectedSites"></div>
</div>
<p class="text-muted">
Filters, proxies and advanced options now live in
<a href="#" data-bs-toggle="modal" data-bs-target="#settingsModal">Settings</a> (&#9881; top right).
</p>
<div class="mb-3">
<label class="form-label">Tags (click to cycle: include → exclude → neutral)</label>
<div class="mb-2">
<small class="text-muted">
<span style="display:inline-block;width:12px;height:12px;background:#28a745;border-radius:50%;"></span> Included (whitelist)
&nbsp;&nbsp;
<span style="display:inline-block;width:12px;height:12px;background:#343a40;border-radius:50%;"></span> Excluded (blacklist)
&nbsp;&nbsp;
<span style="display:inline-block;width:12px;height:12px;background:#dc3545;border-radius:50%;"></span> Neutral
</small>
</div>
<div class="tag-cloud" id="tagCloud"></div>
<select multiple class="hidden-select" id="tags" name="tags">
<option value="gaming">Gaming</option>
<option value="coding">Coding</option>
<option value="photo">Photo</option>
<option value="music">Music</option>
<option value="blog">Blog</option>
<option value="finance">Finance</option>
<option value="freelance">Freelance</option>
<option value="dating">Dating</option>
<option value="tech">Tech</option>
<option value="forum">Forum</option>
<option value="porn">Porn</option>
<option value="erotic">Erotic</option>
<option value="webcam">Webcam</option>
<option value="video">Video</option>
<option value="movies">Movies</option>
<option value="hacking">Hacking</option>
<option value="art">Art</option>
<option value="discussion">Discussion</option>
<option value="sharing">Sharing</option>
<option value="writing">Writing</option>
<option value="wiki">Wiki</option>
<option value="business">Business</option>
<option value="shopping">Shopping</option>
<option value="sport">Sport</option>
<option value="books">Books</option>
<option value="news">News</option>
<option value="documents">Documents</option>
<option value="travel">Travel</option>
<option value="maps">Maps</option>
<option value="hobby">Hobby</option>
<option value="apps">Apps</option>
<option value="classified">Classified</option>
<option value="career">Career</option>
<option value="geosocial">Geosocial</option>
<option value="streaming">Streaming</option>
<option value="education">Education</option>
<option value="networking">Networking</option>
<option value="torrent">Torrent</option>
<option value="science">Science</option>
<option value="medicine">Medicine</option>
<option value="reading">Reading</option>
<option value="stock">Stock</option>
<option value="messaging">Messaging</option>
<option value="trading">Trading</option>
<option value="links">Links</option>
<option value="fashion">Fashion</option>
<option value="tasks">Tasks</option>
<option value="military">Military</option>
<option value="auto">Auto</option>
<option value="gambling">Gambling</option>
<option value="cybercriminal">Cybercriminal</option>
<option value="review">Review</option>
<option value="bookmarks">Bookmarks</option>
<option value="design">Design</option>
<option value="tor">Tor</option>
<option value="i2p">I2P</option>
<option value="q&a">Q&A</option>
<option value="crypto">Crypto</option>
<option value="ai">AI</option>
<!-- Country tags -->
<option value="ae" data-group="country">AE - United Arab Emirates</option>
<option value="ao" data-group="country">AO - Angola</option>
<option value="ar" data-group="country">AR - Argentina</option>
<option value="at" data-group="country">AT - Austria</option>
<option value="au" data-group="country">AU - Australia</option>
<option value="az" data-group="country">AZ - Azerbaijan</option>
<option value="bd" data-group="country">BD - Bangladesh</option>
<option value="be" data-group="country">BE - Belgium</option>
<option value="bg" data-group="country">BG - Bulgaria</option>
<option value="br" data-group="country">BR - Brazil</option>
<option value="by" data-group="country">BY - Belarus</option>
<option value="ca" data-group="country">CA - Canada</option>
<option value="ch" data-group="country">CH - Switzerland</option>
<option value="cl" data-group="country">CL - Chile</option>
<option value="cn" data-group="country">CN - China</option>
<option value="co" data-group="country">CO - Colombia</option>
<option value="cr" data-group="country">CR - Costa Rica</option>
<option value="cz" data-group="country">CZ - Czechia</option>
<option value="de" data-group="country">DE - Germany</option>
<option value="dk" data-group="country">DK - Denmark</option>
<option value="dz" data-group="country">DZ - Algeria</option>
<option value="ee" data-group="country">EE - Estonia</option>
<option value="eg" data-group="country">EG - Egypt</option>
<option value="es" data-group="country">ES - Spain</option>
<option value="eu" data-group="country">EU - European Union</option>
<option value="fi" data-group="country">FI - Finland</option>
<option value="fr" data-group="country">FR - France</option>
<option value="gb" data-group="country">GB - United Kingdom</option>
<option value="global" data-group="country">🌍 Global</option>
<option value="gr" data-group="country">GR - Greece</option>
<option value="hk" data-group="country">HK - Hong Kong</option>
<option value="hr" data-group="country">HR - Croatia</option>
<option value="hu" data-group="country">HU - Hungary</option>
<option value="id" data-group="country">ID - Indonesia</option>
<option value="ie" data-group="country">IE - Ireland</option>
<option value="il" data-group="country">IL - Israel</option>
<option value="in" data-group="country">IN - India</option>
<option value="ir" data-group="country">IR - Iran</option>
<option value="it" data-group="country">IT - Italy</option>
<option value="jp" data-group="country">JP - Japan</option>
<option value="kg" data-group="country">KG - Kyrgyzstan</option>
<option value="kr" data-group="country">KR - Korea</option>
<option value="kz" data-group="country">KZ - Kazakhstan</option>
<option value="la" data-group="country">LA - Laos</option>
<option value="lk" data-group="country">LK - Sri Lanka</option>
<option value="lt" data-group="country">LT - Lithuania</option>
<option value="ma" data-group="country">MA - Morocco</option>
<option value="md" data-group="country">MD - Moldova</option>
<option value="mg" data-group="country">MG - Madagascar</option>
<option value="mk" data-group="country">MK - North Macedonia</option>
<option value="mx" data-group="country">MX - Mexico</option>
<option value="ng" data-group="country">NG - Nigeria</option>
<option value="nl" data-group="country">NL - Netherlands</option>
<option value="no" data-group="country">NO - Norway</option>
<option value="ph" data-group="country">PH - Philippines</option>
<option value="pk" data-group="country">PK - Pakistan</option>
<option value="pl" data-group="country">PL - Poland</option>
<option value="pt" data-group="country">PT - Portugal</option>
<option value="re" data-group="country">RE - Réunion</option>
<option value="ro" data-group="country">RO - Romania</option>
<option value="rs" data-group="country">RS - Serbia</option>
<option value="ru" data-group="country">RU - Russia</option>
<option value="sa" data-group="country">SA - Saudi Arabia</option>
<option value="sd" data-group="country">SD - Sudan</option>
<option value="se" data-group="country">SE - Sweden</option>
<option value="sg" data-group="country">SG - Singapore</option>
<option value="sk" data-group="country">SK - Slovakia</option>
<option value="sv" data-group="country">SV - El Salvador</option>
<option value="th" data-group="country">TH - Thailand</option>
<option value="tn" data-group="country">TN - Tunisia</option>
<option value="tr" data-group="country">TR - Türkiye</option>
<option value="tw" data-group="country">TW - Taiwan</option>
<option value="ua" data-group="country">UA - Ukraine</option>
<option value="uk" data-group="country">UK - United Kingdom</option>
<option value="us" data-group="country">US - United States</option>
<option value="uz" data-group="country">UZ - Uzbekistan</option>
<option value="ve" data-group="country">VE - Venezuela</option>
<option value="vi" data-group="country">VI - Virgin Islands</option>
<option value="vn" data-group="country">VN - Viet Nam</option>
<option value="za" data-group="country">ZA - South Africa</option>
</select>
<select multiple class="hidden-select" id="excludedTags" name="excluded_tags">
</select>
</div>
</div>
</div>
<!-- Advanced Options Section -->
<div class="mb-4">
<div class="section-header" onclick="toggleSection('advanced')">
<h5 class="mb-0">Advanced Options</h5>
<span class="chevron"></span>
</div>
<div id="advanced" class="section-content">
<div class="mb-3 form-check">
<input type="checkbox" class="form-check-input" id="permute" name="permute">
<label class="form-check-label" for="permute">Enable Username Permutations</label>
</div>
<div class="mb-3 form-check">
<input type="checkbox" class="form-check-input" id="disable_recursive_search"
name="disable_recursive_search">
<label class="form-check-label" for="disable_recursive_search">Disable Recursive Search</label>
</div>
<div class="mb-3 form-check">
<input type="checkbox" class="form-check-input" id="disable_extracting" name="disable_extracting">
<label class="form-check-label" for="disable_extracting">Disable Information Extraction</label>
</div>
<div class="mb-3 form-check">
<input type="checkbox" class="form-check-input" id="with_domains" name="with_domains">
<label class="form-check-label" for="with_domains">Check Domains</label>
</div>
<div class="mb-3">
<label for="proxy" class="form-label">Proxy URL</label>
<input type="text" class="form-control" id="proxy" name="proxy"
placeholder="e.g., 127.0.0.1:1080">
</div>
<div class="mb-3">
<label for="tor_proxy" class="form-label">TOR Proxy URL</label>
<input type="text" class="form-control" id="tor_proxy" name="tor_proxy"
placeholder="Default: 127.0.0.1:9050">
</div>
<div class="mb-3">
<label for="i2p_proxy" class="form-label">I2P Proxy URL</label>
<input type="text" class="form-control" id="i2p_proxy" name="i2p_proxy"
placeholder="Default: 127.0.0.1:4444">
</div>
</div>
</div>
<button type="submit" class="btn search-button" style="background-color: rgb(249, 207, 0); color: black;">
<button type="submit" id="startBtn" class="btn search-button" style="background-color: rgb(249, 207, 0); color: black;">
Start Search
</button>
</form>
</div>
<script>
function toggleSection(sectionId) {
const content = document.getElementById(sectionId);
const header = content.previousElementSibling;
content.classList.toggle('show');
header.querySelector('.chevron').classList.toggle('collapsed');
}
document.addEventListener('DOMContentLoaded', function () {
// Tag cloud functionality with include/exclude (whitelist/blacklist) support
const tagCloud = document.getElementById('tagCloud');
const hiddenSelect = document.getElementById('tags');
const excludedSelect = document.getElementById('excludedTags');
const allTags = Array.from(hiddenSelect.options).map(opt => ({
value: opt.value,
label: opt.text,
group: opt.dataset.group || 'category'
}));
function updateTagSelects() {
// Clear and repopulate hidden selects based on tag states
Array.from(hiddenSelect.options).forEach(opt => opt.selected = false);
// Clear excluded select
excludedSelect.innerHTML = '';
document.querySelectorAll('#tagCloud .tag').forEach(tagEl => {
const val = tagEl.dataset.value;
if (tagEl.classList.contains('selected')) {
const option = Array.from(hiddenSelect.options).find(opt => opt.value === val);
if (option) option.selected = true;
} else if (tagEl.classList.contains('excluded')) {
const opt = document.createElement('option');
opt.value = val;
opt.selected = true;
excludedSelect.appendChild(opt);
}
});
}
let lastGroup = '';
allTags.forEach(tag => {
if (tag.group !== lastGroup && tag.group === 'country') {
const separator = document.createElement('div');
separator.style.cssText = 'width:100%;margin:8px 0 4px;padding:4px 0;border-top:1px solid rgba(0,0,0,0.15);font-size:13px;color:#666;';
separator.textContent = 'Countries';
tagCloud.appendChild(separator);
}
lastGroup = tag.group;
const tagElement = document.createElement('span');
tagElement.className = 'tag';
tagElement.textContent = tag.label;
tagElement.dataset.value = tag.value;
// Single click cycles: neutral -> included -> excluded -> neutral
tagElement.addEventListener('click', function (e) {
e.preventDefault();
if (this.classList.contains('selected')) {
// included -> excluded
this.classList.remove('selected');
this.classList.add('excluded');
} else if (this.classList.contains('excluded')) {
// excluded -> neutral
this.classList.remove('excluded');
} else {
// neutral -> included
this.classList.add('selected');
}
updateTagSelects();
});
tagCloud.appendChild(tagElement);
});
// Site selection functionality
const siteInput = document.getElementById('siteInput');
const hiddenInput = document.getElementById('site');
const selectedSitesContainer = document.getElementById('selectedSites');
let selectedSites = new Set();
function updateHiddenInput() {
hiddenInput.value = Array.from(selectedSites).join(',');
}
function addSite(site) {
if (site && !selectedSites.has(site)) {
selectedSites.add(site);
updateHiddenInput();
const siteElement = document.createElement('span');
siteElement.className = 'selected-site';
siteElement.innerHTML = `${site}<span class="remove-site" data-site="${site}">&times;</span>`;
selectedSitesContainer.appendChild(siteElement);
}
}
function removeSite(site) {
selectedSites.delete(site);
updateHiddenInput();
const siteElements = selectedSitesContainer.querySelectorAll('.selected-site');
siteElements.forEach(el => {
if (el.querySelector('.remove-site').dataset.site === site) {
el.remove();
}
});
}
siteInput.addEventListener('change', function (e) {
const value = this.value.trim();
if (value) {
addSite(value);
this.value = '';
}
});
selectedSitesContainer.addEventListener('click', function (e) {
if (e.target.classList.contains('remove-site')) {
removeSite(e.target.dataset.site);
}
});
siteInput.addEventListener('paste', function (e) {
e.preventDefault();
const paste = (e.clipboardData || window.clipboardData).getData('text');
const sites = paste.split(',').map(site => site.trim()).filter(site => site);
sites.forEach(addSite);
});
const form = document.querySelector('form');
form.addEventListener('submit', function (e) {
const selectedTags = Array.from(tagCloud.querySelectorAll('.tag.selected'));
Array.from(hiddenSelect.options).forEach(opt => {
opt.selected = selectedTags.some(tag => tag.dataset.value === opt.value);
});
updateHiddenInput();
});
});
</script>
{% endblock %}
{% endblock %}
maigret/web/templates/live.html
+162
View File
@@ -0,0 +1,162 @@
{% extends "base.html" %}
{% block content %}
<style>
#graph {
height: 600px;
border: 1px solid var(--bs-border-color);
border-radius: 8px;
background: rgba(255, 255, 255, 0.02);
}
.live-stats {
display: flex;
gap: 1.5rem;
flex-wrap: wrap;
align-items: center;
margin-bottom: 0.75rem;
}
</style>
<div class="form-container">
<div class="d-flex justify-content-between align-items-center mb-2">
<h1 class="mb-0">Live Results</h1>
<div>
<a id="analyzeBtn" href="#" class="btn btn-warning" style="display:none;">Analyze</a>
<button type="button" id="stopBtn" class="btn btn-danger">Stop</button>
</div>
</div>
<div class="live-stats">
<span id="live-status" class="text-muted">Starting…</span>
<span>Checked: <strong id="stat-checked">0</strong> / <span id="stat-total">?</span></span>
<span>Found: <strong id="stat-found">0</strong></span>
</div>
<div class="progress mb-3" style="height: 8px;">
<div id="live-progress" class="progress-bar" role="progressbar" style="width: 0%;"></div>
</div>
<div id="graph"></div>
<p class="text-muted mt-2" style="font-size: 0.85rem;">Double-click a site node to open the profile.</p>
</div>
<script src="https://cdn.jsdelivr.net/npm/vis-network@9.1.9/standalone/umd/vis-network.min.js"></script>
<script>
const jobId = {{ job_id|tojson }};
const doneRedirect = {{ done_redirect|tojson }};
let nodes = null, edges = null, network = null;
let foundCount = 0;
function ensureGraph() {
if (network) return;
nodes = new vis.DataSet();
edges = new vis.DataSet();
network = new vis.Network(document.getElementById('graph'), { nodes, edges }, {
nodes: { shape: 'dot', size: 14, font: { color: '#dee2e6' } },
edges: { color: { color: '#555' }, smooth: false },
physics: {
solver: 'forceAtlas2Based',
forceAtlas2Based: { gravitationalConstant: -60, springLength: 110 },
stabilization: { iterations: 150 },
},
groups: {
username: { color: '#f9cf00', shape: 'star', size: 22 },
site: { color: '#28a745' },
data: { color: '#214e7b', shape: 'box' },
},
});
network.on('doubleClick', function (params) {
if (params.nodes.length) {
const n = nodes.get(params.nodes[0]);
if (n && n.url) window.open(n.url, '_blank');
}
});
}
function addNode(id, label, group, extra) {
if (!nodes.get(id)) nodes.add(Object.assign({ id, label, group }, extra || {}));
}
function addEdge(from, to) {
const id = from + '->' + to;
if (!edges.get(id)) edges.add({ id, from, to });
}
// Noisy/derived keys that add clutter without identity value (mirrors the
// report graph's filtering intent).
const SKIP_ID = /(_count$|^is_|_at$|image|avatar|thumbnail)/i;
function shortLabel(s) {
s = String(s);
return s.length > 28 ? s.slice(0, 27) + '…' : s;
}
function addFound(ev) {
ensureGraph();
const uId = 'u:' + ev.username.toLowerCase();
addNode(uId, ev.username, 'username');
const sId = 's:' + ev.site;
addNode(sId, ev.site, 'site', { url: ev.url, title: ev.url });
addEdge(uId, sId);
for (const [k, v] of Object.entries(ev.ids || {})) {
if (SKIP_ID.test(k) || String(v).length > 80) continue;
const vId = k + ':' + String(v).toLowerCase();
addNode(vId, shortLabel(v), 'data', { title: k + ': ' + v });
addEdge(sId, vId);
}
}
function finish(statusText, redirect) {
document.getElementById('live-status').textContent = statusText;
document.getElementById('live-progress').style.width = '100%';
document.getElementById('stopBtn').disabled = true;
if (redirect) {
const btn = document.getElementById('analyzeBtn');
btn.href = redirect;
btn.style.display = 'inline-block';
}
}
function stopScan() {
fetch('/api/scan/' + jobId + '/stop', { method: 'POST' });
document.getElementById('live-status').textContent = 'Stopping…';
document.getElementById('stopBtn').disabled = true;
}
function onScanEvent(e) {
const ev = JSON.parse(e.data);
const statusEl = document.getElementById('live-status');
if (ev.type === 'start') {
document.getElementById('stat-total').textContent = ev.total;
statusEl.textContent = 'Scanning ' + ev.username + '…';
} else if (ev.type === 'progress') {
document.getElementById('stat-checked').textContent = ev.checked;
if (ev.total) {
const pct = Math.min(ev.checked, ev.total) / ev.total * 100;
document.getElementById('live-progress').style.width = pct + '%';
}
} else if (ev.type === 'found') {
foundCount++;
document.getElementById('stat-found').textContent = foundCount;
addFound(ev);
} else if (ev.type === 'stopped') {
statusEl.textContent = 'Stopped.';
} else if (ev.type === 'error') {
statusEl.textContent = 'Error: ' + ev.message;
} else if (ev.type === 'done') {
const text = ev.redirect
? 'Completed — ' + foundCount + ' accounts found.'
: 'Completed — nothing to analyze.';
finish(text, ev.redirect);
}
}
document.getElementById('stopBtn').addEventListener('click', stopScan);
if (doneRedirect) {
finish('Completed — ' + foundCount + ' accounts found.', doneRedirect);
} else {
const evtSource = new EventSource('/api/scan/' + jobId + '/stream');
evtSource.onmessage = onScanEvent;
}
</script>
{% endblock %}
maigret/web/templates/results.html
+5 -13
View File
@@ -78,15 +78,7 @@
<div class="form-container">
<h1 class="mb-4">Search Results</h1>
<!-- Flash messages -->
{% with messages = get_flashed_messages() %}
{% if messages %}
{% for message in messages %}
<div class="alert alert-info">{{ message }}</div>
{% endfor %}
{% endif %}
{% endwith %}
<p>The search has completed. <a href="{{ url_for('index')}}">Back to start.</a></p>
{% if graph_file %}
@@ -109,10 +101,10 @@
</div>
<div id="report-{{ loop.index }}" class="report-content">
<p>
<a href="{{ url_for('download_report', filename=report.csv_file) }}">CSV Report</a> |
<a href="{{ url_for('download_report', filename=report.json_file) }}">JSON Report</a> |
<a href="{{ url_for('download_report', filename=report.pdf_file) }}">PDF Report</a> |
<a href="{{ url_for('download_report', filename=report.html_file) }}">HTML Report</a>
<a href="{{ url_for('download_report', filename=report.csv_file) }}" target="_blank" rel="noopener">CSV Report</a> |
<a href="{{ url_for('download_report', filename=report.json_file) }}" target="_blank" rel="noopener">JSON Report</a> |
<a href="{{ url_for('download_report', filename=report.pdf_file) }}" target="_blank" rel="noopener">PDF Report</a> |
<a href="{{ url_for('download_report', filename=report.html_file) }}" target="_blank" rel="noopener">HTML Report</a>
</p>
{% if report.claimed_profiles %}
<strong>Claimed Profiles:</strong>
tests/test_executors.py
+44
View File
@@ -3,6 +3,7 @@
import pytest
import asyncio
import logging
import time
from typing import List, Tuple, Callable
from maigret.executors import AsyncioQueueGeneratorExecutor
@@ -14,6 +15,19 @@ async def func(n):
return n
async def slow_cleanup_func(n, cleanup_time, **kwargs):
"""Never finishes on its own; its cancellation cleanup is itself slow —
simulates closing an HTTP session on a connection bot protection is
holding open without completing. Accepts **kwargs the same way
check_site_for_username does, since worker() calls f(*args, **kwargs)
with the same dict it later reads 'default' out of."""
try:
await asyncio.sleep(100)
return n
finally:
await asyncio.sleep(cleanup_time)
@pytest.mark.asyncio
async def test_asyncio_queue_generator_executor():
tasks: List[Tuple[Callable, list, dict]] = [(func, [n], {}) for n in range(10)]
@@ -44,3 +58,33 @@ async def test_asyncio_queue_generator_executor():
assert results == [0, 3, 6, 9, 1, 4, 7, 2, 5, 8]
assert executor.execution_time > 0.2
assert executor.execution_time < 1.0
@pytest.mark.asyncio
async def test_worker_does_not_block_on_slow_cancellation_cleanup():
"""A task whose cancellation cleanup itself hangs (e.g. closing a
session on a connection bot protection holds open without completing)
must not make the worker wait past `timeout` for that cleanup to
finish — see the asyncio.wait() vs wait_for() comment in worker()."""
cleanup_time = 0.4
per_task_timeout = 0.15
tasks: List[Tuple[Callable, list, dict]] = [
(slow_cleanup_func, [n, cleanup_time], {'default': f'default-{n}'})
for n in range(3)
]
executor = AsyncioQueueGeneratorExecutor(
logger=logger, in_parallel=3, timeout=per_task_timeout
)
start = time.monotonic()
results = [result async for result in executor.run(tasks)] # type: ignore[arg-type]
elapsed = time.monotonic() - start
assert sorted(results) == ['default-0', 'default-1', 'default-2']
# Must return close to per_task_timeout, not cleanup_time — a
# wait_for()-based implementation blocks until cleanup_time instead.
assert elapsed < cleanup_time
# Let the orphaned cleanup tasks actually finish before the test's event
# loop closes, so they don't leak past this test as pending-task warnings.
await asyncio.sleep(cleanup_time)
tests/test_web.py
+525 -6
View File
@@ -6,16 +6,20 @@ internals are mocked; the report-generation smoke test keeps `save_graph_report`
unmocked so regressions like `nt.options.groups = ...` (AttributeError on a
plain dict) are caught automatically.
"""
import asyncio
import json
import os
import types
import pytest
import maigret
import maigret.report
import maigret.settings
from maigret.result import MaigretCheckResult, MaigretCheckStatus
from maigret.web import app as web_app_module
CUR_PATH = os.path.dirname(os.path.realpath(__file__))
TEST_DB = os.path.join(CUR_PATH, 'db.json')
@@ -37,6 +41,7 @@ def web_app(tmp_path):
web_app_module.app.config['TESTING'] = True
web_app_module.app.config['REPORTS_FOLDER'] = str(tmp_path)
web_app_module.app.config['MAIGRET_DB_FILE'] = TEST_DB
web_app_module.app.config['SETTINGS_FILE'] = str(tmp_path / 'web_settings.json')
web_app_module.background_jobs.clear()
web_app_module.job_results.clear()
@@ -126,6 +131,46 @@ def test_completed_search_redirects_to_results(client, web_app, monkeypatch):
assert b'soxoj' in results_resp.data
def test_results_report_links_open_in_new_tab(client, web_app, monkeypatch):
"""CSV/JSON/PDF/HTML report links must open in a new tab, not navigate away
from the results page."""
def fake_task(usernames, options, timestamp):
web_app.job_results[timestamp] = {
'status': 'completed',
'session_folder': f'search_{timestamp}',
'graph_file': f'search_{timestamp}/combined_graph.html',
'usernames': usernames,
'individual_reports': [
{
'username': 'soxoj',
'csv_file': f'search_{timestamp}/report_soxoj.csv',
'json_file': f'search_{timestamp}/report_soxoj.json',
'pdf_file': f'search_{timestamp}/report_soxoj.pdf',
'html_file': f'search_{timestamp}/report_soxoj.html',
'claimed_profiles': [],
}
],
}
web_app.background_jobs[timestamp]['completed'] = True
monkeypatch.setattr(web_app, 'process_search_task', fake_task)
monkeypatch.setattr(web_app, 'Thread', _SyncThread)
post = client.post('/search', data={'usernames': 'soxoj'})
status_resp = client.get(post.location)
results_resp = client.get(status_resp.location)
body = results_resp.get_data(as_text=True)
for label in ('CSV Report', 'JSON Report', 'PDF Report', 'HTML Report'):
# crude but effective: the link and its target="_blank" must appear
# within the same <a> tag, not just somewhere on the page.
idx = body.index(label)
tag_start = body.rindex('<a ', 0, idx)
tag = body[tag_start : idx + len(label)]
assert 'target="_blank"' in tag, f'{label} link missing target="_blank"'
def test_failed_task_redirects_to_index(client, web_app, monkeypatch):
def failing_task(usernames, options, timestamp):
web_app.job_results[timestamp] = {'status': 'failed', 'error': 'boom'}
@@ -229,9 +274,9 @@ def test_search_passes_cloudflare_bypass_from_settings(client, web_app, monkeypa
client.post('/search', data={'usernames': 'testuser'})
assert 'cloudflare_bypass' in captured, (
'maigret.search was not given a cloudflare_bypass kwarg'
)
assert (
'cloudflare_bypass' in captured
), 'maigret.search was not given a cloudflare_bypass kwarg'
cf = captured['cloudflare_bypass']
assert cf is not None
assert cf['session_prefix'] == 'test-prefix'
@@ -270,6 +315,245 @@ def test_search_omits_cloudflare_bypass_when_disabled(client, web_app, monkeypat
assert captured.get('cloudflare_bypass') is None
def test_live_scan_streams_found_and_done(client, web_app, monkeypatch):
"""POST /api/scan starts a background scan; GET .../stream yields the per-site
'found' event and a terminating 'done' event. Guards the SSE + StreamNotify wiring.
"""
async def fake_search(*args, **kwargs):
notify = kwargs['query_notify']
result = MaigretCheckResult(
username='soxoj',
site_name='GitHub',
site_url_user='https://github.com/soxoj',
status=MaigretCheckStatus.CLAIMED,
ids_data={'fullname': 'Soxoj', '_extractor': 'x'},
tags=['dev'],
)
notify.update(result)
return {'GitHub': {'status': result, 'url_user': result.site_url_user}}
monkeypatch.setattr(maigret, 'search', fake_search)
# csv/json/pdf report internals are exercised by
# test_real_report_generation_does_not_crash; here we only care that a
# completed live scan wires into the same report + results-page flow.
monkeypatch.setattr(maigret.report, 'save_csv_report', lambda *a, **kw: None)
monkeypatch.setattr(maigret.report, 'save_json_report', lambda *a, **kw: None)
monkeypatch.setattr(maigret.report, 'save_pdf_report', lambda *a, **kw: None)
monkeypatch.setattr(maigret.report, 'save_html_report', lambda *a, **kw: None)
monkeypatch.setattr(maigret.report, 'generate_report_context', lambda *a, **kw: {})
start = client.post('/api/scan', data={'usernames': 'soxoj'})
assert start.status_code == 200
job_id = start.get_json()['job_id']
body = client.get(f'/api/scan/{job_id}/stream').get_data(as_text=True)
events = [
json.loads(line[6:]) for line in body.splitlines() if line.startswith('data: ')
]
types_seen = [e['type'] for e in events]
assert 'done' in types_seen
found = [e for e in events if e['type'] == 'found']
assert found and found[0]['site'] == 'GitHub'
# _extractor metadata is stripped from the graph payload
assert '_extractor' not in found[0]['ids']
assert found[0]['ids']['fullname'] == 'Soxoj'
# Regression guard: a completed live scan must still produce the same
# report files + profile list as the classic /search flow, and hand the
# browser a redirect to the results page that shows them.
done_event = next(e for e in events if e['type'] == 'done')
assert done_event['redirect'] == f'/results/search_{job_id}'
result = web_app.job_results[job_id]
assert result['status'] == 'completed'
reports = result['individual_reports']
assert reports and reports[0]['username'] == 'soxoj'
assert reports[0]['claimed_profiles'][0]['site_name'] == 'GitHub'
results_page = client.get(done_event['redirect']).get_data(as_text=True)
assert 'GitHub' in results_page
assert 'CSV Report' in results_page
def test_live_scan_empty_username_rejected(client, web_app):
resp = client.post('/api/scan', data={'usernames': ''})
assert resp.status_code == 400
def test_live_scan_stop_unknown_job_404(client, web_app):
resp = client.post('/api/scan/nope/stop')
assert resp.status_code == 404
def test_live_start_empty_username_redirects_to_index(client, web_app):
resp = client.post('/live', data={'usernames': ''})
assert resp.status_code == 302
assert resp.location.endswith('/')
def test_live_start_redirects_to_dedicated_live_page(client, web_app, monkeypatch):
"""POST /live starts a job on a NEW page (/live/<job_id>), not inline on
the index page. That page must show the graph + a Stop button, and must
NOT unconditionally redirect away on completion (only via the Analyze
button — see test_live_scan_done_event_offers_redirect_not_auto_navigation)."""
async def fake_search(*args, **kwargs):
notify = kwargs['query_notify']
notify.set_total(0)
return {}
monkeypatch.setattr(maigret, 'search', fake_search)
start = client.post('/live', data={'usernames': 'soxoj'})
assert start.status_code == 302
assert start.location.startswith('/live/')
job_id = start.location.rsplit('/', 1)[1]
page = client.get(start.location)
assert page.status_code == 200
body = page.get_data(as_text=True)
assert 'id="graph"' in body
assert 'id="stopBtn"' in body
assert 'id="analyzeBtn"' in body
assert job_id in body
# No unconditional navigation on completion anymore.
assert 'window.location.href = ev.redirect' not in body
# Drain the SSE stream so the background thread's queue is consumed and
# the job entry is cleaned up tidily.
client.get(f'/api/scan/{job_id}/stream')
def test_live_results_unknown_job_redirects_to_index(client, web_app):
resp = client.get('/live/does-not-exist')
assert resp.status_code == 302
assert resp.location.endswith('/')
def test_live_results_for_finished_job_skips_sse_and_shows_analyze(client, web_app):
"""If the job already finished (e.g. the user reloaded the Live Results
page), the page must offer the Analyze redirect immediately instead of
trying to reopen a dead SSE stream."""
web_app.job_results['finishedjob'] = {
'status': 'completed',
'session_folder': 'search_finishedjob',
'graph_file': 'search_finishedjob/combined_graph.html',
'usernames': ['soxoj'],
'individual_reports': [],
'found_count': 0,
}
resp = client.get('/live/finishedjob')
assert resp.status_code == 200
body = resp.get_data(as_text=True)
assert 'const doneRedirect = "/results/search_finishedjob";' in body
def test_live_scan_done_event_offers_redirect_not_auto_navigation(
client, web_app, monkeypatch
):
"""The SSE 'done' payload still carries the redirect URL (consumed by the
Analyze button), but nothing server- or client-side forces navigation."""
async def fake_search(*args, **kwargs):
notify = kwargs['query_notify']
result = MaigretCheckResult(
username='soxoj',
site_name='GitHub',
site_url_user='https://github.com/soxoj',
status=MaigretCheckStatus.CLAIMED,
ids_data={},
)
notify.update(result)
return {'GitHub': {'status': result, 'url_user': result.site_url_user}}
monkeypatch.setattr(maigret, 'search', fake_search)
monkeypatch.setattr(maigret.report, 'save_graph_report', lambda *a, **kw: None)
monkeypatch.setattr(maigret.report, 'save_csv_report', lambda *a, **kw: None)
monkeypatch.setattr(maigret.report, 'save_json_report', lambda *a, **kw: None)
monkeypatch.setattr(maigret.report, 'save_pdf_report', lambda *a, **kw: None)
monkeypatch.setattr(maigret.report, 'save_html_report', lambda *a, **kw: None)
monkeypatch.setattr(maigret.report, 'generate_report_context', lambda *a, **kw: {})
start = client.post('/live', data={'usernames': 'soxoj'})
job_id = start.location.rsplit('/', 1)[1]
body = client.get(f'/api/scan/{job_id}/stream').get_data(as_text=True)
events = [
json.loads(line[6:]) for line in body.splitlines() if line.startswith('data: ')
]
done_event = next(e for e in events if e['type'] == 'done')
assert done_event['redirect'] == f'/results/search_{job_id}'
result = web_app.job_results[job_id]
assert result['status'] == 'completed'
assert result['found_count'] == 1
assert 'started_at' in result
def test_live_scan_stop_mid_scan_keeps_already_found_results(
client, web_app, monkeypatch
):
"""Regression: clicking Stop while a username's scan is still in-flight
used to discard every 'found' result already streamed to the live graph,
because the cancelled search() task never returns its own results dict —
general_results stayed empty, build_reports never ran, and the browser
got 'Completed — nothing to analyze.' despite the graph showing hits.
StreamNotify now keeps a running copy of what it already streamed, and
that's what gets reported when the task is cancelled mid-scan.
"""
async def fake_search(*args, **kwargs):
notify = kwargs['query_notify']
assert 'ValidActive' in notify.sites, 'site map not wired into StreamNotify'
found = MaigretCheckResult(
username='soxoj',
site_name='ValidActive',
site_url_user='https://play.google.com/store/apps/developer?id=soxoj',
status=MaigretCheckStatus.CLAIMED,
)
notify.update(found)
# Simulate task.cancel() firing mid-scan, after this one site was
# already checked and streamed to the browser but before the other
# (still in-flight) sites finished.
raise asyncio.CancelledError()
monkeypatch.setattr(maigret, 'search', fake_search)
monkeypatch.setattr(maigret.report, 'save_graph_report', lambda *a, **kw: None)
monkeypatch.setattr(maigret.report, 'save_csv_report', lambda *a, **kw: None)
monkeypatch.setattr(maigret.report, 'save_json_report', lambda *a, **kw: None)
monkeypatch.setattr(maigret.report, 'save_pdf_report', lambda *a, **kw: None)
monkeypatch.setattr(maigret.report, 'save_html_report', lambda *a, **kw: None)
monkeypatch.setattr(maigret.report, 'generate_report_context', lambda *a, **kw: {})
start = client.post('/live', data={'usernames': 'soxoj'})
job_id = start.location.rsplit('/', 1)[1]
body = client.get(f'/api/scan/{job_id}/stream').get_data(as_text=True)
events = [
json.loads(line[6:]) for line in body.splitlines() if line.startswith('data: ')
]
types_seen = [e['type'] for e in events]
assert 'stopped' in types_seen
found = [e for e in events if e['type'] == 'found']
assert found and found[0]['site'] == 'ValidActive'
done_event = next(e for e in events if e['type'] == 'done')
assert (
done_event.get('redirect') == f'/results/search_{job_id}'
), "Stop must not discard already-found results ('nothing to analyze' bug)"
result = web_app.job_results[job_id]
assert result['status'] == 'completed'
assert result['found_count'] == 1
assert result['individual_reports'][0]['claimed_profiles'][0]['site_name'] == (
'ValidActive'
)
def test_real_report_generation_does_not_crash(client, web_app, monkeypatch):
"""End-to-end with mocked maigret.search but REAL report generation.
@@ -297,6 +581,241 @@ def test_real_report_generation_does_not_crash(client, web_app, monkeypatch):
assert timestamp in web_app.job_results, 'background task did not record any result'
result = web_app.job_results[timestamp]
assert result['status'] == 'completed', (
f"report generation failed: {result.get('error')!r}"
assert (
result['status'] == 'completed'
), f"report generation failed: {result.get('error')!r}"
# Regression guard: pyvis's default cdn_resources="local" writes a lib/
# folder relative to the process cwd instead of next to the graph HTML,
# so the browser 404s fetching lib/bindings/utils.js from /reports/...
graph_path = os.path.join(web_app.app.config['REPORTS_FOLDER'], result['graph_file'])
with open(graph_path, encoding='utf-8') as f:
graph_html = f.read()
assert 'lib/bindings' not in graph_html
assert not os.path.exists(os.path.join(os.path.dirname(graph_path), 'lib'))
def test_history_empty_state(client, web_app):
resp = client.get('/history')
assert resp.status_code == 200
assert 'No searches have been run yet.' in resp.get_data(as_text=True)
def test_history_link_present_on_every_page(client, web_app):
resp = client.get('/')
body = resp.get_data(as_text=True)
assert 'href="/history"' in body
def test_new_search_link_present_on_every_page(client, web_app):
resp = client.get('/history')
body = resp.get_data(as_text=True)
assert 'New Search' in body
assert 'href="/"' in body
def test_history_lists_completed_and_failed_runs(client, web_app):
web_app.job_results['ts_completed'] = {
'status': 'completed',
'session_folder': 'search_ts_completed',
'graph_file': 'search_ts_completed/combined_graph.html',
'usernames': ['soxoj', 'alice'],
'individual_reports': [],
'found_count': 7,
'started_at': '2026-07-28 10:00:00',
}
web_app.job_results['ts_failed'] = {
'status': 'failed',
'error': 'boom',
'usernames': ['bob'],
'started_at': '2026-07-28 09:00:00',
}
resp = client.get('/history')
assert resp.status_code == 200
body = resp.get_data(as_text=True)
assert '2026-07-28 10:00:00' in body
assert 'soxoj, alice' in body
assert '>7<' in body
assert 'completed' in body
assert '/results/search_ts_completed' in body
assert '2026-07-28 09:00:00' in body
assert 'bob' in body
assert 'failed' in body
# Newest run listed first.
assert body.index('search_ts_completed') < body.index('bob')
def test_build_reports_computes_found_count(web_app, monkeypatch):
"""Regression guard: History reads `found_count` off the dict build_reports
returns, so it must count claimed profiles across all usernames."""
monkeypatch.setattr(maigret.report, 'save_csv_report', lambda *a, **kw: None)
monkeypatch.setattr(maigret.report, 'save_json_report', lambda *a, **kw: None)
monkeypatch.setattr(maigret.report, 'save_pdf_report', lambda *a, **kw: None)
monkeypatch.setattr(maigret.report, 'save_html_report', lambda *a, **kw: None)
monkeypatch.setattr(maigret.report, 'generate_report_context', lambda *a, **kw: {})
claimed = MaigretCheckResult(
username='soxoj',
site_name='GitHub',
site_url_user='https://github.com/soxoj',
status=MaigretCheckStatus.CLAIMED,
)
general_results = [
(
'soxoj',
'username',
{'GitHub': {'status': claimed, 'url_user': claimed.site_url_user}},
)
]
report = web_app.build_reports(general_results, ['soxoj'], 'testkey')
assert report['found_count'] == 1
assert report['individual_reports'][0]['claimed_profiles'][0]['site_name'] == 'GitHub'
def test_process_search_task_records_started_at_on_success(web_app, monkeypatch):
async def fake_search_multi(usernames, options):
return []
monkeypatch.setattr(web_app, 'search_multiple_usernames', fake_search_multi)
monkeypatch.setattr(
web_app,
'build_reports',
lambda *a, **kw: {
'status': 'completed',
'session_folder': 'x',
'graph_file': 'x',
'usernames': [],
'individual_reports': [],
'found_count': 0,
},
)
web_app.background_jobs['ts_ok'] = {'completed': False, 'thread': None}
web_app.process_search_task(['soxoj'], {}, 'ts_ok')
assert web_app.job_results['ts_ok']['status'] == 'completed'
assert web_app.job_results['ts_ok']['started_at']
def test_process_search_task_records_started_at_on_failure(web_app, monkeypatch):
async def failing_search_multi(usernames, options):
raise RuntimeError('boom')
monkeypatch.setattr(web_app, 'search_multiple_usernames', failing_search_multi)
web_app.background_jobs['ts_fail'] = {'completed': False, 'thread': None}
web_app.process_search_task(['soxoj'], {}, 'ts_fail')
assert web_app.job_results['ts_fail']['status'] == 'failed'
assert web_app.job_results['ts_fail']['started_at']
def test_load_settings_defaults_when_no_file(web_app):
settings = web_app.load_settings()
assert settings['timeout'] == 10
assert settings['top_sites'] == 500
assert settings['tags'] == []
assert settings['proxy'] == ''
assert settings['permute'] is False
def test_save_settings_persists_to_file_and_reloads(web_app):
web_app.save_settings(
{**web_app.DEFAULT_SETTINGS, 'timeout': 42, 'proxy': '127.0.0.1:9999'}
)
assert os.path.exists(web_app.app.config['SETTINGS_FILE'])
reloaded = web_app.load_settings()
assert reloaded['timeout'] == 42
assert reloaded['proxy'] == '127.0.0.1:9999'
def test_settings_update_saves_and_redirects_back(client, web_app):
resp = client.post(
'/settings',
data={
'timeout': '15',
'top_sites': '250',
'tags': ['coding', 'tech'],
'excluded_tags': ['porn'],
'site': 'GitHub, Reddit',
'proxy': '127.0.0.1:1080',
'permute': 'on',
'with_domains': 'on',
},
headers={'Referer': '/history'},
)
assert resp.status_code == 302
assert resp.headers['Location'] == '/history'
settings = web_app.load_settings()
assert settings['timeout'] == 15
assert settings['top_sites'] == 250
assert settings['tags'] == ['coding', 'tech']
assert settings['excluded_tags'] == ['porn']
assert settings['site_list'] == ['GitHub', 'Reddit']
assert settings['proxy'] == '127.0.0.1:1080'
assert settings['permute'] is True
assert settings['with_domains'] is True
assert settings['disable_recursive_search'] is False
def test_settings_update_invalid_timeout_falls_back_to_default(client, web_app):
client.post('/settings', data={'timeout': 'not-a-number', 'top_sites': 'nope'})
settings = web_app.load_settings()
assert settings['timeout'] == web_app.DEFAULT_SETTINGS['timeout']
assert settings['top_sites'] == web_app.DEFAULT_SETTINGS['top_sites']
def test_parse_search_options_uses_saved_settings(web_app):
web_app.save_settings(
{
**web_app.DEFAULT_SETTINGS,
'timeout': 20,
'top_sites': 100,
'proxy': '127.0.0.1:8080',
'tags': ['gaming'],
'site_list': ['GitHub'],
'disable_extracting': True,
}
)
options = web_app.parse_search_options({})
assert options['timeout'] == 20
assert options['top_sites'] == 100
assert options['proxy'] == '127.0.0.1:8080'
assert options['tags'] == ['gaming']
assert options['site_list'] == ['GitHub']
assert options['disable_extracting'] is True
assert options['all_sites'] is False
def test_parse_search_options_full_mode_ignores_top_sites(web_app):
options = web_app.parse_search_options({'mode': 'full'})
assert options['all_sites'] is True
def test_api_sites_returns_site_list(client, web_app):
resp = client.get('/api/sites')
assert resp.status_code == 200
data = resp.get_json()
assert 'sites' in data
assert isinstance(data['sites'], list)
def test_settings_modal_present_on_every_page(client, web_app):
resp = client.get('/')
body = resp.get_data(as_text=True)
assert 'id="settingsModal"' in body
assert 'name="timeout"' in body
resp = client.get('/history')
body = resp.get_data(as_text=True)
assert 'id="settingsModal"' in body