From 5e8b1c02c74ee14a203375f7ce6b8c559fcb1dd2 Mon Sep 17 00:00:00 2001 From: haikdc Date: Mon, 30 Mar 2026 20:24:28 -0700 Subject: [PATCH] [Haik]: added in better linter from the debugger. This good for now till the linter pip module done. Also i just peeked at the output and there are 799 linter errors, holy SLOP. --- .vscode/extensions.json | 5 + .vscode/settings.json | 3 +- .vscode/tasks.json | 138 +++++++++++++++++---- backend/pyproject.toml | 1 + backend/uv.lock | 14 +++ frontend/eslint.config.mjs | 26 ++++ frontend/knip.json | 3 + frontend/package.json | 10 +- frontend/tsconfig.json | 2 +- linter/README.md | 138 +++++++++++++++------ linter/checks/__init__.py | 22 ++++ linter/checks/eslint.py | 45 +++++++ linter/checks/knip.py | 61 ++++++++++ linter/checks/structural.py | 104 ++++++++++++++++ linter/checks/vulture.py | 55 +++++++++ linter/config/config.json | 43 +++++++ linter/config/pyrightconfig.json | 14 +++ linter/config/vulture_whitelist.py | 24 ++++ linter/lint.py | 145 ++++++++++++++++++++++ linter/print_errors.sh | 64 ++++++++++ linter/pyrightconfig.json | 14 --- linter/structlint.json | 28 ----- linter/structlint.py | 187 ----------------------------- 23 files changed, 851 insertions(+), 295 deletions(-) create mode 100644 .vscode/extensions.json create mode 100644 frontend/eslint.config.mjs create mode 100644 frontend/knip.json create mode 100644 linter/checks/__init__.py create mode 100644 linter/checks/eslint.py create mode 100644 linter/checks/knip.py create mode 100644 linter/checks/structural.py create mode 100644 linter/checks/vulture.py create mode 100644 linter/config/config.json create mode 100644 linter/config/pyrightconfig.json create mode 100644 linter/config/vulture_whitelist.py create mode 100644 linter/lint.py create mode 100755 linter/print_errors.sh delete mode 100644 linter/pyrightconfig.json delete mode 100644 linter/structlint.json delete mode 100644 linter/structlint.py diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 00000000..b308e589 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,5 @@ +{ + "recommendations": [ + "dbaeumer.vscode-eslint" + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json index bd57e8c7..a95a04c0 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -10,5 +10,6 @@ ], "python.analysis.diagnosticSeverityOverrides": { "reportMissingTypeStubs": "none" - } + }, + "eslint.workingDirectories": ["frontend"] } diff --git a/.vscode/tasks.json b/.vscode/tasks.json index d7d7dfc3..1f833712 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -2,10 +2,10 @@ "version": "2.0.0", "tasks": [ { - "label": "structlint:watch", + "label": "lint:watch", "type": "shell", "command": "python3", - "args": ["${workspaceFolder}/linter/structlint.py", "--watch", "--root", "${workspaceFolder}"], + "args": ["${workspaceFolder}/linter/lint.py", "--watch", "--root", "${workspaceFolder}"], "isBackground": true, "runOptions": { "runOn": "folderOpen" @@ -15,40 +15,128 @@ "panel": "dedicated", "close": true }, - "problemMatcher": { - "owner": "structlint", - "fileLocation": ["relative", "${workspaceFolder}"], - "pattern": { - "regexp": "^(.+):(\\d+):(\\d+):\\s+(error|warning):\\s+(.+)$", - "file": 1, - "line": 2, - "column": 3, - "severity": 4, - "message": 5 + "problemMatcher": [ + { + "owner": "structural", + "fileLocation": ["relative", "${workspaceFolder}"], + "pattern": { + "regexp": "^(.+):(\\d+):(\\d+):\\s+(error|warning):\\s+(.+)$", + "file": 1, + "line": 2, + "column": 3, + "severity": 4, + "message": 5 + }, + "background": { + "activeOnStart": true, + "beginsPattern": "^structural: checking\\.\\.\\.$", + "endsPattern": "^structural: done\\." + } }, - "background": { - "activeOnStart": true, - "beginsPattern": "^structlint: checking\\.\\.\\.$", - "endsPattern": "^structlint: done\\." + { + "owner": "vulture", + "fileLocation": ["relative", "${workspaceFolder}"], + "pattern": { + "regexp": "^(.+):(\\d+):(\\d+):\\s+(error|warning):\\s+(.+)$", + "file": 1, + "line": 2, + "column": 3, + "severity": 4, + "message": 5 + }, + "background": { + "activeOnStart": true, + "beginsPattern": "^vulture: checking\\.\\.\\.$", + "endsPattern": "^vulture: done\\." + } + }, + { + "owner": "eslint-project", + "fileLocation": ["relative", "${workspaceFolder}"], + "pattern": { + "regexp": "^(.+):(\\d+):(\\d+):\\s+(error|warning):\\s+(.+)$", + "file": 1, + "line": 2, + "column": 3, + "severity": 4, + "message": 5 + }, + "background": { + "activeOnStart": true, + "beginsPattern": "^eslint: checking\\.\\.\\.$", + "endsPattern": "^eslint: done\\." + } + }, + { + "owner": "knip", + "fileLocation": ["relative", "${workspaceFolder}"], + "pattern": { + "regexp": "^(.+):(\\d+):(\\d+):\\s+(error|warning):\\s+(.+)$", + "file": 1, + "line": 2, + "column": 3, + "severity": 4, + "message": 5 + }, + "background": { + "activeOnStart": true, + "beginsPattern": "^knip: checking\\.\\.\\.$", + "endsPattern": "^knip: done\\." + } } - } + ] }, { - "label": "structlint:check", + "label": "lint:check", "type": "shell", "command": "python3", - "args": ["${workspaceFolder}/linter/structlint.py", "--root", "${workspaceFolder}"], + "args": ["${workspaceFolder}/linter/lint.py", "--root", "${workspaceFolder}"], + "problemMatcher": [ + { + "owner": "structural", + "fileLocation": ["relative", "${workspaceFolder}"], + "pattern": { + "regexp": "^(.+):(\\d+):(\\d+):\\s+(error|warning):\\s+(.+)$", + "file": 1, + "line": 2, + "column": 3, + "severity": 4, + "message": 5 + } + }, + { + "owner": "eslint-project", + "fileLocation": ["relative", "${workspaceFolder}"], + "pattern": { + "regexp": "^(.+):(\\d+):(\\d+):\\s+(error|warning):\\s+(.+)$", + "file": 1, + "line": 2, + "column": 3, + "severity": 4, + "message": 5 + } + } + ] + }, + { + "label": "knip:check", + "type": "shell", + "command": "npm", + "args": ["run", "knip"], + "options": { + "cwd": "${workspaceFolder}/frontend" + }, "problemMatcher": { - "owner": "structlint", - "fileLocation": ["relative", "${workspaceFolder}"], + "owner": "knip", + "fileLocation": ["relative", "${workspaceFolder}/frontend"], "pattern": { - "regexp": "^(.+):(\\d+):(\\d+):\\s+(error|warning):\\s+(.+)$", + "regexp": "^(.+):(\\d+):(\\d+)\\s+(.+)$", "file": 1, "line": 2, "column": 3, - "severity": 4, - "message": 5 - } + "message": 4 + }, + "severity": "warning" } } ] diff --git a/backend/pyproject.toml b/backend/pyproject.toml index b9fc5edf..61fe7b55 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -20,6 +20,7 @@ dev = [ "pytest==8.3.4", "pytest-asyncio==0.25.2", "typeguard==4.4.2", + "vulture", ] [tool.uv] diff --git a/backend/uv.lock b/backend/uv.lock index e182e88f..1896bae4 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -894,6 +894,7 @@ dev = [ { name = "pytest" }, { name = "pytest-asyncio" }, { name = "typeguard" }, + { name = "vulture" }, ] [package.metadata] @@ -915,6 +916,7 @@ dev = [ { name = "pytest", specifier = "==8.3.4" }, { name = "pytest-asyncio", specifier = "==0.25.2" }, { name = "typeguard", specifier = "==4.4.2" }, + { name = "vulture" }, ] [[package]] @@ -1883,6 +1885,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e4/16/c1fd27e9549f3c4baf1dc9c20c456cd2f822dbf8de9f463824b0c0357e06/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6cde23eeda1a25c75b2e07d39970f3374105d5eafbaab2a4482be82f272d5a5e", size = 4296730, upload-time = "2025-10-16T22:17:00.744Z" }, ] +[[package]] +name = "vulture" +version = "2.16" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/3e/4d08c5903b2c0c70cad583c170cc4a663fc6a61e2ad00b711fcda61358cd/vulture-2.16.tar.gz", hash = "sha256:f8d9f6e2af03011664a3c6c240c9765b3f392917d3135fddca6d6a68d359f717", size = 52680, upload-time = "2026-03-25T14:41:27.141Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/be/f935130312330614811dae2ea9df3f395f6d63889eb6c2e68c14507152ee/vulture-2.16-py3-none-any.whl", hash = "sha256:6e0f1c312cef1c87856957e5c2ca9608834a7c794c2180477f30bf0e4cc58eee", size = 26993, upload-time = "2026-03-25T14:41:26.21Z" }, +] + [[package]] name = "watchfiles" version = "1.1.1" diff --git a/frontend/eslint.config.mjs b/frontend/eslint.config.mjs new file mode 100644 index 00000000..6f90f3c3 --- /dev/null +++ b/frontend/eslint.config.mjs @@ -0,0 +1,26 @@ +import eslint from "@eslint/js"; +import tseslint from "typescript-eslint"; +import reactHooks from "eslint-plugin-react-hooks"; + +export default tseslint.config( + eslint.configs.recommended, + ...tseslint.configs.recommended, + { + plugins: { + "react-hooks": reactHooks, + }, + rules: { + ...reactHooks.configs.recommended.rules, + "@typescript-eslint/no-unused-vars": [ + "error", + { + argsIgnorePattern: "^_", + varsIgnorePattern: "^_", + }, + ], + }, + }, + { + ignores: ["dist/", "webpack.config.js"], + }, +); diff --git a/frontend/knip.json b/frontend/knip.json new file mode 100644 index 00000000..d931f5f9 --- /dev/null +++ b/frontend/knip.json @@ -0,0 +1,3 @@ +{ + "project": ["src/**/*.{ts,tsx}"] +} diff --git a/frontend/package.json b/frontend/package.json index c2fb563c..1460e4c0 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -6,7 +6,10 @@ "build": "webpack --mode=production", "build:watch": "webpack --mode=development --watch", "dev": "webpack serve --mode=development", - "clean": "rm -rf dist" + "clean": "rm -rf dist", + "lint": "eslint src/", + "lint:fix": "eslint src/ --fix", + "knip": "knip" }, "dependencies": { "@assistant-ui/react": "^0.12.21", @@ -20,6 +23,7 @@ "@codemirror/view": "^6.39.16", "@emotion/react": "^11.14.0", "@emotion/styled": "^11.14.1", + "@eslint/js": "^9.39.4", "@mui/icons-material": "^7.3.9", "@mui/material": "^7.3.9", "@pierre/diffs": "^1.1.7", @@ -56,7 +60,10 @@ "copy-webpack-plugin": "^14.0.0", "css-loader": "^6.8.0", "css-modules-types-loader": "^0.6.10", + "eslint": "^9.39.4", + "eslint-plugin-react-hooks": "^7.0.1", "html-webpack-plugin": "^5.5.0", + "knip": "^6.1.0", "postcss": "^8.5.8", "postcss-loader": "^8.2.1", "sass": "^1.89.2", @@ -64,6 +71,7 @@ "style-loader": "^3.3.0", "tailwindcss": "^4.2.2", "typescript": "^5.0.0", + "typescript-eslint": "^8.58.0", "webpack": "^5.88.0", "webpack-cli": "^5.1.0", "webpack-dev-server": "^4.15.0" diff --git a/frontend/tsconfig.json b/frontend/tsconfig.json index 31de931e..29b254c2 100644 --- a/frontend/tsconfig.json +++ b/frontend/tsconfig.json @@ -17,6 +17,6 @@ "@/*": ["./src/*"] } }, - "include": ["**/*.ts", "**/*.tsx"], + "include": ["src/**/*.ts", "src/**/*.tsx"], "exclude": ["node_modules"] } \ No newline at end of file diff --git a/linter/README.md b/linter/README.md index 75d168f8..e22555c0 100644 --- a/linter/README.md +++ b/linter/README.md @@ -1,81 +1,143 @@ -# Structural Linter +# Code Quality Tools -A linter is a tool that automatically checks your code for problems. This one doesn't check for bugs — it enforces **structural rules** that keep the codebase organized and easy to navigate. +This folder contains the project's code quality tooling: a structural linter, dead code detection, and type checking — covering both the Python backend and TypeScript frontend. -## What it checks +## What gets checked -**1. File length** — Every source file must be under 250 lines. +### Structural rules -Big files are hard to read, hard to review, and hard to maintain. If a file is getting long, it's a sign it should be split into smaller, more focused pieces. +**File length** — Every source file must be under 250 lines. Big files are hard to read, review, and maintain. If a file is getting long, it's a sign it should be split. -**2. Folder size** — Every folder must contain fewer than 6 items (files or subfolders). +**Folder size** — Every folder must contain fewer than 6 items. Keeping folders small forces you to organize code into logical groups. -When a folder has dozens of files it becomes a junk drawer. Keeping folders small forces you to organize code into logical groups. +**Unused Python code (Vulture)** — Flags unused functions, classes, variables, and imports in the backend. Integrated into the linter's watch loop — findings appear as warnings in the Problems panel alongside structural errors. Only reports findings with >= 80% confidence to reduce noise. -These rules apply to `.py`, `.ts`, `.tsx`, `.js`, and `.jsx` files. Non-code files (images, JSON data, configs, lock files, etc.) are ignored. +These rules apply to `.py`, `.ts`, `.tsx`, `.js`, and `.jsx` files. + +### Unused TypeScript code + +**Per-file (ESLint)** — Catches unused variables, parameters, and imports within each file. Runs in real-time through the VS Code ESLint extension. + +**Project-wide (Knip)** — Finds unused exports, unused files, and unused `package.json` dependencies across the entire frontend. Run manually or in CI. + +### Type checking + +**Python (Pyright/Pylance)** — Strict type checking for the backend, configured via `config/pyrightconfig.json`. Works through the Pylance extension in real-time. + +**TypeScript** — The `tsconfig.json` in `frontend/` has strict mode enabled. TypeScript errors show in the editor automatically. ## How it runs -You don't need to do anything — it runs automatically. +### Linter watch (automatic) -When you open the project in Cursor/VS Code, a background task starts watching for file changes. Every time you save, it re-checks the codebase. Violations show up as errors in the **Problems panel** (`Cmd+Shift+M`) and as red badges on files in the sidebar, just like any other linter. - -If you want to run it manually from the terminal: +When you open the project in Cursor/VS Code, a background task starts watching for file changes. Every save re-checks the codebase. Violations show up in the **Problems panel** (`Cmd+Shift+M`). ```bash # one-shot check (exits with code 1 if violations exist) -python3 linter/structlint.py --root . +python3 linter/lint.py --root . # continuous watch mode -python3 linter/structlint.py --watch --root . +python3 linter/lint.py --watch --root . ``` +### ESLint (automatic) + +The VS Code ESLint extension picks up `frontend/eslint.config.mjs` and shows errors inline as you type. To run from the terminal: + +```bash +cd frontend + +# check for problems +npm run lint + +# auto-fix what's possible +npm run lint:fix +``` + +### Knip (manual / CI) + +```bash +cd frontend +npm run knip +``` + +Or use the `knip:check` VS Code task (`Cmd+Shift+P` → "Run Task" → "knip:check"). + ## Configuration -All config lives in `structlint.json` (this folder). Here's what each field does: +### config/config.json ```json { + "enabled": { + "max-file-lines": true, // toggle each check on/off + "max-folder-items": true, + "no-nested-imports": true, + "vulture": true, + "eslint": true, + "knip": true + }, "rules": { "max-file-lines": 250, // files with >= this many lines trigger an error - "max-folder-items": 6 // folders with >= this many items trigger an error + "max-folder-items": 6, // folders with >= this many items trigger an error + "vulture-min-confidence": 80, // minimum confidence (0-100) to flag a finding + "vulture-error-threshold": 90 // confidence at which a finding becomes an error }, - "include_extensions": [".py", ".ts", ".tsx", ".js", ".jsx"], // only these file types are line-counted - "exclude": ["node_modules", ".venv", "..."], // directories to skip entirely + "include_extensions": [".py", ".ts", ".tsx", ".js", ".jsx"], + "exclude": ["node_modules", ".venv", "..."], "exceptions": { - "max-file-lines": [], // glob patterns for files exempt from the line limit - "max-folder-items": [] // glob patterns for folders exempt from the item limit + "max-file-lines": [], // glob patterns for exempt files + "max-folder-items": [], // glob patterns for exempt folders + "vulture": [] // glob patterns for files vulture should ignore } } ``` +Set any key in `"enabled"` to `false` to skip that check entirely. Missing keys default to `true`, so existing configs without the `"enabled"` section behave identically to before. + +### Vulture whitelist + +`config/vulture_whitelist.py` suppresses false positives — symbols used by frameworks, entry points, or external consumers that vulture can't detect statically. Add bare names to the file to mark them as intentionally used. + +### ESLint + +`frontend/eslint.config.mjs` — flat config format (ESLint v9). The key rule for unused code is `@typescript-eslint/no-unused-vars`. Prefix a variable with `_` to suppress the warning. + +### Knip + +`frontend/knip.json` — Knip auto-detects entry points from `webpack.config.js`. The `project` field tells it which files to analyze. + ## Adding exceptions -If a file or folder legitimately needs to exceed a limit, add a glob pattern to the `exceptions` list in `structlint.json`. For example: +If a file legitimately needs to exceed a limit, add a glob to the `exceptions` list in `config/config.json`: ```json { "exceptions": { - "max-file-lines": [ - "backend/tests/test_analytics.py" - ], - "max-folder-items": [ - "backend/apps/agents" - ] + "max-file-lines": ["backend/tests/test_analytics.py"], + "max-folder-items": ["backend/apps/agents"], + "vulture": ["backend/legacy/*"] } } ``` -You can use wildcards: `"backend/tests/*"` exempts all files in the tests folder. +Wildcards work: `"backend/tests/*"` exempts all files in the tests folder. -## Type checking +## Folder structure -This folder also contains `pyrightconfig.json`, which configures strict type checking for the Python backend. This works through the **Pylance** extension in Cursor/VS Code — it shows type errors in real-time as you type, the same way TypeScript checks the frontend. No setup needed beyond having Pylance installed. - -## Files in this folder - -| File | Purpose | -|---|---| -| `structlint.py` | The linter script | -| `structlint.json` | Rules, exclusions, and exceptions | -| `pyrightconfig.json` | Python type checking config (for CLI `pyright` usage) | +``` +linter/ + checks/ # check implementations + __init__.py # shared filter/match utilities + structural.py # file length, folder size, nested imports + vulture.py # vulture dead-code runner + eslint.py # eslint runner + knip.py # knip unused-code runner + config/ # all configuration files + config.json # enabled checks, rules, exclusions, exceptions + pyrightconfig.json # python type checking config + vulture_whitelist.py # false positive suppressions for vulture + lint.py # orchestrator (loads config, runs checks, outputs results) + print_errors.sh # colored terminal reporter + README.md +``` diff --git a/linter/checks/__init__.py b/linter/checks/__init__.py new file mode 100644 index 00000000..18cbd863 --- /dev/null +++ b/linter/checks/__init__.py @@ -0,0 +1,22 @@ +"""Check infrastructure: shared filter/match utilities.""" + +from __future__ import annotations + +import fnmatch +from pathlib import Path + + +def _matches_any(text: str, patterns: list[str]) -> bool: + return any(fnmatch.fnmatch(text, p) for p in patterns) + + +def is_excluded(path: Path, root: Path, excludes: list[str]) -> bool: + rel = path.relative_to(root) + for part in rel.parts: + if _matches_any(part, excludes): + return True + return _matches_any(str(rel), excludes) + + +def is_excepted(rel_path: str, rule: str, exceptions: dict[str, list[str]]) -> bool: + return _matches_any(rel_path, exceptions.get(rule, [])) diff --git a/linter/checks/eslint.py b/linter/checks/eslint.py new file mode 100644 index 00000000..5c58cc88 --- /dev/null +++ b/linter/checks/eslint.py @@ -0,0 +1,45 @@ +"""ESLint runner for the TypeScript frontend.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + + +def run_eslint(root: Path) -> list[str]: + """Run ESLint on the TypeScript frontend and return errors.""" + frontend_dir = root / "frontend" + eslint_bin = frontend_dir / "node_modules" / ".bin" / "eslint" + if not eslint_bin.exists(): + return [] + + cmd = [str(eslint_bin), "src/", "--format", "json", "--no-warn-ignored"] + try: + result = subprocess.run( + cmd, capture_output=True, text=True, + cwd=str(frontend_dir), timeout=60, + ) + except (OSError, subprocess.TimeoutExpired): + return [] + + try: + data = json.loads(result.stdout) + except (json.JSONDecodeError, ValueError): + return [] + + errors: list[str] = [] + for entry in data: + try: + rel = str(Path(entry["filePath"]).relative_to(root)) + except (ValueError, KeyError): + continue + for msg in entry.get("messages", []): + sev = "error" if msg.get("severity", 0) >= 2 else "warning" + text = msg.get("message", "").replace("\n", " ").strip() + rule = msg.get("ruleId") or "unknown" + errors.append( + f"{rel}:{msg.get('line', 1)}:{msg.get('column', 1)}: " + f"{sev}: [eslint] {text} ({rule})" + ) + return errors diff --git a/linter/checks/knip.py b/linter/checks/knip.py new file mode 100644 index 00000000..0824552e --- /dev/null +++ b/linter/checks/knip.py @@ -0,0 +1,61 @@ +"""Knip unused-code runner for the TypeScript frontend.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +KIND_LABELS = { + "dependencies": "Unused dependency", + "devDependencies": "Unused devDependency", + "exports": "Unused export", + "types": "Unused exported type", + "unlisted": "Unlisted dependency", + "binaries": "Unused binary", + "files": "Unused file", + "duplicates": "Duplicate export", +} + + +def run_knip(root: Path) -> list[str]: + """Run Knip on the TypeScript frontend and return errors.""" + frontend_dir = root / "frontend" + knip_bin = frontend_dir / "node_modules" / ".bin" / "knip" + if not knip_bin.exists(): + return [] + + cmd = [str(knip_bin), "--reporter", "json"] + try: + result = subprocess.run( + cmd, capture_output=True, text=True, + cwd=str(frontend_dir), timeout=60, + ) + except (OSError, subprocess.TimeoutExpired): + return [] + + try: + data = json.loads(result.stdout) + except (json.JSONDecodeError, ValueError): + return [] + + errors: list[str] = [] + for entry in data.get("issues", []): + filepath = entry.get("file", "") + rel = f"frontend/{filepath}" + for kind, label in KIND_LABELS.items(): + for item in entry.get(kind, []): + if isinstance(item, dict): + name = item.get("name", "") + line = item.get("line", 1) + col = item.get("col", 1) + elif isinstance(item, str): + name = item + line, col = 1, 1 + else: + continue + errors.append( + f"{rel}:{line}:{col}: error: " + f"[knip] {label} '{name}'" + ) + return errors diff --git a/linter/checks/structural.py b/linter/checks/structural.py new file mode 100644 index 00000000..68d41f84 --- /dev/null +++ b/linter/checks/structural.py @@ -0,0 +1,104 @@ +"""Structural checks: file length, folder size, and nested imports.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +from . import _matches_any + +ANCHOR_FILES = ("__init__.py", "index.ts", "index.tsx", "index.js") + + +def _find_anchor_file(dirpath: Path, root: Path) -> str: + """Find a real file inside the folder to attach the diagnostic to. + + Prefers common entry-point files (__init__.py, index.ts, etc.) so the + error shows up inline when you open that file. Falls back to the first + file alphabetically, then the directory path itself. + """ + for name in ANCHOR_FILES: + candidate = dirpath / name + if candidate.exists(): + return str(candidate.relative_to(root)) + try: + first = sorted( + f for f in dirpath.iterdir() + if f.is_file() and not f.name.startswith(".") + ) + if first: + return str(first[0].relative_to(root)) + except OSError: + pass + return str(dirpath.relative_to(root)) + + +def check_file_lines( + filepath: Path, root: Path, max_lines: int, +) -> tuple[str, int] | None: + try: + count = len(filepath.read_text(errors="ignore").splitlines()) + except OSError: + return None + if count >= max_lines: + rel = filepath.relative_to(root) + msg = ( + f"{rel}:1:1: error: " + f"[max-file-lines] File has {count} lines (limit {max_lines})" + ) + return (msg, count) + return None + + +def check_folder_items( + dirpath: Path, root: Path, max_items: int, excludes: list[str], +) -> tuple[str, int] | None: + try: + items = [ + i for i in dirpath.iterdir() + if not i.name.startswith(".") and not _matches_any(i.name, excludes) + ] + except OSError: + return None + count = len(items) + if count >= max_items: + anchor = _find_anchor_file(dirpath, root) + rel = dirpath.relative_to(root) + msg = ( + f"{anchor}:1:1: error: " + f"[max-folder-items] Folder '{rel}' has {count} items (limit {max_items})" + ) + return (msg, count) + return None + + +def check_nested_imports(filepath: Path, root: Path) -> list[str]: + """Detect import statements inside function or method bodies.""" + if filepath.suffix != ".py": + return [] + try: + source = filepath.read_text(errors="ignore") + tree = ast.parse(source, filename=str(filepath)) + except (OSError, SyntaxError): + return [] + + errors: list[str] = [] + rel = str(filepath.relative_to(root)) + + def _visit(node: ast.AST, in_function: bool) -> None: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + in_function = True + if in_function and isinstance(node, (ast.Import, ast.ImportFrom)): + if isinstance(node, ast.ImportFrom): + name = node.module or "" + else: + name = ", ".join(a.name for a in node.names) + errors.append( + f"{rel}:{node.lineno}:1: error: " + f"[no-nested-imports] Nested import '{name}'" + ) + for child in ast.iter_child_nodes(node): + _visit(child, in_function) + + _visit(tree, False) + return errors diff --git a/linter/checks/vulture.py b/linter/checks/vulture.py new file mode 100644 index 00000000..b29b489d --- /dev/null +++ b/linter/checks/vulture.py @@ -0,0 +1,55 @@ +"""Vulture dead-code detection runner.""" + +from __future__ import annotations + +import re +import shutil +import subprocess +from pathlib import Path + +from . import is_excepted + +CONFIG_DIR = Path(__file__).resolve().parent.parent / "config" + + +def run_vulture( + root: Path, min_confidence: int, error_threshold: int, + exceptions: dict[str, list[str]], +) -> list[str]: + """Run vulture on the Python backend and return errors.""" + vulture_bin = root / "backend" / ".venv" / "bin" / "vulture" + if not vulture_bin.exists(): + found = shutil.which("vulture") + if not found: + return [] + vulture_bin = Path(found) + + whitelist = CONFIG_DIR / "vulture_whitelist.py" + cmd = [str(vulture_bin), "backend", "debug.py"] + if whitelist.exists(): + cmd.append(str(whitelist)) + cmd.extend([ + "--min-confidence", str(min_confidence), + "--exclude", ".venv,__pycache__,data,uv-bin", + ]) + + try: + result = subprocess.run( + cmd, capture_output=True, text=True, cwd=str(root), timeout=30, + ) + except (OSError, subprocess.TimeoutExpired): + return [] + + errors: list[str] = [] + for line in result.stdout.strip().splitlines(): + m = re.match(r"^(.+):(\d+): (.+)$", line) + if not m: + continue + filepath, lineno, message = m.groups() + if is_excepted(filepath, "vulture", exceptions): + continue + conf = re.search(r"\((\d+)% confidence\)", message) + confidence = int(conf.group(1)) if conf else 0 + severity = "error" if confidence >= error_threshold else "warning" + errors.append(f"{filepath}:{lineno}:1: {severity}: [vulture] {message}") + return errors diff --git a/linter/config/config.json b/linter/config/config.json new file mode 100644 index 00000000..6705831a --- /dev/null +++ b/linter/config/config.json @@ -0,0 +1,43 @@ +{ + "enabled": { + "max-file-lines": true, + "max-folder-items": true, + "no-nested-imports": true, + "vulture": true, + "eslint": true, + "knip": true + }, + "rules": { + "max-file-lines": 250, + "max-folder-items": 7, + "vulture-min-confidence": 1, + "vulture-error-threshold": 1, + "no-nested-imports": true + }, + "include_extensions": [".py", ".ts", ".tsx", ".js", ".jsx"], + "exclude": [ + "node_modules", + ".venv", + "dist", + "build", + "__pycache__", + ".git", + ".cursor", + ".vscode", + "uv-bin", + "data", + "public", + "readme_assets", + "openswarm_debug.egg-info", + "frontend/src/assets" + ], + "exceptions": { + "max-file-lines": ["openswarm_debug.egg-info/**"], + "max-folder-items": [ + "frontend", + "backend" + ], + "no-nested-imports": ["linter/lint.py"], + "vulture": [] + } +} diff --git a/linter/config/pyrightconfig.json b/linter/config/pyrightconfig.json new file mode 100644 index 00000000..96fe0ba2 --- /dev/null +++ b/linter/config/pyrightconfig.json @@ -0,0 +1,14 @@ +{ + "include": ["../../backend"], + "exclude": [ + "../../backend/.venv", + "../../backend/uv-bin", + "../../backend/data", + "../../backend/tests" + ], + "typeCheckingMode": "strict", + "pythonVersion": "3.11", + "venvPath": "../../backend", + "venv": ".venv", + "reportMissingTypeStubs": false +} diff --git a/linter/config/vulture_whitelist.py b/linter/config/vulture_whitelist.py new file mode 100644 index 00000000..9406a7ff --- /dev/null +++ b/linter/config/vulture_whitelist.py @@ -0,0 +1,24 @@ +# Vulture whitelist — suppress false positives for symbols used by +# frameworks, entry points, and external consumers. +# +# Pass this file as an argument to vulture alongside source directories. +# Each bare name tells vulture "this symbol is intentionally used." + +# backend/main.py — entry points referenced by string, not direct call +main +app + +# FastAPI route handlers — registered via decorators, called by framework +pull_structure +push_structure +reset_color +reset_emoji +check + +# FastAPI lifespan context managers — passed to SubApp constructor +debugger_lifespan +health_lifespan + +# debug.py — module replaces itself with the debug() function via +# sys.modules[__name__] = debug, consumed by external packages +debug diff --git a/linter/lint.py b/linter/lint.py new file mode 100644 index 00000000..44444e9d --- /dev/null +++ b/linter/lint.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Unified linter: orchestrates structural checks, dead-code detection, and lint tools.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +from checks import is_excluded, is_excepted +from checks.structural import check_file_lines, check_folder_items, check_nested_imports +from checks.vulture import run_vulture +from checks.eslint import run_eslint +from checks.knip import run_knip + +SCRIPT_DIR = Path(__file__).resolve().parent +CONFIG_FILE = SCRIPT_DIR / "config" / "config.json" + + +def load_config() -> dict[str, Any]: + with open(CONFIG_FILE) as f: + return json.load(f) + + +def run_checks(root: Path) -> tuple[list[str], list[str], list[str], list[str]]: + config = load_config() + enabled: dict[str, bool] = config.get("enabled", {}) + rules: dict[str, int] = config["rules"] + excludes: list[str] = config["exclude"] + exceptions: dict[str, list[str]] = config["exceptions"] + extensions: list[str] = config["include_extensions"] + + max_lines: int = rules["max-file-lines"] + max_items: int = rules["max-folder-items"] + check_imports: bool = rules.get("no-nested-imports", False) + structural_errors: list[str] = [] + + file_lines_on = enabled.get("max-file-lines", True) + folder_items_on = enabled.get("max-folder-items", True) + nested_imports_on = enabled.get("no-nested-imports", True) + + for dirpath_str, dirnames, filenames in os.walk(root): + dp = Path(dirpath_str) + + if is_excluded(dp, root, excludes): + dirnames.clear() + continue + + rel_dir = str(dp.relative_to(root)) + if folder_items_on and rel_dir != "." and not is_excepted(rel_dir, "max-folder-items", exceptions): + result = check_folder_items(dp, root, max_items, excludes) + if result: + structural_errors.append(result[0]) + + for fname in filenames: + fp = dp / fname + if fp.suffix not in extensions: + continue + if is_excluded(fp, root, excludes): + continue + rel_file = str(fp.relative_to(root)) + if file_lines_on and not is_excepted(rel_file, "max-file-lines", exceptions): + result = check_file_lines(fp, root, max_lines) + if result: + structural_errors.append(result[0]) + if nested_imports_on and check_imports and not is_excepted(rel_file, "no-nested-imports", exceptions): + structural_errors.extend(check_nested_imports(fp, root)) + + vulture_errors: list[str] = [] + if enabled.get("vulture", True): + vulture_confidence = rules.get("vulture-min-confidence") + if vulture_confidence is not None: + vulture_error_threshold = rules.get("vulture-error-threshold", 100) + vulture_errors = run_vulture( + root, vulture_confidence, vulture_error_threshold, exceptions, + ) + + eslint_errors = run_eslint(root) if enabled.get("eslint", True) else [] + knip_errors = run_knip(root) if enabled.get("knip", True) else [] + + return sorted(structural_errors), sorted(vulture_errors), sorted(eslint_errors), sorted(knip_errors) + + +def _print_section(name: str, errors: list[str]) -> None: + print(f"{name}: checking...", flush=True) + for e in errors: + print(e, flush=True) + print(f"{name}: done. {len(errors)} error(s) found.", flush=True) + + +def print_results( + structural_errors: list[str], vulture_errors: list[str], + eslint_errors: list[str], knip_errors: list[str], +) -> None: + _print_section("structural", structural_errors) + _print_section("vulture", vulture_errors) + _print_section("eslint", eslint_errors) + _print_section("knip", knip_errors) + + +def watch_loop(root: Path) -> None: + from watchfiles import watch, DefaultFilter + + config_dir = SCRIPT_DIR / "config" + + print_results(*run_checks(root)) + + class SourceFilter(DefaultFilter): + allowed_extensions = (".py", ".ts", ".tsx", ".js", ".jsx") + + def __call__(self, change: Any, path: str) -> bool: + if not super().__call__(change, path): + return False + if Path(path).suffix in self.allowed_extensions: + return True + p = Path(path) + if p.suffix == ".json" and (p.parent == SCRIPT_DIR or p.parent == config_dir): + return True + return Path(path).is_dir() + + for _changes in watch(root, watch_filter=SourceFilter()): + print_results(*run_checks(root)) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Unified linter") + parser.add_argument("--watch", action="store_true", help="Watch for changes") + parser.add_argument("--root", type=str, default=".", help="Root directory") + args = parser.parse_args() + + root = Path(args.root).resolve() + + if args.watch: + watch_loop(root) + else: + results = run_checks(root) + print_results(*results) + sys.exit(1 if any(results) else 0) + + +if __name__ == "__main__": + main() diff --git a/linter/print_errors.sh b/linter/print_errors.sh new file mode 100755 index 00000000..ac6c8d89 --- /dev/null +++ b/linter/print_errors.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Print lint violations to stdout with colored formatting. +# Usage: bash linter/print_errors.sh [ROOT_DIR] + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="${1:-$(dirname "$SCRIPT_DIR")}" + +YELLOW='\033[33m' +CYAN='\033[36m' +BOLD='\033[1m' +RESET='\033[0m' + +LINT_OUTPUT=$(python3 "$SCRIPT_DIR/lint.py" --root "$ROOT_DIR" 2>&1) +LINT_EXIT=$? + +if [ $LINT_EXIT -ne 0 ]; then + STRUCT_LINES=$(echo "$LINT_OUTPUT" | grep -v "^structural:" | grep -v "^vulture:" | grep -v "^eslint:" | grep -v "^knip:" | grep -v '\[vulture\]' | grep -v '\[eslint\]' | grep -v '\[knip\]') + VULTURE_LINES=$(echo "$LINT_OUTPUT" | grep '\[vulture\]') + ESLINT_LINES=$(echo "$LINT_OUTPUT" | grep '\[eslint\]') + KNIP_LINES=$(echo "$LINT_OUTPUT" | grep '\[knip\]') + + STRUCT_COUNT=$(echo "$STRUCT_LINES" | grep -cE ':\s+(error|warning):\s+') + VULTURE_COUNT=$(echo "$VULTURE_LINES" | grep -cE ':\s+(error|warning):\s+') + ESLINT_COUNT=$(echo "$ESLINT_LINES" | grep -cE ':\s+(error|warning):\s+') + KNIP_COUNT=$(echo "$KNIP_LINES" | grep -cE ':\s+(error|warning):\s+') + + if [ "$STRUCT_COUNT" -gt 0 ]; then + echo "" + echo -e "${YELLOW}${BOLD}[structural] Violations found:${RESET}" + echo "$STRUCT_LINES" | while IFS= read -r line; do + [ -n "$line" ] && echo -e "${YELLOW} $line${RESET}" + done + echo -e "${YELLOW}${BOLD} ${STRUCT_COUNT} violation(s) — fix or add exceptions in linter/config/config.json${RESET}" + fi + + if [ "$VULTURE_COUNT" -gt 0 ]; then + echo "" + echo -e "${CYAN}${BOLD}[vulture] Dead code found:${RESET}" + echo "$VULTURE_LINES" | while IFS= read -r line; do + [ -n "$line" ] && echo -e "${CYAN} $line${RESET}" + done + echo -e "${CYAN}${BOLD} ${VULTURE_COUNT} finding(s) — fix or add to linter/config/vulture_whitelist.py${RESET}" + fi + + if [ "$ESLINT_COUNT" -gt 0 ]; then + echo "" + echo -e "${YELLOW}${BOLD}[eslint] Lint errors found:${RESET}" + echo "$ESLINT_LINES" | while IFS= read -r line; do + [ -n "$line" ] && echo -e "${YELLOW} $line${RESET}" + done + echo -e "${YELLOW}${BOLD} ${ESLINT_COUNT} error(s) — fix or disable rules in frontend/eslint.config.mjs${RESET}" + fi + + if [ "$KNIP_COUNT" -gt 0 ]; then + echo "" + echo -e "${CYAN}${BOLD}[knip] Unused code/dependencies found:${RESET}" + echo "$KNIP_LINES" | while IFS= read -r line; do + [ -n "$line" ] && echo -e "${CYAN} $line${RESET}" + done + echo -e "${CYAN}${BOLD} ${KNIP_COUNT} finding(s) — remove unused code or update frontend/knip.json${RESET}" + fi + + echo "" +fi diff --git a/linter/pyrightconfig.json b/linter/pyrightconfig.json deleted file mode 100644 index 4533e7fd..00000000 --- a/linter/pyrightconfig.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "include": ["../backend"], - "exclude": [ - "../backend/.venv", - "../backend/uv-bin", - "../backend/data", - "../backend/tests" - ], - "typeCheckingMode": "strict", - "pythonVersion": "3.11", - "venvPath": "../backend", - "venv": ".venv", - "reportMissingTypeStubs": false -} diff --git a/linter/structlint.json b/linter/structlint.json deleted file mode 100644 index ec656496..00000000 --- a/linter/structlint.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "rules": { - "max-file-lines": 250, - "max-folder-items": 6 - }, - "include_extensions": [".py", ".ts", ".tsx", ".js", ".jsx"], - "exclude": [ - "node_modules", - ".venv", - "dist", - "build", - "__pycache__", - ".git", - ".cursor", - ".vscode", - "uv-bin", - "data", - "public", - "readme_assets", - "ASSISTANT_UI_MIGRATION" - ], - "exceptions": { - "max-file-lines": [], - "max-folder-items": [ - "frontend" - ] - } -} diff --git a/linter/structlint.py b/linter/structlint.py deleted file mode 100644 index f866ed57..00000000 --- a/linter/structlint.py +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env python3 -"""Structural linter: enforces file line limits and folder item limits.""" - -from __future__ import annotations - -import argparse -import fnmatch -import json -import os -import sys -from pathlib import Path -from typing import Any - -SCRIPT_DIR = Path(__file__).resolve().parent -CONFIG_FILE = SCRIPT_DIR / "structlint.json" - - -def load_config() -> dict[str, Any]: - with open(CONFIG_FILE) as f: - return json.load(f) - - -def _matches_any(text: str, patterns: list[str]) -> bool: - return any(fnmatch.fnmatch(text, p) for p in patterns) - - -def is_excluded(path: Path, root: Path, excludes: list[str]) -> bool: - rel = path.relative_to(root) - for part in rel.parts: - if _matches_any(part, excludes): - return True - return _matches_any(str(rel), excludes) - - -def is_excepted(rel_path: str, rule: str, exceptions: dict[str, list[str]]) -> bool: - return _matches_any(rel_path, exceptions.get(rule, [])) - - -def check_file_lines( - filepath: Path, root: Path, max_lines: int, -) -> tuple[str, int] | None: - try: - count = len(filepath.read_text(errors="ignore").splitlines()) - except OSError: - return None - if count >= max_lines: - rel = filepath.relative_to(root) - msg = ( - f"{rel}:1:1: error: File has {count} lines " - f"(limit {max_lines}) [max-file-lines]" - ) - return (msg, count) - return None - - -ANCHOR_FILES = ("__init__.py", "index.ts", "index.tsx", "index.js") - - -def _find_anchor_file(dirpath: Path, root: Path) -> str: - """Find a real file inside the folder to attach the diagnostic to. - - Prefers common entry-point files (__init__.py, index.ts, etc.) so the - error shows up inline when you open that file. Falls back to the first - file alphabetically, then the directory path itself. - """ - for name in ANCHOR_FILES: - candidate = dirpath / name - if candidate.exists(): - return str(candidate.relative_to(root)) - try: - first = sorted( - f for f in dirpath.iterdir() - if f.is_file() and not f.name.startswith(".") - ) - if first: - return str(first[0].relative_to(root)) - except OSError: - pass - return str(dirpath.relative_to(root)) - - -def check_folder_items( - dirpath: Path, root: Path, max_items: int, excludes: list[str], -) -> tuple[str, int] | None: - try: - items = [ - i for i in dirpath.iterdir() - if not i.name.startswith(".") and not _matches_any(i.name, excludes) - ] - except OSError: - return None - count = len(items) - if count >= max_items: - anchor = _find_anchor_file(dirpath, root) - rel = dirpath.relative_to(root) - msg = ( - f"{anchor}:1:1: error: Folder '{rel}' has {count} items " - f"(limit {max_items}) [max-folder-items]" - ) - return (msg, count) - return None - - -def run_checks(root: Path) -> list[str]: - config = load_config() - rules: dict[str, int] = config["rules"] - excludes: list[str] = config["exclude"] - exceptions: dict[str, list[str]] = config["exceptions"] - extensions: list[str] = config["include_extensions"] - - max_lines: int = rules["max-file-lines"] - max_items: int = rules["max-folder-items"] - errors: list[str] = [] - - for dirpath_str, dirnames, filenames in os.walk(root): - dp = Path(dirpath_str) - - if is_excluded(dp, root, excludes): - dirnames.clear() - continue - - rel_dir = str(dp.relative_to(root)) - if rel_dir != "." and not is_excepted(rel_dir, "max-folder-items", exceptions): - result = check_folder_items(dp, root, max_items, excludes) - if result: - errors.append(result[0]) - - for fname in filenames: - fp = dp / fname - if fp.suffix not in extensions: - continue - if is_excluded(fp, root, excludes): - continue - rel_file = str(fp.relative_to(root)) - if not is_excepted(rel_file, "max-file-lines", exceptions): - result = check_file_lines(fp, root, max_lines) - if result: - errors.append(result[0]) - - return sorted(errors) - - -def print_results(errors: list[str]) -> None: - print("structlint: checking...", flush=True) - for err in errors: - print(err, flush=True) - count = len(errors) - print(f"structlint: done. {count} error(s) found.", flush=True) - - -def watch_loop(root: Path) -> None: - from watchfiles import watch, DefaultFilter - - print_results(run_checks(root)) - - class SourceFilter(DefaultFilter): - allowed_extensions = (".py", ".ts", ".tsx", ".js", ".jsx") - - def __call__(self, change: Any, path: str) -> bool: - if not super().__call__(change, path): - return False - if Path(path).suffix in self.allowed_extensions: - return True - return Path(path).is_dir() - - for _changes in watch(root, watch_filter=SourceFilter()): - print_results(run_checks(root)) - - -def main() -> None: - parser = argparse.ArgumentParser(description="Structural linter") - parser.add_argument("--watch", action="store_true", help="Watch for changes") - parser.add_argument("--root", type=str, default=".", help="Root directory") - args = parser.parse_args() - - root = Path(args.root).resolve() - - if args.watch: - watch_loop(root) - else: - errors = run_checks(root) - print_results(errors) - sys.exit(1 if errors else 0) - - -if __name__ == "__main__": - main()