"""End-to-end smoke: does a scheduled workflow actually fire when its time hits, with the full scheduler loop running? Runs the real scheduler.start() loop with the executor mocked so we don't need a live agent_manager. Then arms a workflow whose next_run_at is one second in the future, waits, and asserts the mocked executor was called. Run: cd backend && .venv/bin/python -m pytest tests/test_schedule_e2e.py -v """ from __future__ import annotations import asyncio from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock from zoneinfo import ZoneInfo import pytest pytestmark = pytest.mark.asyncio @pytest.fixture(autouse=True) def isolated_data_dir(monkeypatch, tmp_path): from backend.apps.workflows import storage as _storage from backend.apps.workflows import escalation as _escalation from backend.apps.workflows import audit as _audit from backend.apps.workflows import scheduler as _scheduler monkeypatch.setattr(_storage, "DATA_DIR", str(tmp_path / "workflows")) monkeypatch.setattr(_storage, "RUNS_DIR", str(tmp_path / "workflows" / "runs")) monkeypatch.setattr(_storage, "PAUSED_FILE", str(tmp_path / "workflows" / "paused.json")) monkeypatch.setattr(_storage, "MISSED_FILE", str(tmp_path / "workflows" / "missed.json")) monkeypatch.setattr(_storage, "_workflow_cache", {}) monkeypatch.setattr(_storage, "_runs_cache", {}) monkeypatch.setattr(_storage, "_missed_cache", []) monkeypatch.setattr(_storage, "_cache_loaded", False) monkeypatch.setattr(_storage, "_paused", False) monkeypatch.setattr(_audit, "AUDIT_DIR", str(tmp_path / "workflows" / "audit")) # Module-level scheduler state survives across tests; reset it so each test gets a fresh _wake Event bound to its own event loop. _scheduler._loop_task = None _scheduler._wake = asyncio.Event() _escalation._tasks.clear() _escalation._state.clear() yield def _make_wf(**overrides): from backend.apps.workflows.models import Workflow, ScheduleConfig, WorkflowStep base = dict( title="smoke", steps=[WorkflowStep(text="hi")], schedule=ScheduleConfig( enabled=True, repeat_unit="day", repeat_every=1, hour=9, minute=0, timezone="America/Los_Angeles", ), ) base.update(overrides) return Workflow(**base) async def test_loop_fires_due_workflow(monkeypatch): """Arm a workflow to fire ~now and assert the executor was actually invoked by the scheduler loop within the test window. Note the save happens AFTER scheduler.start() so reconcile_on_startup doesn't clobber next_run_at.""" from backend.apps.workflows import storage, scheduler, executor fired = asyncio.Event() captured: dict = {} async def fake_execute(wf, triggered_by="schedule", scheduled_for=None): captured["wf_id"] = wf.id captured["triggered_by"] = triggered_by captured["scheduled_for"] = scheduled_for from backend.apps.workflows.models import WorkflowRun run = WorkflowRun( workflow_id=wf.id, status="success", scheduled_for=scheduled_for, started_at=datetime.now(timezone.utc), finished_at=datetime.now(timezone.utc), triggered_by=triggered_by, ) storage.record_run(run) fired.set() return run monkeypatch.setattr(executor, "execute", fake_execute) await scheduler.start() try: wf = _make_wf() wf.next_run_at = datetime.now(timezone.utc) + timedelta(seconds=1) storage.save_workflow(wf) scheduler.kick() # force immediate tick # Wait up to 5s for the fire to land. await asyncio.wait_for(fired.wait(), timeout=5.0) finally: await scheduler.stop() assert captured.get("wf_id") == wf.id assert captured.get("triggered_by") == "schedule" runs = storage.list_runs(wf.id, limit=10) assert len(runs) == 1 assert runs[0].status == "success" # Scheduler should have rolled next_run_at forward to a future slot. after = storage.get_workflow(wf.id) assert after.next_run_at is not None assert after.next_run_at > datetime.now(timezone.utc) async def test_disabled_workflow_does_not_fire(monkeypatch): """Master switch off => loop never invokes the executor even if next_run_at is in the past.""" from backend.apps.workflows import storage, scheduler, executor fake = AsyncMock() monkeypatch.setattr(executor, "execute", fake) wf = _make_wf() wf.schedule.enabled = False wf.next_run_at = datetime.now(timezone.utc) - timedelta(seconds=10) storage.save_workflow(wf) await scheduler.start() try: scheduler.kick() await asyncio.sleep(2.0) finally: await scheduler.stop() fake.assert_not_called() async def test_paused_state_blocks_all_fires(monkeypatch): """Global pause flag wins over per-workflow enabled state.""" from backend.apps.workflows import storage, scheduler, executor fake = AsyncMock() monkeypatch.setattr(executor, "execute", fake) wf = _make_wf() wf.next_run_at = datetime.now(timezone.utc) - timedelta(seconds=1) storage.save_workflow(wf) storage.set_paused(True) await scheduler.start() try: scheduler.kick() await asyncio.sleep(2.0) finally: await scheduler.stop() fake.assert_not_called() storage.set_paused(False) async def test_reconcile_captures_missed_fires(monkeypatch): """A daily workflow whose next_run_at elapsed while the app was closed => startup captures the missed fires as pending MissedRuns and rolls next_run_at forward (no auto-firing).""" from backend.apps.workflows import storage, scheduler # created_at must predate the missed window; occurrences_between never enumerates fires from before the workflow existed. wf = _make_wf(created_at=datetime.now(timezone.utc) - timedelta(days=10)) wf.next_run_at = datetime.now(timezone.utc) - timedelta(days=3) storage.save_workflow(wf) scheduler.reconcile_on_startup() missed = [m for m in storage.list_missed() if m.workflow_id == wf.id] assert len(missed) >= 1 assert all(m.scheduled_for < datetime.now(timezone.utc) for m in missed) after = storage.get_workflow(wf.id) assert after.next_run_at is not None assert after.next_run_at > datetime.now(timezone.utc) async def test_reconcile_no_missed_when_future(monkeypatch): """next_run_at in the future => nothing missed, nothing captured.""" from backend.apps.workflows import storage, scheduler wf = _make_wf(created_at=datetime.now(timezone.utc) - timedelta(days=10)) wf.next_run_at = datetime.now(timezone.utc) + timedelta(hours=6) storage.save_workflow(wf) scheduler.reconcile_on_startup() assert [m for m in storage.list_missed() if m.workflow_id == wf.id] == [] async def test_reconcile_over_cap_collapses_to_skipped(monkeypatch): """A 15-minute schedule off for days => only the cap is kept reviewable; the rest collapse into a single skipped run in history.""" from backend.apps.workflows import storage, scheduler from backend.apps.workflows.models import ScheduleConfig wf = _make_wf( created_at=datetime.now(timezone.utc) - timedelta(days=30), schedule=ScheduleConfig( enabled=True, repeat_unit="minute", repeat_every=15, timezone="America/Los_Angeles", ), ) wf.next_run_at = datetime.now(timezone.utc) - timedelta(days=2) storage.save_workflow(wf) scheduler.reconcile_on_startup() missed = [m for m in storage.list_missed() if m.workflow_id == wf.id] assert len(missed) == scheduler.PER_WORKFLOW_MISSED_CAP skipped = [r for r in storage.list_runs(wf.id, limit=50) if r.status == "skipped"] assert len(skipped) == 1 async def test_dismiss_missed_records_skipped(monkeypatch): """Dismissing a missed run drops it from pending and leaves a skipped run in history.""" from backend.apps.workflows import storage from backend.apps.workflows.models import MissedRun, MissedRunAction from backend.apps.workflows.workflows import dismiss_missed_runs wf = _make_wf() storage.save_workflow(wf) m = MissedRun(workflow_id=wf.id, scheduled_for=datetime.now(timezone.utc) - timedelta(hours=2)) storage.add_missed(m) res = await dismiss_missed_runs(MissedRunAction(ids=[m.id])) assert res["dismissed"] == 1 assert storage.list_missed() == [] skipped = [r for r in storage.list_runs(wf.id, limit=10) if r.status == "skipped"] assert len(skipped) == 1 async def test_run_missed_runs_clears_pending_and_fires(monkeypatch): """Running selected missed fires removes them from pending and invokes the executor once per fire, sequentially.""" from backend.apps.workflows import storage, executor, scheduler from backend.apps.workflows.models import MissedRun, MissedRunAction from backend.apps.workflows.workflows import run_missed_runs calls: list = [] async def fake_execute(wf, triggered_by="schedule", scheduled_for=None): calls.append(scheduled_for) from backend.apps.workflows.models import WorkflowRun return WorkflowRun(workflow_id=wf.id, status="ran_late", scheduled_for=scheduled_for) monkeypatch.setattr(executor, "execute", fake_execute) wf = _make_wf() storage.save_workflow(wf) m1 = MissedRun(workflow_id=wf.id, scheduled_for=datetime.now(timezone.utc) - timedelta(hours=3)) m2 = MissedRun(workflow_id=wf.id, scheduled_for=datetime.now(timezone.utc) - timedelta(hours=2)) storage.add_missed(m1) storage.add_missed(m2) res = await run_missed_runs(MissedRunAction(ids=[m1.id, m2.id])) assert res["started"] == 2 assert storage.list_missed() == [] # The endpoint spawns the sequence as a background task; give it a beat. await asyncio.sleep(0.05) assert len(calls) == 2 async def test_create_workflow_schedules_next_fire(): """POST-like create path: enabled schedule => next_run_at populated by compute_next_fire.""" from backend.apps.workflows.models import Workflow, ScheduleConfig, WorkflowStep from backend.apps.workflows import scheduler wf = Workflow( title="t", steps=[WorkflowStep(text="hi")], schedule=ScheduleConfig( enabled=True, repeat_unit="week", repeat_every=1, on_days=[0], hour=9, minute=0, timezone="America/Los_Angeles", ), ) nxt = scheduler.compute_next_fire(wf) assert nxt is not None assert nxt > datetime.now(timezone.utc) tz = ZoneInfo("America/Los_Angeles") local = nxt.astimezone(tz) assert local.weekday() == 6 # Python: Sunday assert (local.hour, local.minute) == (9, 0) async def test_next_run_at_advances_after_fire(monkeypatch): """After a fire the loop should re-compute next_run_at into the future and persist it, so the same fire can't repeat in the same minute.""" from backend.apps.workflows import storage, scheduler, executor fired = asyncio.Event() async def fake_execute(wf, triggered_by="schedule", scheduled_for=None): from backend.apps.workflows.models import WorkflowRun run = WorkflowRun( workflow_id=wf.id, status="success", scheduled_for=scheduled_for, started_at=datetime.now(timezone.utc), finished_at=datetime.now(timezone.utc), triggered_by=triggered_by, ) storage.record_run(run) fired.set() return run monkeypatch.setattr(executor, "execute", fake_execute) await scheduler.start() try: wf = _make_wf() armed_at = datetime.now(timezone.utc) + timedelta(seconds=1) wf.next_run_at = armed_at storage.save_workflow(wf) scheduler.kick() await asyncio.wait_for(fired.wait(), timeout=5.0) # Give the loop one extra tick to persist next_run_at. await asyncio.sleep(0.2) finally: await scheduler.stop() after = storage.get_workflow(wf.id) assert after.next_run_at is not None assert after.next_run_at > armed_at, "scheduler did not advance next_run_at past the slot it just fired" async def test_kick_wakes_loop_before_timeout(monkeypatch): """kick() should wake the loop early so manual schedule edits don't have to wait a full minute for the next tick boundary.""" from backend.apps.workflows import storage, scheduler, executor fired = asyncio.Event() async def fake_execute(wf, triggered_by="schedule", scheduled_for=None): from backend.apps.workflows.models import WorkflowRun run = WorkflowRun( workflow_id=wf.id, status="success", started_at=datetime.now(timezone.utc), finished_at=datetime.now(timezone.utc), triggered_by=triggered_by, ) storage.record_run(run) fired.set() return run monkeypatch.setattr(executor, "execute", fake_execute) await scheduler.start() try: wf = _make_wf() wf.next_run_at = datetime.now(timezone.utc) - timedelta(seconds=1) storage.save_workflow(wf) scheduler.kick() # Without kick(), the loop would sleep up to 60s before checking the freshly-saved workflow. With kick, it should fire fast. await asyncio.wait_for(fired.wait(), timeout=3.0) finally: await scheduler.stop() async def test_last_day_of_month_fires_on_month_end(): """last_day_of_month ignores day_of_month and lands on the calendar's final day, so it survives short months (Feb) instead of clamping.""" from backend.apps.workflows.models import ScheduleConfig from backend.apps.workflows import scheduler sched = ScheduleConfig( enabled=True, repeat_unit="month", repeat_every=1, day_of_month=15, last_day_of_month=True, hour=9, minute=0, timezone="America/Los_Angeles", ) wf = _make_wf(schedule=sched) tz = ZoneInfo("America/Los_Angeles") # From mid-February, the next fire is Feb 28 (or 29 on a leap year), NOT the 15th. ref = datetime(2026, 2, 10, 12, 0, tzinfo=tz).astimezone(timezone.utc) nxt = scheduler.compute_next_fire(wf, ref=ref) local = nxt.astimezone(tz) assert local.month == 2 and local.day == 28 assert local.hour == 9 # From end of Feb, the following fire rolls to Mar 31 (last day again). nxt2 = scheduler.compute_next_fire(wf, ref=nxt) local2 = nxt2.astimezone(tz) assert local2.month == 3 and local2.day == 31 async def test_soft_deleted_excluded_from_list(): """Soft-deleted workflows drop out of list_workflows (so the scheduler and every list view skip them) but stay visible to list_deleted_workflows.""" from backend.apps.workflows import storage live = _make_wf(title="live") trashed = _make_wf(title="trashed") trashed.deleted_at = datetime.now() storage.save_workflow(live) storage.save_workflow(trashed) active_ids = {w.id for w in storage.list_workflows()} deleted_ids = {w.id for w in storage.list_deleted_workflows()} assert live.id in active_ids and trashed.id not in active_ids assert trashed.id in deleted_ids and live.id not in deleted_ids # get_workflow still resolves a trashed record so restore/purge can fetch it. assert storage.get_workflow(trashed.id) is not None