mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-17 18:25:42 +02:00
386 lines
15 KiB
Python
386 lines
15 KiB
Python
"""End-to-end smoke: does a scheduled workflow actually fire when its
|
|
time hits, with the full scheduler loop running?
|
|
|
|
Runs the real scheduler.start() loop with the executor mocked so we
|
|
don't need a live agent_manager. Then arms a workflow whose
|
|
next_run_at is one second in the future, waits, and asserts the
|
|
mocked executor was called.
|
|
|
|
Run:
|
|
cd backend && .venv/bin/python -m pytest tests/test_schedule_e2e.py -v
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from datetime import datetime, timedelta, timezone
|
|
from unittest.mock import AsyncMock
|
|
from zoneinfo import ZoneInfo
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.asyncio
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def isolated_data_dir(monkeypatch, tmp_path):
|
|
from backend.apps.workflows import storage as _storage
|
|
from backend.apps.workflows import escalation as _escalation
|
|
from backend.apps.workflows import audit as _audit
|
|
from backend.apps.workflows import scheduler as _scheduler
|
|
monkeypatch.setattr(_storage, "DATA_DIR", str(tmp_path / "workflows"))
|
|
monkeypatch.setattr(_storage, "RUNS_DIR", str(tmp_path / "workflows" / "runs"))
|
|
monkeypatch.setattr(_storage, "PAUSED_FILE", str(tmp_path / "workflows" / "paused.json"))
|
|
monkeypatch.setattr(_storage, "MISSED_FILE", str(tmp_path / "workflows" / "missed.json"))
|
|
monkeypatch.setattr(_storage, "_workflow_cache", {})
|
|
monkeypatch.setattr(_storage, "_runs_cache", {})
|
|
monkeypatch.setattr(_storage, "_missed_cache", [])
|
|
monkeypatch.setattr(_storage, "_cache_loaded", False)
|
|
monkeypatch.setattr(_storage, "_paused", False)
|
|
monkeypatch.setattr(_audit, "AUDIT_DIR", str(tmp_path / "workflows" / "audit"))
|
|
# Module-level scheduler state survives across tests; reset it so each test gets a fresh _wake Event bound to its own event loop.
|
|
_scheduler._loop_task = None
|
|
_scheduler._wake = asyncio.Event()
|
|
_escalation._tasks.clear()
|
|
_escalation._state.clear()
|
|
yield
|
|
|
|
|
|
def _make_wf(**overrides):
|
|
from backend.apps.workflows.models import Workflow, ScheduleConfig, WorkflowStep
|
|
base = dict(
|
|
title="smoke",
|
|
steps=[WorkflowStep(text="hi")],
|
|
schedule=ScheduleConfig(
|
|
enabled=True, repeat_unit="day", repeat_every=1,
|
|
hour=9, minute=0, timezone="America/Los_Angeles",
|
|
),
|
|
)
|
|
base.update(overrides)
|
|
return Workflow(**base)
|
|
|
|
|
|
async def test_loop_fires_due_workflow(monkeypatch):
|
|
"""Arm a workflow to fire ~now and assert the executor was actually
|
|
invoked by the scheduler loop within the test window. Note the save
|
|
happens AFTER scheduler.start() so reconcile_on_startup doesn't
|
|
clobber next_run_at."""
|
|
from backend.apps.workflows import storage, scheduler, executor
|
|
|
|
fired = asyncio.Event()
|
|
captured: dict = {}
|
|
|
|
async def fake_execute(wf, triggered_by="schedule", scheduled_for=None):
|
|
captured["wf_id"] = wf.id
|
|
captured["triggered_by"] = triggered_by
|
|
captured["scheduled_for"] = scheduled_for
|
|
from backend.apps.workflows.models import WorkflowRun
|
|
run = WorkflowRun(
|
|
workflow_id=wf.id,
|
|
status="success",
|
|
scheduled_for=scheduled_for,
|
|
started_at=datetime.now(timezone.utc),
|
|
finished_at=datetime.now(timezone.utc),
|
|
triggered_by=triggered_by,
|
|
)
|
|
storage.record_run(run)
|
|
fired.set()
|
|
return run
|
|
|
|
monkeypatch.setattr(executor, "execute", fake_execute)
|
|
|
|
await scheduler.start()
|
|
try:
|
|
wf = _make_wf()
|
|
wf.next_run_at = datetime.now(timezone.utc) + timedelta(seconds=1)
|
|
storage.save_workflow(wf)
|
|
scheduler.kick() # force immediate tick
|
|
# Wait up to 5s for the fire to land.
|
|
await asyncio.wait_for(fired.wait(), timeout=5.0)
|
|
finally:
|
|
await scheduler.stop()
|
|
|
|
assert captured.get("wf_id") == wf.id
|
|
assert captured.get("triggered_by") == "schedule"
|
|
runs = storage.list_runs(wf.id, limit=10)
|
|
assert len(runs) == 1
|
|
assert runs[0].status == "success"
|
|
# Scheduler should have rolled next_run_at forward to a future slot.
|
|
after = storage.get_workflow(wf.id)
|
|
assert after.next_run_at is not None
|
|
assert after.next_run_at > datetime.now(timezone.utc)
|
|
|
|
|
|
async def test_disabled_workflow_does_not_fire(monkeypatch):
|
|
"""Master switch off => loop never invokes the executor even if
|
|
next_run_at is in the past."""
|
|
from backend.apps.workflows import storage, scheduler, executor
|
|
|
|
fake = AsyncMock()
|
|
monkeypatch.setattr(executor, "execute", fake)
|
|
|
|
wf = _make_wf()
|
|
wf.schedule.enabled = False
|
|
wf.next_run_at = datetime.now(timezone.utc) - timedelta(seconds=10)
|
|
storage.save_workflow(wf)
|
|
|
|
await scheduler.start()
|
|
try:
|
|
scheduler.kick()
|
|
await asyncio.sleep(2.0)
|
|
finally:
|
|
await scheduler.stop()
|
|
fake.assert_not_called()
|
|
|
|
|
|
async def test_paused_state_blocks_all_fires(monkeypatch):
|
|
"""Global pause flag wins over per-workflow enabled state."""
|
|
from backend.apps.workflows import storage, scheduler, executor
|
|
|
|
fake = AsyncMock()
|
|
monkeypatch.setattr(executor, "execute", fake)
|
|
|
|
wf = _make_wf()
|
|
wf.next_run_at = datetime.now(timezone.utc) - timedelta(seconds=1)
|
|
storage.save_workflow(wf)
|
|
storage.set_paused(True)
|
|
|
|
await scheduler.start()
|
|
try:
|
|
scheduler.kick()
|
|
await asyncio.sleep(2.0)
|
|
finally:
|
|
await scheduler.stop()
|
|
fake.assert_not_called()
|
|
storage.set_paused(False)
|
|
|
|
|
|
async def test_reconcile_captures_missed_fires(monkeypatch):
|
|
"""A daily workflow whose next_run_at elapsed while the app was closed =>
|
|
startup captures the missed fires as pending MissedRuns and rolls
|
|
next_run_at forward (no auto-firing)."""
|
|
from backend.apps.workflows import storage, scheduler
|
|
# created_at must predate the missed window; occurrences_between never enumerates fires from before the workflow existed.
|
|
wf = _make_wf(created_at=datetime.now(timezone.utc) - timedelta(days=10))
|
|
wf.next_run_at = datetime.now(timezone.utc) - timedelta(days=3)
|
|
storage.save_workflow(wf)
|
|
scheduler.reconcile_on_startup()
|
|
missed = [m for m in storage.list_missed() if m.workflow_id == wf.id]
|
|
assert len(missed) >= 1
|
|
assert all(m.scheduled_for < datetime.now(timezone.utc) for m in missed)
|
|
after = storage.get_workflow(wf.id)
|
|
assert after.next_run_at is not None
|
|
assert after.next_run_at > datetime.now(timezone.utc)
|
|
|
|
|
|
async def test_reconcile_no_missed_when_future(monkeypatch):
|
|
"""next_run_at in the future => nothing missed, nothing captured."""
|
|
from backend.apps.workflows import storage, scheduler
|
|
wf = _make_wf(created_at=datetime.now(timezone.utc) - timedelta(days=10))
|
|
wf.next_run_at = datetime.now(timezone.utc) + timedelta(hours=6)
|
|
storage.save_workflow(wf)
|
|
scheduler.reconcile_on_startup()
|
|
assert [m for m in storage.list_missed() if m.workflow_id == wf.id] == []
|
|
|
|
|
|
async def test_reconcile_over_cap_collapses_to_skipped(monkeypatch):
|
|
"""A 15-minute schedule off for days => only the cap is kept reviewable;
|
|
the rest collapse into a single skipped run in history."""
|
|
from backend.apps.workflows import storage, scheduler
|
|
from backend.apps.workflows.models import ScheduleConfig
|
|
wf = _make_wf(
|
|
created_at=datetime.now(timezone.utc) - timedelta(days=30),
|
|
schedule=ScheduleConfig(
|
|
enabled=True, repeat_unit="minute", repeat_every=15,
|
|
timezone="America/Los_Angeles",
|
|
),
|
|
)
|
|
wf.next_run_at = datetime.now(timezone.utc) - timedelta(days=2)
|
|
storage.save_workflow(wf)
|
|
scheduler.reconcile_on_startup()
|
|
missed = [m for m in storage.list_missed() if m.workflow_id == wf.id]
|
|
assert len(missed) == scheduler.PER_WORKFLOW_MISSED_CAP
|
|
skipped = [r for r in storage.list_runs(wf.id, limit=50) if r.status == "skipped"]
|
|
assert len(skipped) == 1
|
|
|
|
|
|
async def test_dismiss_missed_records_skipped(monkeypatch):
|
|
"""Dismissing a missed run drops it from pending and leaves a skipped
|
|
run in history."""
|
|
from backend.apps.workflows import storage
|
|
from backend.apps.workflows.models import MissedRun, MissedRunAction
|
|
from backend.apps.workflows.workflows import dismiss_missed_runs
|
|
wf = _make_wf()
|
|
storage.save_workflow(wf)
|
|
m = MissedRun(workflow_id=wf.id, scheduled_for=datetime.now(timezone.utc) - timedelta(hours=2))
|
|
storage.add_missed(m)
|
|
res = await dismiss_missed_runs(MissedRunAction(ids=[m.id]))
|
|
assert res["dismissed"] == 1
|
|
assert storage.list_missed() == []
|
|
skipped = [r for r in storage.list_runs(wf.id, limit=10) if r.status == "skipped"]
|
|
assert len(skipped) == 1
|
|
|
|
|
|
async def test_run_missed_runs_clears_pending_and_fires(monkeypatch):
|
|
"""Running selected missed fires removes them from pending and invokes
|
|
the executor once per fire, sequentially."""
|
|
from backend.apps.workflows import storage, executor, scheduler
|
|
from backend.apps.workflows.models import MissedRun, MissedRunAction
|
|
from backend.apps.workflows.workflows import run_missed_runs
|
|
|
|
calls: list = []
|
|
|
|
async def fake_execute(wf, triggered_by="schedule", scheduled_for=None):
|
|
calls.append(scheduled_for)
|
|
from backend.apps.workflows.models import WorkflowRun
|
|
return WorkflowRun(workflow_id=wf.id, status="ran_late", scheduled_for=scheduled_for)
|
|
|
|
monkeypatch.setattr(executor, "execute", fake_execute)
|
|
|
|
wf = _make_wf()
|
|
storage.save_workflow(wf)
|
|
m1 = MissedRun(workflow_id=wf.id, scheduled_for=datetime.now(timezone.utc) - timedelta(hours=3))
|
|
m2 = MissedRun(workflow_id=wf.id, scheduled_for=datetime.now(timezone.utc) - timedelta(hours=2))
|
|
storage.add_missed(m1)
|
|
storage.add_missed(m2)
|
|
res = await run_missed_runs(MissedRunAction(ids=[m1.id, m2.id]))
|
|
assert res["started"] == 2
|
|
assert storage.list_missed() == []
|
|
# The endpoint spawns the sequence as a background task; give it a beat.
|
|
await asyncio.sleep(0.05)
|
|
assert len(calls) == 2
|
|
|
|
|
|
async def test_create_workflow_schedules_next_fire():
|
|
"""POST-like create path: enabled schedule => next_run_at populated
|
|
by compute_next_fire."""
|
|
from backend.apps.workflows.models import Workflow, ScheduleConfig, WorkflowStep
|
|
from backend.apps.workflows import scheduler
|
|
wf = Workflow(
|
|
title="t",
|
|
steps=[WorkflowStep(text="hi")],
|
|
schedule=ScheduleConfig(
|
|
enabled=True, repeat_unit="week", repeat_every=1, on_days=[0],
|
|
hour=9, minute=0, timezone="America/Los_Angeles",
|
|
),
|
|
)
|
|
nxt = scheduler.compute_next_fire(wf)
|
|
assert nxt is not None
|
|
assert nxt > datetime.now(timezone.utc)
|
|
tz = ZoneInfo("America/Los_Angeles")
|
|
local = nxt.astimezone(tz)
|
|
assert local.weekday() == 6 # Python: Sunday
|
|
assert (local.hour, local.minute) == (9, 0)
|
|
|
|
|
|
async def test_next_run_at_advances_after_fire(monkeypatch):
|
|
"""After a fire the loop should re-compute next_run_at into the
|
|
future and persist it, so the same fire can't repeat in the same
|
|
minute."""
|
|
from backend.apps.workflows import storage, scheduler, executor
|
|
|
|
fired = asyncio.Event()
|
|
|
|
async def fake_execute(wf, triggered_by="schedule", scheduled_for=None):
|
|
from backend.apps.workflows.models import WorkflowRun
|
|
run = WorkflowRun(
|
|
workflow_id=wf.id, status="success", scheduled_for=scheduled_for,
|
|
started_at=datetime.now(timezone.utc), finished_at=datetime.now(timezone.utc),
|
|
triggered_by=triggered_by,
|
|
)
|
|
storage.record_run(run)
|
|
fired.set()
|
|
return run
|
|
|
|
monkeypatch.setattr(executor, "execute", fake_execute)
|
|
|
|
await scheduler.start()
|
|
try:
|
|
wf = _make_wf()
|
|
armed_at = datetime.now(timezone.utc) + timedelta(seconds=1)
|
|
wf.next_run_at = armed_at
|
|
storage.save_workflow(wf)
|
|
scheduler.kick()
|
|
await asyncio.wait_for(fired.wait(), timeout=5.0)
|
|
# Give the loop one extra tick to persist next_run_at.
|
|
await asyncio.sleep(0.2)
|
|
finally:
|
|
await scheduler.stop()
|
|
|
|
after = storage.get_workflow(wf.id)
|
|
assert after.next_run_at is not None
|
|
assert after.next_run_at > armed_at, "scheduler did not advance next_run_at past the slot it just fired"
|
|
|
|
|
|
async def test_kick_wakes_loop_before_timeout(monkeypatch):
|
|
"""kick() should wake the loop early so manual schedule edits don't
|
|
have to wait a full minute for the next tick boundary."""
|
|
from backend.apps.workflows import storage, scheduler, executor
|
|
|
|
fired = asyncio.Event()
|
|
|
|
async def fake_execute(wf, triggered_by="schedule", scheduled_for=None):
|
|
from backend.apps.workflows.models import WorkflowRun
|
|
run = WorkflowRun(
|
|
workflow_id=wf.id, status="success",
|
|
started_at=datetime.now(timezone.utc),
|
|
finished_at=datetime.now(timezone.utc), triggered_by=triggered_by,
|
|
)
|
|
storage.record_run(run)
|
|
fired.set()
|
|
return run
|
|
|
|
monkeypatch.setattr(executor, "execute", fake_execute)
|
|
|
|
await scheduler.start()
|
|
try:
|
|
wf = _make_wf()
|
|
wf.next_run_at = datetime.now(timezone.utc) - timedelta(seconds=1)
|
|
storage.save_workflow(wf)
|
|
scheduler.kick()
|
|
# Without kick(), the loop would sleep up to 60s before checking the freshly-saved workflow. With kick, it should fire fast.
|
|
await asyncio.wait_for(fired.wait(), timeout=3.0)
|
|
finally:
|
|
await scheduler.stop()
|
|
|
|
|
|
async def test_last_day_of_month_fires_on_month_end():
|
|
"""last_day_of_month ignores day_of_month and lands on the calendar's
|
|
final day, so it survives short months (Feb) instead of clamping."""
|
|
from backend.apps.workflows.models import ScheduleConfig
|
|
from backend.apps.workflows import scheduler
|
|
sched = ScheduleConfig(
|
|
enabled=True, repeat_unit="month", repeat_every=1,
|
|
day_of_month=15, last_day_of_month=True, hour=9, minute=0,
|
|
timezone="America/Los_Angeles",
|
|
)
|
|
wf = _make_wf(schedule=sched)
|
|
tz = ZoneInfo("America/Los_Angeles")
|
|
# From mid-February, the next fire is Feb 28 (or 29 on a leap year), NOT the 15th.
|
|
ref = datetime(2026, 2, 10, 12, 0, tzinfo=tz).astimezone(timezone.utc)
|
|
nxt = scheduler.compute_next_fire(wf, ref=ref)
|
|
local = nxt.astimezone(tz)
|
|
assert local.month == 2 and local.day == 28
|
|
assert local.hour == 9
|
|
# From end of Feb, the following fire rolls to Mar 31 (last day again).
|
|
nxt2 = scheduler.compute_next_fire(wf, ref=nxt)
|
|
local2 = nxt2.astimezone(tz)
|
|
assert local2.month == 3 and local2.day == 31
|
|
|
|
|
|
async def test_soft_deleted_excluded_from_list():
|
|
"""Soft-deleted workflows drop out of list_workflows (so the scheduler and
|
|
every list view skip them) but stay visible to list_deleted_workflows."""
|
|
from backend.apps.workflows import storage
|
|
live = _make_wf(title="live")
|
|
trashed = _make_wf(title="trashed")
|
|
trashed.deleted_at = datetime.now()
|
|
storage.save_workflow(live)
|
|
storage.save_workflow(trashed)
|
|
active_ids = {w.id for w in storage.list_workflows()}
|
|
deleted_ids = {w.id for w in storage.list_deleted_workflows()}
|
|
assert live.id in active_ids and trashed.id not in active_ids
|
|
assert trashed.id in deleted_ids and live.id not in deleted_ids
|
|
# get_workflow still resolves a trashed record so restore/purge can fetch it.
|
|
assert storage.get_workflow(trashed.id) is not None
|