import uuid from datetime import timedelta from unittest.mock import patch,Mock import pytest from django.utils import timezone from django.contrib.auth import get_user_model from rest_framework.test import APIClient from monitoring.models import * from monitoring.auth import digest from monitoring.ingest import ingest from monitoring.tasks import evaluate_alarms,cleanup,deliver_notifications from agent.outbox import Outbox from agent.main import envelope,safe_event,send_batch @pytest.fixture def server(db): seg=Segment.objects.create(name='Production') return Server.objects.create(name='host-01',segment=seg) @pytest.fixture def client():return APIClient() @pytest.fixture def auth(client,db): user=get_user_model().objects.create_user(username='operator',password='test-secure-password-834!',is_staff=True) token=AccessToken.objects.create(user=user,digest=digest('test-token'),expires_at=timezone.now()+timedelta(hours=1)) client.credentials(HTTP_AUTHORIZATION='Bearer test-token');return client def ev(kind='metric',payload=None,at=None): return {'id':uuid.uuid4(),'kind':kind,'occurred_at':at or timezone.now(),'payload':payload or {'cpu':22,'ram':45,'disk':61}} @pytest.mark.django_db def test_ingest_auth_scope_idempotence(server,client): payload=ev();payload['id']=str(payload['id']);payload['occurred_at']=payload['occurred_at'].isoformat() assert client.post('/api/ingest/',{'events':[payload]},format='json').status_code==401 AgentCredential.objects.create(server=server,digest=digest('agent-secret')) client.credentials(HTTP_AUTHORIZATION='Agent agent-secret') for _ in range(2):assert client.post('/api/ingest/',{'events':[payload]},format='json').status_code==200 assert Event.objects.count()==1 and Metric.objects.count()==1 payload['id']=str(uuid.uuid4());payload['payload']['cpu']=120 assert client.post('/api/ingest/',{'events':[payload]},format='json').status_code==400 assert Event.objects.count()==1 @pytest.mark.django_db def test_transaction_validation_no_partial_batch(server,client): AgentCredential.objects.create(server=server,digest=digest('agent-secret'));client.credentials(HTTP_AUTHORIZATION='Agent agent-secret') events=[envelope('metric',{'cpu':30,'ram':40,'disk':50}),envelope('metric',{'cpu':-1,'ram':40,'disk':50})] assert client.post('/api/ingest/',{'events':events},format='json').status_code==400 assert Event.objects.count()==0 @pytest.mark.django_db def test_task_out_of_order_and_terminal_state(server): now=timezone.now();task={'uuid':'t1','name':'x'} ingest(server.pk,[ev('task-succeeded',dict(task,runtime=2),now),ev('task-started',task,now-timedelta(seconds=4)),ev('task-received',task,now+timedelta(seconds=1))]) t=Task.objects.get();assert t.state=='SUCCESS' and t.runtime==2 @pytest.mark.django_db def test_stale_metric_replay_not_online(server,auth): ingest(server.pk,[ev(at=timezone.now()-timedelta(hours=1))]) assert auth.get('/api/summary/').json()['servers'][0]['status']=='Offline' evaluate_alarms();assert Alarm.objects.filter(code='heartbeat',resolved_at__isnull=True).exists() @pytest.mark.django_db def test_alarm_recovery_ack_and_permissions(server,auth): ingest(server.pk,[ev(payload={'cpu':95,'ram':20,'disk':30})]);evaluate_alarms();evaluate_alarms() a=Alarm.objects.get(code='cpu');assert Notification.objects.count()==1 assert auth.post(f'/api/alarms/{a.pk}/ack/').status_code==200 assert Audit.objects.filter(action='alarm_ack').count()==1 assert auth.post(f'/api/alarms/{a.pk}/ack/').status_code==200 assert Audit.objects.filter(action='alarm_ack').count()==1 ingest(server.pk,[ev()]);evaluate_alarms();a.refresh_from_db();assert a.resolved_at user=get_user_model().objects.get(username='operator');user.is_staff=False;user.save() assert auth.put('/api/settings/',{},format='json').status_code==403 @pytest.mark.django_db def test_auth_expiry_and_revocation(auth): assert auth.get('/api/summary/').status_code==200 AccessToken.objects.update(expires_at=timezone.now()-timedelta(seconds=1)) assert auth.get('/api/summary/').status_code==401 @pytest.mark.django_db def test_retention(server): ingest(server.pk,[ev(at=timezone.now()-timedelta(days=100)),ev()]);cleanup() assert Event.objects.count()==1 and Metric.objects.count()==1 @pytest.mark.django_db def test_notification_retries(server,settings): settings.ALERT_RECIPIENTS=['ops@example.com'];evaluate_alarms() with patch('monitoring.tasks.send_mail',side_effect=OSError()):deliver_notifications() n=Notification.objects.first();assert n.attempts==1 and not n.sent_at Notification.objects.update(next_attempt=timezone.now()) with patch('monitoring.tasks.send_mail',return_value=1):deliver_notifications() n.refresh_from_db();assert n.sent_at @pytest.mark.django_db def test_report_csv_and_all_views(server,auth): ingest(server.pk,[ev()]) for path in ['summary','servers','tasks','events','alarms','settings','flower','health']: assert auth.get('/api/'+path+'/').status_code==200 assert auth.get(f'/api/servers/{server.pk}/').status_code==200 r=auth.get('/api/report/');assert r.status_code==200 and r.content.startswith(b'\x89PNG') server.name='=EVIL()';server.save() assert "'=EVIL()" in auth.get('/api/export/').content.decode() @pytest.mark.django_db def test_login_rate_limit_and_logout(client): from django.core.cache import cache cache.clear() get_user_model().objects.create_user(username='test-login',password='example-Password!31') response=client.post('/api/auth/login/',{'username':'test-login','password':'example-Password!31'},format='json') assert response.status_code==200 client.credentials(HTTP_AUTHORIZATION='Bearer '+response.json()['token']) assert client.post('/api/auth/logout/').status_code==200 assert client.get('/api/summary/').status_code==401 client.credentials() for _ in range(10):client.post('/api/auth/login/',{'username':'bad','password':'bad'},format='json') assert client.post('/api/auth/login/',{'username':'bad','password':'bad'},format='json').status_code==429 def test_outbox_survives_restart_and_failed_delivery(tmp_path): path=tmp_path/'queue.db';out=Outbox(path);item=envelope('metric',{'cpu':1,'ram':2,'disk':3});out.add(item);out.add(item);out.close() out=Outbox(path);assert out.count()==1 session=Mock();session.post.side_effect=OSError('offline') with pytest.raises(OSError):send_batch(session,out,'https://example.com','secret') assert out.count()==1 session.post.side_effect=None;session.post.return_value.json.return_value={'received':1} assert send_batch(session,out,'https://example.com','secret')==1 and out.count()==0 def test_payload_redaction(): e=safe_event({'type':'task-succeeded','uuid':'a','hostname':'worker','args':'SECRET','result':'SECRET','kwargs':{'password':'SECRET'},'runtime':2}) assert set(e['payload'])=={'uuid','hostname','runtime'} @pytest.mark.django_db def test_ingest_normalizes_numeric_input(server,client): AgentCredential.objects.create(server=server,digest=digest('numeric-agent')) client.credentials(HTTP_AUTHORIZATION='Agent numeric-agent') event=envelope('metric',{'cpu':'20.5','ram':'30','disk':'40','services':{'web':'false'}}) assert client.post('/api/ingest/',{'events':[event]},format='json').status_code==200 server.refresh_from_db() assert server.latest['cpu']==20.5 and server.latest['services']['web'] is False @pytest.mark.django_db def test_batch_cross_agent_collision_rolls_back(server): existing=ev();ingest(server.pk,[existing]) other=Server.objects.create(name='other',segment=server.segment) with pytest.raises(ValueError):ingest(other.pk,[ev(),existing]) assert not Event.objects.filter(server=other).exists() @pytest.mark.django_db def test_payload_allowlist_and_future_clock(server,client): AgentCredential.objects.create(server=server,digest=digest('agent-secret'));client.credentials(HTTP_AUTHORIZATION='Agent agent-secret') item=envelope('task-succeeded',{'uuid':'x','name':'task','args':'secret','result':'secret'}) assert client.post('/api/ingest/',{'events':[item]},format='json').status_code==200 assert Event.objects.get().payload=={'uuid':'x','name':'task'} item=envelope('metric',{'cpu':3,'ram':4,'disk':5},timezone.now()+timedelta(hours=1)) assert client.post('/api/ingest/',{'events':[item]},format='json').status_code==400 @pytest.mark.django_db def test_metrics_survive_new_client_and_flower_restart(server,auth): ingest(server.pk,[ev(),ev('flower-status',{'online':False})]) assert auth.get('/api/summary/').json()['online']==1 ingest(server.pk,[ev('flower-status',{'online':True})]) assert Metric.objects.count()==1 and Event.objects.count()==3 assert auth.get('/api/flower/').json()[0]['online'] is True