init

agent/.env.example

@@ -0,0 +1,16 @@
+DASHBOARD_URL=https://dashboard.example.com/api/ingest/
+AGENT_TOKEN=PASTE_TOKEN_FROM_REGISTER_AGENT
+AGENT_INTERVAL=5
+AGENT_SPOOL=/var/lib/dilib-agent/outbox.sqlite3
+AGENT_MAX_PENDING=1000000
+# On every host: optional HTTP health endpoints (name -> URL), checked locally.
+AGENT_HEALTH_URLS={"django":"http://127.0.0.1:8000/health/"}
+# Exactly ONE collector per independent Celery broker/cluster:
+AGENT_COLLECT_EVENTS=0
+AGENT_BROKER_URL=redis://127.0.0.1:6379/0
+AGENT_COLLECTOR_ID=dilib-monitor-production
+# Flower local API/status, optional. API basic auth via separate variables.
+AGENT_FLOWER_URL=http://127.0.0.1:5555
+AGENT_FLOWER_USER=monitor
+AGENT_FLOWER_PASSWORD=
+AGENT_CA_BUNDLE=

agent/import_flower.py

@@ -0,0 +1,33 @@
+"""One-time import of tasks still available in Flower into the persistent outbox."""
+import argparse,os,uuid
+from datetime import datetime,timezone
+from dotenv import load_dotenv
+import requests
+from .outbox import Outbox
+from .main import envelope
+STATES={'SUCCESS':'task-succeeded','FAILURE':'task-failed','REVOKED':'task-revoked','STARTED':'task-started','RECEIVED':'task-received','RETRY':'task-retried','PENDING':'task-sent'}
+def main():
+    p=argparse.ArgumentParser();p.add_argument('--env',default='.env');p.add_argument('--limit',type=int,default=100000);args=p.parse_args()
+    load_dotenv(args.env);os.umask(0o077)
+    if args.limit<1 or args.limit>1000000:raise SystemExit('limit must be between 1 and 1000000')
+    out=Outbox(os.getenv('AGENT_SPOOL','var/outbox.sqlite3'))
+    base=os.environ['AGENT_FLOWER_URL'].rstrip('/')
+    count=0;skipped=0;offset=0
+    while offset<args.limit:
+        r=requests.get(base+'/api/tasks',params={'limit':min(200,args.limit-offset),'offset':offset},auth=(os.getenv('AGENT_FLOWER_USER',''),os.getenv('AGENT_FLOWER_PASSWORD','')),timeout=30)
+        r.raise_for_status();tasks=r.json()
+        if not tasks:break
+        for task_id,t in tasks.items():
+            kind=STATES.get(t.get('state'))
+            stamp=t.get('timestamp') or t.get('succeeded') or t.get('failed') or t.get('started') or t.get('received') or t.get('sent')
+            if not kind or not isinstance(stamp,(int,float)):
+                skipped+=1;continue
+            payload={'uuid':str(task_id),'name':str(t.get('name') or ''),'hostname':str(t.get('worker') or t.get('hostname') or '')}
+            if isinstance(t.get('runtime'),(int,float)):payload['runtime']=t['runtime']
+            e=envelope(kind,payload,datetime.fromtimestamp(stamp,timezone.utc))
+            e['id']=str(uuid.uuid5(uuid.NAMESPACE_URL,base+'|'+task_id+'|'+kind+'|'+str(stamp)))
+            out.add(e);count+=1
+        offset+=len(tasks)
+        if len(tasks)<200:break
+    out.close();print(f'{count} task snapshots queued; {skipped} skipped because state/timestamp was unavailable.')
+if __name__=='__main__':main()

agent/main.py

@@ -0,0 +1,101 @@
+"""Outbound-only host metrics and optional Celery events collector."""
+import json,logging,os,signal,socket,threading,time,uuid
+from datetime import datetime,timezone
+from pathlib import Path
+import psutil,requests
+from celery import Celery
+from dotenv import load_dotenv
+from .outbox import Outbox
+log=logging.getLogger('dilib.agent')
+STOP=threading.Event()
+ALLOWED_TASK={'uuid','name','hostname','runtime','retries'}
+ALLOWED_WORKER={'hostname','active','processed'}
+KINDS={'worker-online','worker-offline','worker-heartbeat','task-sent','task-received','task-started','task-succeeded','task-failed','task-retried','task-revoked','task-rejected'}
+def envelope(kind,payload,at=None):
+    return {'id':str(uuid.uuid4()),'kind':kind,'occurred_at':(at or datetime.now(timezone.utc)).isoformat(),'payload':payload}
+def safe_event(e):
+    kind=e.get('type')
+    if kind not in KINDS:return None
+    keep=ALLOWED_TASK if kind.startswith('task-') else ALLOWED_WORKER
+    p={k:e[k] for k in keep if k in e and e[k] is not None}
+    if kind.startswith('task-') and not p.get('uuid'):return None
+    if kind.startswith('worker-') and not p.get('hostname'):return None
+    return envelope(kind,p,datetime.fromtimestamp(float(e.get('timestamp',time.time())),timezone.utc))
+def collect(outbox,state):
+    app=Celery('dilib-agent',broker=os.environ['AGENT_BROKER_URL'])
+    app.conf.update(accept_content=['json'],event_queue_ttl=300,event_queue_expires=86400)
+    while not STOP.is_set():
+        try:
+            with app.connection() as conn:
+                conn.ensure_connection(max_retries=3)
+                def handler(event):
+                    state['last_event']=time.monotonic()
+                    cleaned=safe_event(event)
+                    if cleaned:outbox.add(cleaned)
+                receiver=app.events.Receiver(conn,handlers={'*':handler},node_id=os.getenv('AGENT_COLLECTOR_ID','dilib-monitor'),accept=['json'])
+                with receiver.consumer_context():
+                    state['connected']=True
+                    while not STOP.is_set():
+                        try:conn.drain_events(timeout=2)
+                        except socket.timeout:pass
+        except Exception as exc:
+            state['connected']=False
+            log.warning('Collector reconnect required (%s)',type(exc).__name__)
+            STOP.wait(5)
+        finally:state['connected']=False
+
+def send_batch(session,outbox,url,token,verify=True):
+    batch=outbox.batch()
+    if not batch:return 0
+    r=session.post(url,json={'events':[e for _,e in batch]},headers={'Authorization':'Agent '+token},timeout=(5,30),verify=verify)
+    r.raise_for_status()
+    if r.json().get('received')!=len(batch): raise RuntimeError('Server acknowledgement mismatch')
+    outbox.ack([seq for seq,_ in batch]);return len(batch)
+def sender(outbox):
+    session=requests.Session();backoff=1
+    while not STOP.is_set():
+        try:
+            n=send_batch(session,outbox,os.environ['DASHBOARD_URL'],os.environ['AGENT_TOKEN'],os.getenv('AGENT_CA_BUNDLE') or True)
+            backoff=1
+            if not n:STOP.wait(1)
+        except Exception as exc:
+            # Tokens, broker URLs, payloads and raw errors are never logged.
+            code=getattr(getattr(exc,'response',None),'status_code',None)
+            log.warning('Upload failed (%s, HTTP %s); data remains in outbox',type(exc).__name__,code)
+            STOP.wait(backoff);backoff=min(backoff*2,60)
+def check_url(url,auth=None):
+    try:
+        r=requests.get(url,auth=auth,timeout=(2,3));return 200<=r.status_code<300
+    except requests.RequestException:return False
+def main():
+    load_dotenv(os.getenv('AGENT_ENV_FILE','.env'))
+    logging.basicConfig(level=logging.INFO,format='%(asctime)s %(levelname)s %(message)s')
+    for key in ('AGENT_TOKEN','DASHBOARD_URL'):
+        if not os.getenv(key):raise RuntimeError(key+' missing')
+    if not os.environ['DASHBOARD_URL'].startswith('https://') and os.getenv('AGENT_ALLOW_HTTP')!='1':
+        raise RuntimeError('HTTPS required (AGENT_ALLOW_HTTP=1 only for local testing)')
+    os.umask(0o077)
+    outbox=Outbox(os.getenv('AGENT_SPOOL','var/outbox.sqlite3'),int(os.getenv('AGENT_MAX_PENDING','1000000')))
+    for sig in (signal.SIGTERM,signal.SIGINT):signal.signal(sig,lambda *_:STOP.set())
+    threads=[threading.Thread(target=sender,args=(outbox,),daemon=True)]
+    state={'connected':False};collecting=os.getenv('AGENT_COLLECT_EVENTS','0')=='1'
+    if collecting:threads.append(threading.Thread(target=collect,args=(outbox,state),daemon=True))
+    for t in threads:t.start()
+    urls=json.loads(os.getenv('AGENT_HEALTH_URLS','{}'))
+    boot=str(int(psutil.boot_time()))
+    psutil.cpu_percent()
+    try:
+        while not STOP.is_set():
+            metrics={'cpu':psutil.cpu_percent(),'ram':psutil.virtual_memory().percent,'disk':psutil.disk_usage('/').percent,'boot_id':boot,'services':{name:check_url(url) for name,url in urls.items()},'outbox':outbox.count()}
+            outbox.add(envelope('metric',metrics))
+            if collecting:outbox.add(envelope('collector-status',{'online':state['connected']}))
+            if os.getenv('AGENT_FLOWER_URL'):
+                auth=(os.getenv('AGENT_FLOWER_USER',''),os.getenv('AGENT_FLOWER_PASSWORD',''))
+                ok=check_url(os.environ['AGENT_FLOWER_URL'].rstrip('/')+'/api/workers',auth)
+                outbox.add(envelope('flower-status',{'online':ok}))
+            STOP.wait(max(1,float(os.getenv('AGENT_INTERVAL','5'))))
+    finally:
+        STOP.set()
+        for t in threads:t.join(timeout=40)
+        if not any(t.is_alive() for t in threads):outbox.close()
+if __name__=='__main__':main()

agent/outbox.py

@@ -0,0 +1,21 @@
+import json,sqlite3,threading
+from pathlib import Path
+class Outbox:
+    def __init__(self,path,limit=1000000):
+        Path(path).parent.mkdir(parents=True,exist_ok=True)
+        self.db=sqlite3.connect(path,check_same_thread=False)
+        self.db.execute('PRAGMA journal_mode=WAL');self.db.execute('PRAGMA synchronous=FULL')
+        self.db.execute('CREATE TABLE IF NOT EXISTS outbox (seq INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT UNIQUE, body TEXT NOT NULL)')
+        self.db.commit();self.lock=threading.Lock();self.limit=limit
+    def add(self,event):
+        with self.lock:
+            if self.db.execute('SELECT COUNT(*) FROM outbox').fetchone()[0]>=self.limit:
+                raise RuntimeError('Outbox full: collector stopped to avoid silent data loss')
+            with self.db:self.db.execute('INSERT OR IGNORE INTO outbox(id,body) VALUES (?,?)',(event['id'],json.dumps(event,allow_nan=False)))
+    def batch(self,limit=200):
+        with self.lock:return [(r[0],json.loads(r[1])) for r in self.db.execute('SELECT seq,body FROM outbox ORDER BY seq LIMIT ?',(limit,))]
+    def ack(self,ids):
+        with self.lock,self.db:self.db.executemany('DELETE FROM outbox WHERE seq=?',[(i,) for i in ids])
+    def count(self):
+        with self.lock:return self.db.execute('SELECT COUNT(*) FROM outbox').fetchone()[0]
+    def close(self): self.db.close()

agent/requirements.txt

@@ -0,0 +1,4 @@
+celery[redis]>=5.5,<6
+requests>=2.32,<3
+psutil>=6,<8
+python-dotenv>=1,<2

agent/spool.py

@@ -0,0 +1,19 @@
+"""Inspect local backlog, or quarantine a known invalid event without losing it."""
+import argparse,json,os
+from .outbox import Outbox
+
+def main():
+    p=argparse.ArgumentParser();p.add_argument('path');p.add_argument('--quarantine-id');p.add_argument('--reason',default='operator review');args=p.parse_args()
+    os.umask(0o077);out=Outbox(args.path)
+    if args.quarantine_id:
+        with out.lock,out.db:
+            out.db.execute('CREATE TABLE IF NOT EXISTS quarantine(id TEXT PRIMARY KEY, body TEXT NOT NULL, reason TEXT NOT NULL)')
+            row=out.db.execute('SELECT id,body FROM outbox WHERE id=?',(args.quarantine_id,)).fetchone()
+            if not row:raise SystemExit('Event ID not found')
+            out.db.execute('INSERT OR REPLACE INTO quarantine VALUES (?,?,?)',(*row,args.reason))
+            out.db.execute('DELETE FROM outbox WHERE id=?',(args.quarantine_id,))
+        print('Event retained in quarantine table; other events can be transmitted.')
+    else:
+        print(json.dumps({'pending':out.count(),'first_events':[e for _,e in out.batch(5)]},indent=2))
+    out.close()
+if __name__=='__main__':main()