Repository navigation
Expand file tree
/
Copy pathneuro_monitor.py
More file actions
1787 lines (1444 loc) · 60.2 KB
/
Copy pathneuro_monitor.py
File metadata and controls
1787 lines (1444 loc) · 60.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""
neuro_monitor.py - Monitoring System for Neuron Framework
This module implements the monitoring and observability functionality for the
Neuron framework. It collects metrics from agents, circuits, and system
components, providing insights into performance, resource usage, and behavior.
The monitoring system is inspired by how neuroscientists observe and measure
brain activity to understand neural circuits and diagnose issues.
"""
import asyncio
import json
import logging
import threading
import time
from dataclasses import asdict, dataclass, field
from enum import Enum
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
from .config import config
from .exceptions import MonitoringError
from .neuron_types import AgentID, CircuitID
logger = logging.getLogger(__name__)
class MetricType(Enum):
"""
Types of metrics collected by the monitoring system.
This categorizes metrics based on what they measure and how
they should be processed and visualized.
"""
COUNTER = "counter" # Monotonically increasing value
GAUGE = "gauge" # Value that can go up and down
HISTOGRAM = "histogram" # Distribution of values
TIMER = "timer" # Duration of operations
EVENT = "event" # Discrete event occurrence
@dataclass
class Metric:
"""
Represents a collected metric.
A metric includes metadata about what it measures and how it
should be interpreted, along with the actual measured values.
"""
name: str # Metric name
metric_type: MetricType # Type of metric
value: Any # Measured value
tags: Dict[str, str] = field(default_factory=dict) # Tags for filtering/grouping
timestamp: float = field(default_factory=time.time) # When the metric was collected
def to_dict(self) -> Dict[str, Any]:
"""Convert to dictionary representation."""
return {
"name": self.name,
"type": self.metric_type.value,
"value": self.value,
"tags": self.tags,
"timestamp": self.timestamp
}
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> 'Metric':
"""Create from dictionary representation."""
return cls(
name=data["name"],
metric_type=MetricType(data["type"]),
value=data["value"],
tags=data.get("tags", {}),
timestamp=data.get("timestamp", time.time())
)
class AlertLevel(Enum):
"""
Severity levels for monitoring alerts.
This defines how urgent and important an alert is, influencing
how it's presented and processed.
"""
INFO = "info" # Informational, no action required
WARNING = "warning" # Potential issue, may require attention
ERROR = "error" # Significant issue, requires attention
CRITICAL = "critical" # Severe issue, requires immediate attention
@dataclass
class Alert:
"""
Represents a monitoring alert.
Alerts are generated when metrics or events indicate a potential
issue or noteworthy condition in the system.
"""
name: str # Alert name
level: AlertLevel # Severity level
message: str # Alert message
source: str # Component that generated the alert
tags: Dict[str, str] = field(default_factory=dict) # Tags for filtering/grouping
timestamp: float = field(default_factory=time.time) # When the alert was generated
related_metrics: List[str] = field(default_factory=list) # Related metric names
def to_dict(self) -> Dict[str, Any]:
"""Convert to dictionary representation."""
return {
"name": self.name,
"level": self.level.value,
"message": self.message,
"source": self.source,
"tags": self.tags,
"timestamp": self.timestamp,
"related_metrics": self.related_metrics
}
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> 'Alert':
"""Create from dictionary representation."""
return cls(
name=data["name"],
level=AlertLevel(data["level"]),
message=data["message"],
source=data["source"],
tags=data.get("tags", {}),
timestamp=data.get("timestamp", time.time()),
related_metrics=data.get("related_metrics", [])
)
@dataclass
class HealthStatus:
"""
Represents the health status of a component.
Health status provides a high-level assessment of a component's
condition, indicating whether it's functioning properly.
"""
component: str # Component name
status: str # Status string (e.g., "healthy", "degraded", "failing")
details: Dict[str, Any] = field(default_factory=dict) # Additional details
timestamp: float = field(default_factory=time.time) # When the status was assessed
def to_dict(self) -> Dict[str, Any]:
"""Convert to dictionary representation."""
return {
"component": self.component,
"status": self.status,
"details": self.details,
"timestamp": self.timestamp
}
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> 'HealthStatus':
"""Create from dictionary representation."""
return cls(
component=data["component"],
status=data["status"],
details=data.get("details", {}),
timestamp=data.get("timestamp", time.time())
)
class AlertRule:
"""
Rule for generating alerts based on metrics.
Alert rules define conditions under which metrics or events
should trigger alerts, allowing for automated monitoring.
"""
def __init__(self, name: str, metric_pattern: str,
condition: Callable[[Any], bool], alert_level: AlertLevel,
message_template: str, cooldown: float = 300.0):
"""
Initialize an alert rule.
Args:
name: Name of the rule
metric_pattern: Pattern for matching metric names
condition: Function that takes a metric value and returns True if the rule should trigger
alert_level: Severity level for generated alerts
message_template: Template for alert messages
cooldown: Minimum time between alerts from this rule (seconds)
"""
self.name = name
self.metric_pattern = metric_pattern
self.condition = condition
self.alert_level = alert_level
self.message_template = message_template
self.cooldown = cooldown
self.last_triggered = 0.0
def matches_metric(self, metric_name: str) -> bool:
"""
Check if a metric matches this rule's pattern.
Args:
metric_name: Name of the metric to check
Returns:
True if the metric matches, False otherwise
"""
# Simple wildcard matching (can be enhanced with regex)
if self.metric_pattern.endswith('*'):
prefix = self.metric_pattern[:-1]
return metric_name.startswith(prefix)
return metric_name == self.metric_pattern
def check_condition(self, metric: Metric) -> bool:
"""
Check if a metric's value triggers this rule.
Args:
metric: Metric to check
Returns:
True if the rule is triggered, False otherwise
"""
# Check cooldown
current_time = time.time()
if current_time - self.last_triggered < self.cooldown:
return False
# Check condition
try:
if self.condition(metric.value):
self.last_triggered = current_time
return True
except Exception as e:
logger.error(f"Error checking alert condition for {metric.name}: {e}")
return False
def generate_alert(self, metric: Metric) -> Alert:
"""
Generate an alert for a triggering metric.
Args:
metric: Metric that triggered the rule
Returns:
Generated alert
"""
# Replace placeholders in message template
message = self.message_template
message = message.replace("{metric_name}", metric.name)
message = message.replace("{value}", str(metric.value))
for tag_key, tag_value in metric.tags.items():
message = message.replace(f"{{{tag_key}}}", str(tag_value))
# Create the alert
alert = Alert(
name=self.name,
level=self.alert_level,
message=message,
source=metric.tags.get("component", "unknown"),
tags=metric.tags,
related_metrics=[metric.name]
)
return alert
class MetricStore:
"""
Stores and manages collected metrics.
The MetricStore is responsible for storing metrics, handling
retention policies, and providing efficient query capabilities.
"""
def __init__(self, max_retention: int = 86400):
"""
Initialize a metric store.
Args:
max_retention: Maximum retention period in seconds
"""
self._metrics = {} # metric_name -> List of (timestamp, value) pairs
self._tags = {} # metric_name -> Dict of tag_name -> tag_value
self._max_retention = max_retention
self._lock = threading.RLock()
# Storage for recent metric values
self._latest_values = {} # metric_name -> (timestamp, value) pair
logger.debug(f"Initialized MetricStore with {max_retention}s retention")
def store_metric(self, metric: Metric) -> None:
"""
Store a metric.
Args:
metric: Metric to store
"""
with self._lock:
# Store the metric value
if metric.name not in self._metrics:
self._metrics[metric.name] = []
self._metrics[metric.name].append((metric.timestamp, metric.value))
# Store tags
self._tags[metric.name] = metric.tags
# Update latest value
self._latest_values[metric.name] = (metric.timestamp, metric.value)
# Apply retention policy
self._apply_retention(metric.name)
def _apply_retention(self, metric_name: str) -> None:
"""
Apply retention policy to a metric.
Args:
metric_name: Name of the metric to apply retention to
"""
if metric_name not in self._metrics:
return
# Get the current time
current_time = time.time()
# Remove values older than max_retention
cutoff_time = current_time - self._max_retention
self._metrics[metric_name] = [
(ts, val) for ts, val in self._metrics[metric_name]
if ts >= cutoff_time
]
# If all values were removed, clean up
if not self._metrics[metric_name]:
del self._metrics[metric_name]
if metric_name in self._tags:
del self._tags[metric_name]
if metric_name in self._latest_values:
del self._latest_values[metric_name]
def get_metric_value(self, metric_name: str,
time_range: Optional[Tuple[float, float]] = None,
aggregation: Optional[str] = None) -> Optional[Any]:
"""
Get a metric value.
Args:
metric_name: Name of the metric
time_range: Optional (start, end) time range
aggregation: Optional aggregation function (mean, max, min, etc.)
Returns:
Metric value or aggregated values, or None if not found
"""
with self._lock:
if metric_name not in self._metrics:
return None
# Get values in the specified time range
values = self._metrics[metric_name]
if time_range:
start_time, end_time = time_range
values = [
(ts, val) for ts, val in values
if start_time <= ts <= end_time
]
if not values:
return None
# Return values based on aggregation
if aggregation:
if aggregation == "latest":
return values[-1][1]
value_list = [val for _, val in values]
if aggregation == "mean":
return sum(value_list) / len(value_list)
elif aggregation == "max":
return max(value_list)
elif aggregation == "min":
return min(value_list)
elif aggregation == "sum":
return sum(value_list)
elif aggregation == "count":
return len(value_list)
# Default: return all values
return values
def get_latest_value(self, metric_name: str) -> Optional[Any]:
"""
Get the latest value of a metric.
Args:
metric_name: Name of the metric
Returns:
Latest value of the metric, or None if not found
"""
with self._lock:
if metric_name in self._latest_values:
return self._latest_values[metric_name][1]
return None
def get_metrics_by_pattern(self, pattern: str) -> Dict[str, Any]:
"""
Get metrics matching a name pattern.
Args:
pattern: Pattern to match metric names against
Returns:
Dictionary of metric_name -> latest_value
"""
with self._lock:
results = {}
# Simple wildcard matching
if pattern.endswith('*'):
prefix = pattern[:-1]
for name, (_, value) in self._latest_values.items():
if name.startswith(prefix):
results[name] = value
else:
# Exact match
if pattern in self._latest_values:
results[pattern] = self._latest_values[pattern][1]
return results
def get_metrics_by_tags(self, tags: Dict[str, str]) -> Dict[str, Any]:
"""
Get metrics with matching tags.
Args:
tags: Tags to match
Returns:
Dictionary of metric_name -> latest_value
"""
with self._lock:
results = {}
for name, metric_tags in self._tags.items():
# Check if all specified tags match
match = True
for tag_key, tag_value in tags.items():
if tag_key not in metric_tags or metric_tags[tag_key] != tag_value:
match = False
break
if match and name in self._latest_values:
results[name] = self._latest_values[name][1]
return results
def clear_metrics(self) -> None:
"""Clear all stored metrics."""
with self._lock:
self._metrics.clear()
self._tags.clear()
self._latest_values.clear()
class AlertManager:
"""
Manages alert rules and generated alerts.
The AlertManager is responsible for defining alert rules,
checking metrics against those rules, and handling triggered alerts.
"""
def __init__(self, max_alerts: int = 1000):
"""
Initialize an alert manager.
Args:
max_alerts: Maximum number of alerts to store
"""
self._rules = [] # List of AlertRule objects
self._alerts = [] # List of Alert objects
self._alert_handlers = [] # List of alert handler functions
self._max_alerts = max_alerts
self._lock = threading.RLock()
logger.debug(f"Initialized AlertManager with {max_alerts} max alerts")
def add_rule(self, rule: AlertRule) -> None:
"""
Add an alert rule.
Args:
rule: Alert rule to add
"""
with self._lock:
self._rules.append(rule)
def remove_rule(self, rule_name: str) -> bool:
"""
Remove an alert rule.
Args:
rule_name: Name of the rule to remove
Returns:
True if the rule was removed, False if not found
"""
with self._lock:
for i, rule in enumerate(self._rules):
if rule.name == rule_name:
del self._rules[i]
return True
return False
def add_alert_handler(self, handler: Callable[[Alert], None]) -> None:
"""
Add an alert handler function.
Args:
handler: Function to call when an alert is triggered
"""
with self._lock:
self._alert_handlers.append(handler)
def check_metric(self, metric: Metric) -> List[Alert]:
"""
Check a metric against all alert rules.
Args:
metric: Metric to check
Returns:
List of triggered alerts
"""
triggered_alerts = []
with self._lock:
for rule in self._rules:
if rule.matches_metric(metric.name) and rule.check_condition(metric):
alert = rule.generate_alert(metric)
self._alerts.append(alert)
triggered_alerts.append(alert)
# Call alert handlers
for handler in self._alert_handlers:
try:
handler(alert)
except Exception as e:
logger.error(f"Error in alert handler: {e}")
# Trim alerts if needed
if len(self._alerts) > self._max_alerts:
self._alerts = self._alerts[-self._max_alerts:]
return triggered_alerts
def get_alerts(self, level: Optional[AlertLevel] = None,
time_range: Optional[Tuple[float, float]] = None,
max_alerts: Optional[int] = None) -> List[Alert]:
"""
Get stored alerts.
Args:
level: Optional filter by alert level
time_range: Optional (start, end) time range
max_alerts: Maximum number of alerts to return
Returns:
List of matching alerts
"""
with self._lock:
# Filter alerts
filtered_alerts = self._alerts
if level:
filtered_alerts = [a for a in filtered_alerts if a.level == level]
if time_range:
start_time, end_time = time_range
filtered_alerts = [
a for a in filtered_alerts
if start_time <= a.timestamp <= end_time
]
# Sort by timestamp (newest first)
filtered_alerts = sorted(filtered_alerts, key=lambda a: a.timestamp, reverse=True)
# Apply limit
if max_alerts is not None:
filtered_alerts = filtered_alerts[:max_alerts]
return filtered_alerts
def clear_alerts(self) -> None:
"""Clear all stored alerts."""
with self._lock:
self._alerts.clear()
class HealthChecker:
"""
Monitors the health of system components.
The HealthChecker assesses the health of various components
based on their metrics and behavior, providing a high-level
view of system status.
"""
def __init__(self, metric_store: MetricStore):
"""
Initialize a health checker.
Args:
metric_store: MetricStore to use for health assessment
"""
self._metric_store = metric_store
self._health_checks = {} # component_name -> health check function
self._health_status = {} # component_name -> HealthStatus
self._lock = threading.RLock()
logger.debug("Initialized HealthChecker")
def add_health_check(self, component: str,
check_function: Callable[[MetricStore], HealthStatus]) -> None:
"""
Add a health check function.
Args:
component: Name of the component to check
check_function: Function that assesses component health
"""
with self._lock:
self._health_checks[component] = check_function
def remove_health_check(self, component: str) -> bool:
"""
Remove a health check function.
Args:
component: Name of the component to remove
Returns:
True if the check was removed, False if not found
"""
with self._lock:
if component in self._health_checks:
del self._health_checks[component]
return True
return False
def run_health_checks(self) -> Dict[str, HealthStatus]:
"""
Run all health checks.
Returns:
Dictionary of component -> HealthStatus
"""
with self._lock:
for component, check_function in self._health_checks.items():
try:
status = check_function(self._metric_store)
self._health_status[component] = status
except Exception as e:
logger.error(f"Error in health check for {component}: {e}")
# Create a failing status
self._health_status[component] = HealthStatus(
component=component,
status="failing",
details={"error": str(e)}
)
return self._health_status.copy()
def get_health_status(self, component: Optional[str] = None) -> Union[Dict[str, HealthStatus], Optional[HealthStatus]]:
"""
Get health status.
Args:
component: Optional component name to get status for
Returns:
Health status for the component, or dictionary of all statuses
"""
with self._lock:
if component:
return self._health_status.get(component)
return self._health_status.copy()
def clear_health_status(self) -> None:
"""Clear all health status records."""
with self._lock:
self._health_status.clear()
class MetricCollector:
"""
Collects metrics from system components.
The MetricCollector is responsible for gathering metrics from
various sources, preprocessing them, and storing them for analysis.
"""
def __init__(self, metric_store: MetricStore, alert_manager: AlertManager):
"""
Initialize a metric collector.
Args:
metric_store: MetricStore to store collected metrics
alert_manager: AlertManager to check metrics against rules
"""
self._metric_store = metric_store
self._alert_manager = alert_manager
self._collection_tasks = {} # task_name -> (interval, collector_function)
self._stop_event = threading.Event()
self._collection_threads = {} # task_name -> thread
logger.debug("Initialized MetricCollector")
def add_collection_task(self, task_name: str, interval: float,
collector: Callable[[], List[Metric]]) -> None:
"""
Add a metric collection task.
Args:
task_name: Name of the collection task
interval: Collection interval in seconds
collector: Function that collects metrics
"""
self._collection_tasks[task_name] = (interval, collector)
logger.debug(f"Added collection task: {task_name} (interval: {interval}s)")
def remove_collection_task(self, task_name: str) -> bool:
"""
Remove a collection task.
Args:
task_name: Name of the task to remove
Returns:
True if the task was removed, False if not found
"""
if task_name in self._collection_tasks:
del self._collection_tasks[task_name]
# Stop the collection thread if running
if task_name in self._collection_threads:
self._collection_threads[task_name].join(timeout=1.0)
del self._collection_threads[task_name]
logger.debug(f"Removed collection task: {task_name}")
return True
return False
def start_collection(self) -> None:
"""
Start metric collection.
This starts the collection threads for all registered tasks.
"""
self._stop_event.clear()
# Start collection threads
for task_name, (interval, collector) in self._collection_tasks.items():
thread = threading.Thread(
target=self._collection_loop,
args=(task_name, interval, collector),
name=f"MetricCollector-{task_name}",
daemon=True
)
thread.start()
self._collection_threads[task_name] = thread
logger.info(f"Started metric collection for {len(self._collection_tasks)} tasks")
def stop_collection(self) -> None:
"""
Stop metric collection.
This stops all collection threads.
"""
self._stop_event.set()
# Wait for threads to finish
for task_name, thread in list(self._collection_threads.items()):
thread.join(timeout=1.0)
if thread.is_alive():
logger.warning(f"Collection thread for {task_name} did not stop cleanly")
del self._collection_threads[task_name]
logger.info("Stopped metric collection")
def collect_metric(self, metric: Metric) -> List[Alert]:
"""
Collect and process a single metric.
Args:
metric: Metric to collect
Returns:
List of triggered alerts
"""
# Store the metric
self._metric_store.store_metric(metric)
# Check for alerts
alerts = self._alert_manager.check_metric(metric)
return alerts
def _collection_loop(self, task_name: str, interval: float,
collector: Callable[[], List[Metric]]) -> None:
"""
Collection loop for a task.
This runs in a separate thread and periodically collects
metrics using the provided collector function.
Args:
task_name: Name of the collection task
interval: Collection interval in seconds
collector: Function that collects metrics
"""
logger.debug(f"Starting collection loop for {task_name}")
while not self._stop_event.is_set():
try:
# Collect metrics
metrics = collector()
# Process each metric
for metric in metrics:
self.collect_metric(metric)
logger.debug(f"Collected {len(metrics)} metrics for {task_name}")
except Exception as e:
logger.error(f"Error collecting metrics for {task_name}: {e}")
# Wait for next collection cycle
if self._stop_event.wait(interval):
break
logger.debug(f"Stopped collection loop for {task_name}")
class NeuroMonitor:
"""
Main monitoring system for the Neuron framework.
The NeuroMonitor integrates the various monitoring components
and provides a unified interface for monitoring and observability.
"""
def __init__(self):
"""Initialize the monitoring system."""
self._metric_store = None
self._alert_manager = None
self._health_checker = None
self._metric_collector = None
self._agent_manager = None
self._synaptic_bus = None
self._circuit_designer = None
self._system_metrics_interval = 10.0
self._health_check_interval = 60.0
self._initialized = False
self._running = False
self._lock = threading.RLock()
logger.info("Initialized NeuroMonitor")
def initialize(self, agent_manager: Any, synaptic_bus: Any,
circuit_designer: Any) -> None:
"""
Initialize the monitoring system with dependencies.
Args:
agent_manager: AgentManager for monitoring agents
synaptic_bus: SynapticBus for monitoring communication
circuit_designer: CircuitDesigner for monitoring circuits
"""
with self._lock:
if self._initialized:
logger.warning("NeuroMonitor is already initialized")
return
# Save dependencies
self._agent_manager = agent_manager
self._synaptic_bus = synaptic_bus
self._circuit_designer = circuit_designer
# Get configuration
self._system_metrics_interval = config.get(
"monitoring", "metrics_interval", 10.0
)
self._health_check_interval = config.get(
"monitoring", "health_check_interval", 60.0
)
retention_period = config.get(
"monitoring", "retention_period", 86400
)
# Create monitoring components
self._metric_store = MetricStore(max_retention=retention_period)
self._alert_manager = AlertManager()
self._health_checker = HealthChecker(self._metric_store)
self._metric_collector = MetricCollector(
self._metric_store, self._alert_manager
)
# Set up collection tasks
self._setup_collection_tasks()
# Set up health checks
self._setup_health_checks()
# Set up alert rules
self._setup_alert_rules()
self._initialized = True
logger.info("NeuroMonitor initialized with dependencies")
def _setup_collection_tasks(self) -> None:
"""Set up metric collection tasks."""
# System metrics collection
self._metric_collector.add_collection_task(
"system_metrics",
self._system_metrics_interval,
self._collect_system_metrics
)
# Agent metrics collection
self._metric_collector.add_collection_task(
"agent_metrics",
self._system_metrics_interval,
self._collect_agent_metrics
)
# Communication metrics collection
self._metric_collector.add_collection_task(
"communication_metrics",
self._system_metrics_interval,
self._collect_communication_metrics
)
# Circuit metrics collection
self._metric_collector.add_collection_task(
"circuit_metrics",
self._system_metrics_interval,
self._collect_circuit_metrics
)
def _setup_health_checks(self) -> None:
"""Set up health check functions."""
# System health check
self._health_checker.add_health_check(
"system",
self._check_system_health
)
# Agent health check
self._health_checker.add_health_check(
"agents",
self._check_agent_health
)
# Communication health check
self._health_checker.add_health_check(
"communication",
self._check_communication_health
)
# Circuit health check
self._health_checker.add_health_check(
"circuits",
self._check_circuit_health
)
def _setup_alert_rules(self) -> None:
"""Set up alert rules."""
# High memory usage alert
self._alert_manager.add_rule(
AlertRule(
name="high_memory_usage",
metric_pattern="system.memory.usage",
condition=lambda x: x > 0.9, # >90% memory usage
alert_level=AlertLevel.WARNING,
message_template="High memory usage: {value}%"
)
)