#!/usr/bin/env python3
"""
Page Fault Tracer - System Agent Integration
Continuous memory leak detection via page fault pattern analysis
"""

import os
import sys
import json
import time
import logging
import subprocess
from dataclasses import dataclass
from typing import Dict, List, Optional, Callable
from threading import Thread, Event
from collections import defaultdict

from bcc import BPF

@dataclass
class LeakAlert:
    pid: int
    comm: str
    fault_rate: int
    rss_mb: int
    vsz_mb: int
    confidence: int
    severity: int
    timestamp: float
    
    @property
    def severity_label(self) -> str:
        levels = {
            1: "Minimal", 2: "Low", 4: "Medium", 6: "Medium-High", 
            8: "High", 10: "Critical"
        }
        return levels.get(self.severity, "Unknown")

class PageFaultTracer:
    """Production-ready page fault tracer for memory leak detection."""
    
    def __init__(self, config_path: str = None):
        self.config = self._load_config(config_path)
        self.bpf = None
        self.running = Event()
        self.alert_callbacks: List[Callable[[LeakAlert], None]] = []
        self.statistics = {
            'alerts_sent': 0,
            'processes_monitored': 0,
            'uptime_seconds': 0,
            'start_time': time.time()
        }
        
        # Configure logging
        logging.basicConfig(
            level=getattr(logging, self.config['log_level']),
            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
        )
        self.logger = logging.getLogger('page_fault_tracer')
        
    def _load_config(self, config_path: str) -> dict:
        """Load configuration with sensible defaults."""
        default_config = {
            'thresholds': {
                'fault_rate_warning': 500,
                'fault_rate_critical': 1000,
                'rss_growth_percent': 200,
                'monitoring_window_sec': 60,
                'min_process_size_mb': 10
            },
            'sampling': {
                'enabled': False,  # Full tracing for maximum accuracy
                'rate': 1000       # Sample 1 in 1000 if enabled
            },
            'stack_traces': {
                'enabled': True,
                'max_depth': 16
            },
            'alerting': {
                'confidence_threshold': 70,
                'rate_limit_seconds': 300,  # Max 1 alert per process per 5 min
                'webhook_url': None,
                'pagerduty_key': None
            },
            'performance': {
                'max_processes': 8192,
                'ring_buffer_size_mb': 1,
                'cleanup_interval_sec': 300
            },
            'log_level': 'INFO'
        }
        
        if config_path and os.path.exists(config_path):
            with open(config_path, 'r') as f:
                user_config = json.load(f)
                # Deep merge configuration
                default_config.update(user_config)
                
        return default_config
        
    def initialize(self) -> bool:
        """Initialize eBPF program and attach to kernel."""
        try:
            # Load BPF program
            with open('/opt/system-agent/bpf/page_fault_detector.bpf.c', 'r') as f:
                bpf_source = f.read()
                
            self.bpf = BPF(text=bpf_source)
            
            # Configure thresholds
            thresholds = self.bpf['thresholds']
            thresh_config = self.config['thresholds']
            
            thresholds[0] = {
                'fault_rate_warning': thresh_config['fault_rate_warning'],
                'fault_rate_critical': thresh_config['fault_rate_critical'],
                'rss_growth_percent': thresh_config['rss_growth_percent'],
                'monitoring_window_sec': thresh_config['monitoring_window_sec'],
                'min_process_size_mb': thresh_config['min_process_size_mb']
            }
            
            # Attach to page fault tracepoint
            self.bpf.attach_tracepoint(
                tp="exceptions:page_fault_user",
                fn_name="trace_page_fault_user"
            )
            
            self.logger.info("Page fault tracer initialized successfully")
            return True
            
        except Exception as e:
            self.logger.error(f"Failed to initialize eBPF program: {e}")
            return False
            
    def add_alert_callback(self, callback: Callable[[LeakAlert], None]):
        """Add callback function for leak alerts."""
        self.alert_callbacks.append(callback)
        
    def start_monitoring(self):
        """Start continuous monitoring loop."""
        if not self.initialize():
            return False
            
        self.running.set()
        
        # Start alert processing thread
        alert_thread = Thread(target=self._process_alerts, daemon=True)
        alert_thread.start()
        
        # Start statistics thread
        stats_thread = Thread(target=self._update_statistics, daemon=True)
        stats_thread.start()
        
        # Start cleanup thread
        cleanup_thread = Thread(target=self._periodic_cleanup, daemon=True)
        cleanup_thread.start()
        
        self.logger.info("Page fault monitoring started")
        
        try:
            while self.running.is_set():
                time.sleep(1)
        except KeyboardInterrupt:
            self.logger.info("Received interrupt signal")
        finally:
            self.stop_monitoring()
            
        return True
        
    def stop_monitoring(self):
        """Stop monitoring and cleanup resources."""
        self.logger.info("Stopping page fault monitoring...")
        self.running.clear()
        
        if self.bpf:
            self.bpf.cleanup()
            self.bpf = None
            
        self.logger.info("Page fault monitoring stopped")
        
    def _process_alerts(self):
        """Process leak detection alerts from eBPF ring buffer."""
        def handle_alert(cpu, data, size):
            try:
                event = self.bpf['alerts'].event(data)
                
                alert = LeakAlert(
                    pid=event.pid,
                    comm=event.comm.decode('utf-8', 'ignore'),
                    fault_rate=event.fault_rate,
                    rss_mb=event.rss_mb,
                    vsz_mb=event.vsz_mb,
                    confidence=event.confidence,
                    severity=event.severity,
                    timestamp=event.timestamp_ns / 1e9
                )
                
                self.logger.warning(
                    f"Memory leak detected: PID={alert.pid} ({alert.comm}) "
                    f"RSS={alert.rss_mb}MB VSZ={alert.vsz_mb}MB "
                    f"FaultRate={alert.fault_rate}/sec "
                    f"Confidence={alert.confidence}% "
                    f"Severity={alert.severity_label}"
                )
                
                # Trigger callbacks
                for callback in self.alert_callbacks:
                    try:
                        callback(alert)
                    except Exception as e:
                        self.logger.error(f"Alert callback failed: {e}")
                        
                self.statistics['alerts_sent'] += 1
                
                # Trigger Layer 2 analysis
                self._trigger_detailed_analysis(alert)
                
            except Exception as e:
                self.logger.error(f"Error processing alert: {e}")
                
        # Open ring buffer and poll for events
        self.bpf['alerts'].open_ring_buffer(handle_alert)
        
        while self.running.is_set():
            try:
                self.bpf.ring_buffer_poll(timeout=100)
            except Exception as e:
                if self.running.is_set():  # Don't log during shutdown
                    self.logger.error(f"Ring buffer poll error: {e}")
                    
    def _update_statistics(self):
        """Update monitoring statistics periodically."""
        while self.running.is_set():
            try:
                self.statistics['processes_monitored'] = len(self.bpf['process_tracking'])
                self.statistics['uptime_seconds'] = time.time() - self.statistics['start_time']
                
                self.logger.info(
                    f"Statistics: Monitoring {self.statistics['processes_monitored']} processes, "
                    f"{self.statistics['alerts_sent']} alerts sent, "
                    f"uptime {self.statistics['uptime_seconds']:.0f}s"
                )
                
                time.sleep(60)  # Update every minute
                
            except Exception as e:
                self.logger.error(f"Statistics update error: {e}")
                
    def _periodic_cleanup(self):
        """Clean up stale process entries periodically."""
        while self.running.is_set():
            try:
                time.sleep(self.config['performance']['cleanup_interval_sec'])
                
                process_map = self.bpf['process_tracking']
                current_time = time.time() * 1e9  # nanoseconds
                stale_pids = []
                
                # Identify stale processes (no activity for 10 minutes)
                for pid in process_map:
                    metrics = process_map[pid]
                    if current_time - metrics.last_update_ns > 600e9:  # 10 minutes
                        # Verify process doesn't exist
                        if not os.path.exists(f'/proc/{pid}'):
                            stale_pids.append(pid)
                            
                # Remove stale entries
                for pid in stale_pids:
                    del process_map[pid]
                    
                if stale_pids:
                    self.logger.debug(f"Cleaned up {len(stale_pids)} stale process entries")
                    
            except Exception as e:
                self.logger.error(f"Cleanup error: {e}")
                
    def _trigger_detailed_analysis(self, alert: LeakAlert):
        """Trigger Layer 2 detailed analysis for confirmed leaks."""
        try:
            # Detect allocator used by process
            allocator = self._detect_allocator(alert.pid)
            
            self.logger.info(f"Triggering Layer 2 analysis for PID {alert.pid} (allocator: {allocator})")
            
            # Trigger appropriate profiling based on allocator
            if allocator == 'jemalloc':
                self._enable_jemalloc_profiling(alert.pid)
            elif allocator == 'tcmalloc':
                self._enable_tcmalloc_profiling(alert.pid)
            else:
                self.logger.info(f"No specific profiler for {allocator}, using eBPF sampling")
                self._enable_ebpf_sampling(alert.pid)
                
        except Exception as e:
            self.logger.error(f"Failed to trigger detailed analysis for PID {alert.pid}: {e}")
            
    def _detect_allocator(self, pid: int) -> str:
        """Detect which memory allocator a process is using."""
        try:
            with open(f'/proc/{pid}/maps', 'r') as f:
                maps_content = f.read()
                
            if 'libjemalloc' in maps_content:
                return 'jemalloc'
            elif 'libtcmalloc' in maps_content:
                return 'tcmalloc'
            elif 'libmimalloc' in maps_content:
                return 'mimalloc'
            else:
                return 'glibc'
                
        except (OSError, IOError):
            return 'unknown'
            
    def _enable_jemalloc_profiling(self, pid: int):
        """Enable jemalloc heap profiling for detailed analysis."""
        self.logger.info(f"Would enable jemalloc profiling for PID {pid}")
        # Implementation would require CAP_SYS_PTRACE capability
        # gdb -p {pid} -batch -ex "call mallctl(\"prof.active\", ...)"
        
    def _enable_tcmalloc_profiling(self, pid: int):
        """Enable tcmalloc heap profiling for detailed analysis.""" 
        self.logger.info(f"Would enable tcmalloc profiling for PID {pid}")
        # Implementation: send SIGUSR1 if supported by application
        
    def _enable_ebpf_sampling(self, pid: int):
        """Enable eBPF-based allocation sampling as fallback."""
        self.logger.info(f"Enabling eBPF sampling for PID {pid}")
        # Implementation: Load additional BPF program for detailed malloc/free tracing
        
    def get_statistics(self) -> dict:
        """Get current monitoring statistics."""
        return self.statistics.copy()
        
    def get_monitored_processes(self) -> List[dict]:
        """Get list of currently monitored processes with metrics."""
        processes = []
        
        try:
            process_map = self.bpf['process_tracking']
            for pid in process_map:
                metrics = process_map[pid]
                processes.append({
                    'pid': int(pid),
                    'rss_mb': (metrics.rss_pages * 4096) // (1024 * 1024),
                    'vsz_mb': (metrics.vsz_pages * 4096) // (1024 * 1024),
                    'fault_rate': metrics.fault_rate_per_sec,
                    'total_faults': metrics.total_faults,
                    'confidence': metrics.leak_confidence,
                    'alert_triggered': bool(metrics.alert_triggered)
                })
                
        except Exception as e:
            self.logger.error(f"Error getting process list: {e}")
            
        return processes

def main():
    """Main entry point for standalone operation."""
    config_path = sys.argv[1] if len(sys.argv) > 1 else '/etc/system-agent/page-fault-tracer.json'
    
    tracer = PageFaultTracer(config_path)
    
    # Add webhook alert callback if configured
    webhook_url = tracer.config.get('alerting', {}).get('webhook_url')
    if webhook_url:
        def webhook_callback(alert: LeakAlert):
            payload = {
                'type': 'memory_leak_detected',
                'pid': alert.pid,
                'process': alert.comm,
                'severity': alert.severity_label,
                'confidence': alert.confidence,
                'metrics': {
                    'rss_mb': alert.rss_mb,
                    'vsz_mb': alert.vsz_mb,
                    'fault_rate': alert.fault_rate
                },
                'timestamp': alert.timestamp
            }
            
            try:
                import requests
                requests.post(webhook_url, json=payload, timeout=5)
            except Exception as e:
                tracer.logger.error(f"Webhook delivery failed: {e}")
                
        tracer.add_alert_callback(webhook_callback)
    
    # Start monitoring
    tracer.start_monitoring()

if __name__ == '__main__':
    main()

#!/usr/bin/env python3
"""
Advanced page fault tracer using BCC for production monitoring.
Includes stack trace collection and leak pattern analysis.
"""

from bcc import BPF
import time
import json
import argparse
from collections import defaultdict

# BPF program source
bpf_program = """
#include <uapi/linux/ptrace.h>
#include <linux/mm.h>
#include <linux/sched.h>

struct fault_event {
    u32 pid;
    u32 tid;
    u64 timestamp;
    u64 fault_address;
    u32 fault_flags;
    char comm[TASK_COMM_LEN];
};

struct process_stats {
    u64 total_faults;
    u64 anon_faults;
    u64 file_faults;
    u64 last_fault_time;
    u64 rss_pages;
};

BPF_HASH(stats, u32, struct process_stats);
BPF_PERF_OUTPUT(events);

// Stack trace map for leak analysis
BPF_STACK_TRACE(stack_traces, 1024);

TRACEPOINT_PROBE(exceptions, page_fault_user) {
    u32 pid = bpf_get_current_pid_tgid() >> 32;
    u32 tid = bpf_get_current_pid_tgid() & 0xFFFFFFFF;
    
    if (pid == 0) return 0;  // Skip kernel threads
    
    struct fault_event event = {};
    event.pid = pid;
    event.tid = tid;
    event.timestamp = bpf_ktime_get_ns();
    event.fault_address = args->address;
    event.fault_flags = args->error_code;
    bpf_get_current_comm(&event.comm, sizeof(event.comm));
    
    // Update process statistics
    struct process_stats *pstats = stats.lookup(&pid);
    if (!pstats) {
        struct process_stats new_stats = {};
        new_stats.total_faults = 1;
        new_stats.last_fault_time = event.timestamp;
        
        // Classify fault type
        if (event.fault_address > 0x400000000000ULL) {
            new_stats.anon_faults = 1;
        } else {
            new_stats.file_faults = 1;
        }
        
        stats.update(&pid, &new_stats);
    } else {
        pstats->total_faults++;
        pstats->last_fault_time = event.timestamp;
        
        // Update fault type counters
        if (event.fault_address > 0x400000000000ULL) {
            pstats->anon_faults++;
        } else {
            pstats->file_faults++;
        }
    }
    
    // Send event for high-rate processes
    if (pstats && pstats->total_faults % 500 == 0) {
        events.perf_submit(args, &event, sizeof(event));
    }
    
    return 0;
}
"""

class PageFaultAnalyzer:
    def __init__(self, threshold_rate=100):
        self.threshold_rate = threshold_rate
        self.bpf = BPF(text=bpf_program)
        self.process_data = defaultdict(list)
        self.leak_suspects = set()
        
    def handle_event(self, cpu, data, size):
        """Handle page fault events from BPF."""
        event = self.bpf["events"].event(data)
        
        # Decode event data
        comm = event.comm.decode('utf-8', 'ignore')
        timestamp = event.timestamp / 1e9  # Convert to seconds
        
        # Store event data for analysis
        self.process_data[event.pid].append({
            'timestamp': timestamp,
            'address': event.fault_address,
            'flags': event.fault_flags,
            'comm': comm
        })
        
        # Analyze for leak patterns
        self.analyze_process(event.pid)
        
    def analyze_process(self, pid):
        """Analyze page fault patterns for leak detection."""
        events = self.process_data[pid]
        if len(events) < 50:  # Need sufficient data
            return
            
        # Calculate fault rate over last 60 seconds
        current_time = time.time()
        recent_events = [e for e in events 
                        if current_time - e['timestamp'] < 60]
        
        fault_rate = len(recent_events) / 60.0
        
        # Detect sustained high fault rate
        if fault_rate > self.threshold_rate and pid not in self.leak_suspects:
            self.leak_suspects.add(pid)
            
            # Get additional process information
            try:
                with open(f'/proc/{pid}/status') as f:
                    status = f.read()
                    rss_match = [line for line in status.split('\n') 
                                if line.startswith('VmRSS:')]
                    rss_mb = 0
                    if rss_match:
                        rss_mb = int(rss_match[0].split()[1]) // 1024
                        
                print(f"LEAK SUSPECT: PID {pid} ({events[-1]['comm']}) "
                      f"- {fault_rate:.1f} faults/sec, RSS: {rss_mb}MB")
                      
            except (IOError, OSError, ValueError):
                print(f"LEAK SUSPECT: PID {pid} - {fault_rate:.1f} faults/sec")
                
    def run_analysis(self, duration=None):
        """Run continuous page fault analysis."""
        print("Starting page fault analysis for memory leak detection...")
        print(f"Threshold: {self.threshold_rate} faults/sec")
        
        # Open perf buffer
        self.bpf["events"].open_perf_buffer(self.handle_event, page_cnt=64)
        
        start_time = time.time()
        try:
            while True:
                self.bpf.perf_buffer_poll(timeout=100)
                
                # Print periodic statistics
                if int(time.time()) % 30 == 0:  # Every 30 seconds
                    self.print_statistics()
                    
                # Check duration limit
                if duration and (time.time() - start_time) > duration:
                    break
                    
        except KeyboardInterrupt:
            print("\nStopping analysis...")
            
        self.print_final_report()
        
    def print_statistics(self):
        """Print current monitoring statistics."""
        stats_map = self.bpf["stats"]
        active_processes = len(stats_map)
        
        print(f"\n--- Statistics (Monitoring {active_processes} processes) ---")
        
        # Get top fault generators
        top_processes = []
        for pid, stats in stats_map.items():
            total_faults = stats.total_faults
            if total_faults > 100:  # Minimum activity threshold
                top_processes.append((pid.value, total_faults))
                
        # Sort by fault count
        top_processes.sort(key=lambda x: x[1], reverse=True)
        
        print("Top fault generators:")
        for pid, fault_count in top_processes[:5]:
            try:
                comm = self.process_data[pid][-1]['comm'] if self.process_data[pid] else 'unknown'
                print(f"  PID {pid}: {fault_count} faults ({comm})")
            except (IndexError, KeyError):
                print(f"  PID {pid}: {fault_count} faults")
                
        if self.leak_suspects:
            print(f"Leak suspects: {len(self.leak_suspects)} processes")
            
    def print_final_report(self):
        """Print final analysis report."""
        print("\n=== Final Page Fault Analysis Report ===")
        print(f"Total processes monitored: {len(self.process_data)}")
        print(f"Leak suspects identified: {len(self.leak_suspects)}")
        
        if self.leak_suspects:
            print("\nDetailed leak suspect analysis:")
            for pid in self.leak_suspects:
                events = self.process_data[pid]
                if events:
                    comm = events[-1]['comm']
                    total_events = len(events)
                    duration = events[-1]['timestamp'] - events[0]['timestamp']
                    avg_rate = total_events / duration if duration > 0 else 0
                    
                    print(f"  PID {pid} ({comm}):")
                    print(f"    Total faults: {total_events}")
                    print(f"    Average rate: {avg_rate:.1f} faults/sec")
                    print(f"    Monitoring duration: {duration:.1f} seconds")

def main():
    parser = argparse.ArgumentParser(description='Page fault tracer for memory leak detection')
    parser.add_argument('--threshold', type=int, default=100, 
                       help='Fault rate threshold (faults/sec)')
    parser.add_argument('--duration', type=int, default=None,
                       help='Monitoring duration in seconds')
    
    args = parser.parse_args()
    
    analyzer = PageFaultAnalyzer(threshold_rate=args.threshold)
    analyzer.run_analysis(duration=args.duration)

if __name__ == '__main__':
    main()

def detect_rss_growth_anomaly(process_metrics, baseline_stats):
    current_growth_rate = process_metrics.calculate_rss_growth_rate()
    baseline_mean = baseline_stats.growth_rate_mean
    baseline_stddev = baseline_stats.growth_rate_stddev
    
    warning_threshold = baseline_mean + (2 * baseline_stddev)
    critical_threshold = 10 * 1024 * 1024  # 10MB/minute
    
    if current_growth_rate > critical_threshold:
        return AlertLevel.CRITICAL
    elif current_growth_rate > warning_threshold:
        return AlertLevel.WARNING
    
    return AlertLevel.NORMAL

def is_startup_phase(pid, fault_rate):
    """Detect if process is in startup phase."""
    try:
        with open(f'/proc/{pid}/stat') as f:
            stat_data = f.read().split()
            starttime = int(stat_data[21])  # Process start time
            
        boot_time = get_boot_time()  # From /proc/uptime
        process_age_sec = time.time() - (boot_time + starttime / 100.0)
        
        # Consider startup phase if process < 10 minutes old
        return process_age_sec < 600
        
    except (IOError, OSError, ValueError):
        return True  # Assume startup if can't determine

class LeakVsGrowthClassifier:
    """Distinguish memory leaks from legitimate growth."""
    
    def analyze_growth_pattern(self, metrics_history):
        """Analyze memory growth characteristics."""
        
        # Calculate growth smoothness (leaks = smoother)
        growth_variance = np.var([m.rss_delta for m in metrics_history])
        
        # Check for periodic releases (legitimate growth)
        release_periods = self.detect_release_cycles(metrics_history)
        
        # Analyze fault-to-RSS correlation
        fault_rss_correlation = np.corrcoef(
            [m.fault_rate for m in metrics_history],
            [m.rss_mb for m in metrics_history]
        )[0, 1]
        
        # Scoring
        leak_score = 0
        if growth_variance < 0.1:  # Smooth growth = likely leak
            leak_score += 30
        if len(release_periods) == 0:  # No releases = likely leak
            leak_score += 40
        if fault_rss_correlation > 0.8:  # Strong correlation
            leak_score += 30
            
        return leak_score, self.generate_explanation(
            growth_variance, release_periods, fault_rss_correlation
        )

# Configure larger ring buffers for high-throughput
ring_buffer_config = {
    'size_mb': 4,  # Increase from 1MB default
    'cpu_count': os.cpu_count(),
    'poll_timeout_ms': 50  # Reduce polling frequency
}

def enhanced_leak_detection(process_metrics):
    """Multi-signal leak detection to reduce false negatives."""
    
    signals = []
    
    # Signal 1: Page fault rate trends
    fault_trend = calculate_trend(process_metrics.fault_history)
    if fault_trend > 0.1:  # Increasing fault rate
        signals.append(('fault_trend', 25))
    
    # Signal 2: Virtual memory fragmentation
    if process_metrics.vsz_pages > process_metrics.rss_pages * 2:
        fragmentation_score = min(30, 
            (process_metrics.vsz_pages / process_metrics.rss_pages - 1) * 10)
        signals.append(('fragmentation', fragmentation_score))
    
    # Signal 3: Anonymous memory dominance
    if process_metrics.anon_fault_ratio > 0.75:
        signals.append(('anonymous_dominance', 20))
    
    # Signal 4: Memory growth acceleration
    growth_acceleration = calculate_growth_acceleration(process_metrics.rss_history)
    if growth_acceleration > 0:
        signals.append(('growth_acceleration', min(25, growth_acceleration * 100)))
    
    # Combine signals
    total_confidence = sum(score for _, score in signals)
    return min(100, total_confidence), signals