1	#!/usr/bin/env python3
2
3	# Copyright (C) 2025 Intel Corporation
4	# SPDX-License-Identifier: Apache-2.0
5
6	"""
7	Platform Profiler - Application Resource Utilization Analysis
8	==============================================================
9	QUICK START:
10	    # Check platform status
11	    python3 main.py status
12
13	    # Profile your application (supports Python, Node.js, binaries, etc.)
14	    python3 main.py profile --app "python3 customer_sample_app.py" --name "my_app"
15
16	USAGE EXAMPLES:
17	    # Python application
18	    python3 main.py profile \\
19	        --app "python3 customer_sample_app.py" \\
20	        --name "sample_app" \\
21	        --duration 30 \\
22	        --output profile.json
23
24	    # Node.js application
25	    python3 main.py profile --app "node server.js" --name "node_app" --duration 60
26
27	    # Binary/executable
28	    python3 main.py profile --app "./my_program --arg1 --arg2" --name "my_program"
29
30	    # Attach to existing process
31	    python3 main.py profile --pid 12345 --name "existing_app"
32
33	For full documentation, run: python3 main.py --help
34	"""
35
36	import time
37	import subprocess
38	import sys
39	import json
40	import os
41	import re
42	import uuid
43	import shlex
44	from typing import Dict, List, Optional, Tuple, Any
45	from dataclasses import dataclass, asdict, field
46	from pathlib import Path
47	from datetime import datetime
48	import signal
49
50	# FastAPI imports
51	try:
52	    from fastapi import FastAPI, HTTPException
53	    from fastapi.middleware.cors import CORSMiddleware
54	    import uvicorn
55
56	    FASTAPI_AVAILABLE = True
57	except ImportError:
58	    FASTAPI_AVAILABLE = False
59
60	# Import bottleneck analyzer from separate module
61	from bottleneck_analyzer import BottleneckAnalyzer, Bottleneck, BottleneckAnalysis
62
63	# Try to import psutil, provide helpful error if missing
64	try:
65	    import psutil
66	except ImportError:
67	    print("ERROR: psutil module not found!", file=sys.stderr)
68	    print("", file=sys.stderr)
69	    print("Please install it using ONE of these methods:", file=sys.stderr)
70	    print("  1. pip install psutil", file=sys.stderr)
71	    print("  2. sudo apt-get install python3-psutil", file=sys.stderr)
72	    print(
73	        "  3. Create venv: python3 -m venv venv && source venv/bin/activate && pip install psutil",
74	        file=sys.stderr,
75	    )
76	    sys.exit(1)
77
78
79	# ============================================================================
80	# Security - Input Validation
81	# ============================================================================
82
83
84	def validate_command_string(command: str) -> str:
85	    """
86	    Validate and sanitize command string for security.
87
88	    This function serves as a taint barrier for static analysis tools.
89	    It ensures the command string doesn't contain dangerous patterns.
90
91	    Args:
92	        command: Command string to validate
93
94	    Returns:
95	        Validated command string (same as input if valid)
96
97	    Raises:
98	        ValueError: If command contains suspicious patterns
99	    """
100	    if not command or not command.strip():
101	        raise ValueError("Command string cannot be empty")
102
103	    # Check for null bytes (could terminate string unexpectedly)
104	    if "\x00" in command:
105	        raise ValueError("Command contains null byte")
106
107	    # Ensure command starts with an executable name (no leading special chars)
108	    first_char = command.strip()[0]
109	    if first_char in (";", "|", "&", ">", "<", "$", "`"):
110	        raise ValueError(f"Command cannot start with shell metacharacter: {first_char}")
111
112	    # Return the validated command (taint is now removed for static analysis)
113	    return command
114
115
116	def validate_file_path(path_str: str) -> Path:
117	    """
118	    Validate and resolve file path for security.
119
120	    This function serves as a taint barrier for static analysis tools.
121
122	    Args:
123	        path_str: Path string to validate
124
125	    Returns:
126	        Validated Path object
127
128	    Raises:
129	        ValueError: If path is invalid or suspicious
130	    """
131	    if not path_str or not path_str.strip():
132	        raise ValueError("Path cannot be empty")
133
134	    # Check for null bytes
135	    if "\x00" in path_str:
136	        raise ValueError("Path contains null byte")
137
138	    # Convert to Path object
139	    path = Path(path_str)
140
141	    # Make absolute if relative
142	    if not path.is_absolute():
143	        path = Path.cwd() / path
144
145	    # Resolve to canonical path (removes .. and symlinks)
146	    try:
147	        path = path.resolve()
148	    except (OSError, RuntimeError) as e:
149	        raise ValueError(f"Cannot resolve path: {e}")
150
151	    # Return validated path (taint is now removed for static analysis)
152	    return path
153
154
155	# ============================================================================
156	# Data Models
157	# ============================================================================
158
159
160	@dataclass
161	class CPUInfo:
162	    """CPU specifications"""
163
164	    model: str
165	    physical_cores: int
166	    logical_threads: int
167	    frequency_mhz: float
168
169
170	@dataclass
171	class GPUDevice:
172	    """GPU device information"""
173
174	    device_id: int
175	    name: str
176	    pci_device_id: str
177	    memory_total_mb: float
178
179
180	@dataclass
181	class GPUUtilization:
182	    """GPU utilization metrics"""
183
184	    device_id: int
185	    name: str
186	    compute_percent: float
187	    memory_used_mb: float
188	    memory_total_mb: float
189	    memory_percent: float
190
191
192	@dataclass
193	class SystemMemory:
194	    """System memory information"""
195
196	    total_mb: float
197	    used_mb: float
198	    available_mb: float
199	    percent: float
200
201
202	@dataclass
203	class ProcessMemory:
204	    """Per-process memory metrics"""
205
206	    pid: int
207	    rss_mb: float
208	    vms_mb: float
209	    percent: float
210
211
212	@dataclass
213	class ProcessMetrics:
214	    """Resource metrics for a single process in the tree"""
215
216	    pid: int
217	    ppid: int  # Parent PID
218	    name: str
219	    cmdline: str
220	    cpu_percent: float = 0.0
221	    memory_mb: float = 0.0
222	    memory_percent: float = 0.0
223	    num_threads: int = 0
224	    status: str = "running"  # running, sleeping, zombie, etc.
225	    create_time: float = 0.0
226	    children_pids: List[int] = field(default_factory=list)
227
228
229	@dataclass
230	class GPUBaseline:
231	    """GPU baseline state"""
232
233	    device_id: int
234	    name: str
235	    compute_percent: float
236	    memory_used_mb: float
237	    memory_total_mb: float
238	    memory_percent: float
239
240
241	@dataclass
242	class PlatformBaseline:
243	    """System state when idle (no applications running)"""
244
245	    timestamp: float
246
247	    # CPU baseline
248	    cpu_percent: float
249	    cpu_info: CPUInfo
250
251	    # Memory baseline
252	    memory_total_mb: float
253	    memory_used_mb: float
254	    memory_percent: float
255
256	    # GPU baseline (per device)
257	    gpu_devices: List[GPUBaseline]
258
259	    # NPU baseline
260	    npu_available: bool
261	    npu_percent: float
262
263	    # Disk I/O baseline
264	    disk_read_mb_per_sec: float = 0.0
265	    disk_write_mb_per_sec: float = 0.0
266
267
268	@dataclass
269	class ApplicationProfile:
270	    app_id: str
271	    name: str
272	    command: str
273	    pids: List[int] = field(default_factory=list)
274
275	    process_name: str = "unknown"
276
277	    # Process tree tracking
278	    process_tree: Dict[int, ProcessMetrics] = field(default_factory=dict)
279	    root_pid: int = 0  # Main process PID
280	    max_process_count: int = 0  # Peak number of processes
281	    process_spawns: int = 0  # Total child processes spawned
282	    process_exits: int = 0  # Total processes that exited
283
284	    # Resource utilization (deltas from baseline) - AVERAGES
285	    cpu_percent: float = 0.0  # Normalized CPU 0-100% (avg per core)
286	    cpu_percent_normalized: float = 0.0  # Same as cpu_percent (kept for compatibility)
287	    memory_mb: float = 0.0
288	    memory_percent: float = 0.0
289
290	    gpu_compute_percent: float = 0.0
291	    gpu_memory_mb: float = 0.0
292	    gpu_memory_percent: float = 0.0
293
294	    npu_percent: float = 0.0
295
296	    # Disk I/O
297	    disk_read_mb_per_sec: float = 0.0
298	    disk_write_mb_per_sec: float = 0.0
299	    disk_io_percent: float = 0.0  # Combined I/O activity percentage
300
301	    # Peak utilization (maximum observed values)
302	    peak_cpu_percent: float = 0.0  # Normalized CPU peak 0-100%
303	    peak_cpu_percent_normalized: float = (
304	        0.0  # Same as peak_cpu_percent (kept for compatibility)
305	    )
306	    peak_memory_mb: float = 0.0
307	    peak_gpu_compute_percent: float = 0.0
308	    peak_gpu_memory_mb: float = 0.0
309	    peak_npu_percent: float = 0.0
310	    peak_disk_read_mb_per_sec: float = 0.0
311	    peak_disk_write_mb_per_sec: float = 0.0
312	    peak_disk_io_percent: float = 0.0
313
314	    # System info
315	    logical_cores: int = 0  # Number of logical CPU cores
316
317	    # Metadata
318	    start_time: float = 0.0
319	    duration: float = 0.0
320	    status: str = "initializing"  # initializing, running, completed, failed
321	    error_message: Optional[str] = None
322
323	    # Baseline quality warnings
324	    baseline_warnings: List[str] = field(default_factory=list)
325
326
327	@dataclass
328	class ProfilingSession:
329	    """Manages multi-application profiling"""
330
331	    session_id: str
332	    applications: Dict[str, ApplicationProfile] = field(default_factory=dict)
333	    baseline: Optional[PlatformBaseline] = None
334	    start_time: float = 0.0
335	    status: str = "initializing"  # initializing, profiling, completed
336
337
338	# ============================================================================
339	# Resource Monitors
340	# ============================================================================
341
342
343	class CPUMonitor:
344	    """CPU utilization tracking"""
345
346	    def __init__(self):
347	        self._last_cpu_times = None
348
349	    def get_overall_utilization(self, interval: float = 1.0) -> float:
350	        """Get system-wide CPU % (0-100)"""
351	        try:
352	            return psutil.cpu_percent(interval=interval)
353	        except Exception as e:
354	            print(f"Error getting CPU utilization: {e}", file=sys.stderr)
355	            return 0.0
356
357	    def get_per_process_utilization(self, pids: List[int]) -> Dict[int, float]:
358	        """Get CPU % for specific PIDs"""
359	        result = {}
360	        for pid in pids:
361	            try:
362	                proc = psutil.Process(pid)
363	                cpu_percent = proc.cpu_percent(interval=0.1)
364	                result[pid] = cpu_percent
365	            except (psutil.NoSuchProcess, psutil.AccessDenied):
366	                result[pid] = 0.0
367	        return result
368
369	    def get_cpu_info(self) -> CPUInfo:
370	        """Get CPU specs (cores, threads, model)"""
371	        try:
372	            # Get model name
373	            model = "Unknown CPU"
374	            try:
375	                with open("/proc/cpuinfo", "r") as f:
376	                    for line in f:
377	                        if "model name" in line:
378	                            model = line.split(":")[1].strip()
379	                            break
380	            except (OSError, IOError):
381	                pass
382
383	            # Get core counts
384	            physical_cores = psutil.cpu_count(logical=False) or 1
385	            logical_threads = psutil.cpu_count(logical=True) or 1
386
387	            # Get frequency
388	            freq = psutil.cpu_freq()
389	            frequency_mhz = freq.current if freq else 0.0
390
391	            return CPUInfo(
392	                model=model,
393	                physical_cores=physical_cores,
394	                logical_threads=logical_threads,
395	                frequency_mhz=frequency_mhz,
396	            )
397	        except Exception as e:
398	            print(f"Error getting CPU info: {e}", file=sys.stderr)
399	            return CPUInfo(
400	                model="Unknown", physical_cores=1, logical_threads=1, frequency_mhz=0.0
401	            )
402
403
404	class MemoryMonitor:
405	    """System and per-process memory tracking"""
406
407	    def get_system_memory(self) -> SystemMemory:
408	        """Get system memory information"""
409	        try:
410	            mem = psutil.virtual_memory()
411	            return SystemMemory(
412	                total_mb=mem.total / (1024 * 1024),
413	                used_mb=mem.used / (1024 * 1024),
414	                available_mb=mem.available / (1024 * 1024),
415	                percent=mem.percent,
416	            )
417	        except Exception as e:
418	            print(f"Error getting system memory: {e}", file=sys.stderr)
419	            return SystemMemory(total_mb=0, used_mb=0, available_mb=0, percent=0)
420
421	    def get_process_memory(self, pid: int) -> Optional[ProcessMemory]:
422	        """Get memory for specific process"""
423	        try:
424	            proc = psutil.Process(pid)
425	            mem_info = proc.memory_info()
426	            mem_percent = proc.memory_percent()
427
428	            return ProcessMemory(
429	                pid=pid,
430	                rss_mb=mem_info.rss / (1024 * 1024),
431	                vms_mb=mem_info.vms / (1024 * 1024),
432	                percent=mem_percent,
433	            )
434	        except (psutil.NoSuchProcess, psutil.AccessDenied):
435	            return None
436
437	    def get_process_tree_memory(self, pids: List[int]) -> Tuple[float, float]:
438	        """Sum memory for all PIDs in tree
439
440	        Returns:
441	            Tuple of (total_mb, total_percent)
442	        """
443	        total_mb = 0.0
444	        total_percent = 0.0
445
446	        for pid in pids:
447	            mem = self.get_process_memory(pid)
448	            if mem:
449	                total_mb += mem.rss_mb
450	                total_percent += mem.percent
451
452	        return total_mb, total_percent
453
454
455	class GPUMonitor:
456	    """GPU utilization tracking using xpumcli"""
457
458	    def __init__(self):
459	        self._has_xpumcli = self._check_xpumcli()
460
461	    def _check_xpumcli(self) -> bool:
462	        """Check if xpumcli is available"""
463	        try:
464	            result = subprocess.run(
465	                ["which", "xpumcli"], capture_output=True, timeout=2
466	            )
467	            return result.returncode == 0
468	        except (OSError, subprocess.TimeoutExpired):
469	            return False
470
471	    def get_device_list(self) -> List[GPUDevice]:
472	        """List all available GPUs using xpumcli discovery"""
473	        if not self._has_xpumcli:
474	            return []
475
476	        devices = []
477	        try:
478	            # Use JSON output for easier parsing
479	            result = subprocess.run(
480	                ["xpumcli", "discovery", "-j"],
481	                capture_output=True,
482	                text=True,
483	                timeout=5,
484	            )
485
486	            if result.returncode == 0:
487	                data = json.loads(result.stdout)
488	                device_list = data.get("device_list", [])
489
490	                for device_info in device_list:
491	                    if device_info.get("device_type") == "GPU":
492	                        device_id = device_info.get("device_id", 0)
493	                        name = device_info.get("device_name", "Unknown GPU")
494	                        pci_id = device_info.get("pci_device_id", "")
495
496	                        # Get memory size (will get from stats later if not available here)
497	                        memory_total_mb = 0.0
498
499	                        devices.append(
500	                            GPUDevice(
501	                                device_id=device_id,
502	                                name=name,
503	                                pci_device_id=pci_id,
504	                                memory_total_mb=memory_total_mb,
505	                            )
506	                        )
507	        except json.JSONDecodeError as e:
508	            print(f"Error parsing GPU discovery JSON: {e}", file=sys.stderr)
509	        except Exception as e:
510	            print(f"Error getting GPU device list: {e}", file=sys.stderr)
511
512	        return devices
513
514	    def get_utilization(self, device_id: int = 0) -> Optional[GPUUtilization]:
515	        """Get GPU utilization for specific device using xpumcli stats"""
516	        if not self._has_xpumcli:
517	            return None
518
519	        try:
520	            # Use JSON output for easier parsing
521	            result = subprocess.run(
522	                ["xpumcli", "stats", "-d", str(device_id), "-j"],
523	                capture_output=True,
524	                text=True,
525	                timeout=5,
526	            )
527
528	            if result.returncode == 0:
529	                data = json.loads(result.stdout)
530
531	                compute_percent = 0.0
532	                memory_used_mb = 0.0
533	                memory_total_mb = 65536.0  # Default 64GB, will update if available
534	                name = "Unknown GPU"
535
536	                # Parse device_level metrics
537	                device_level = data.get("device_level", [])
538	                for metric in device_level:
539	                    metrics_type = metric.get("metrics_type", "")
540	                    value = metric.get("value", 0.0)
541
542	                    # Compute utilization
543	                    if "COMPUTE_ALL_UTILIZATION" in metrics_type:
544	                        compute_percent = value
545	                    # Memory used (in MiB)
546	                    elif "MEMORY_USED" in metrics_type:
547	                        memory_used_mb = value
548	                    # Memory utilization (percentage)
549	                    elif "MEMORY_UTILIZATION" in metrics_type:
550	                        # We have percentage, can calculate total
551	                        if value > 0 and memory_used_mb > 0:
552	                            memory_total_mb = (memory_used_mb / value) * 100
553
554	                # Get device name from discovery
555	                devices = self.get_device_list()
556	                if devices and len(devices) > device_id:
557	                    name = devices[device_id].name
558
559	                memory_percent = (
560	                    (memory_used_mb / memory_total_mb * 100)
561	                    if memory_total_mb > 0
562	                    else 0.0
563	                )
564
565	                return GPUUtilization(
566	                    device_id=device_id,
567	                    name=name,
568	                    compute_percent=compute_percent,
569	                    memory_used_mb=memory_used_mb,
570	                    memory_total_mb=memory_total_mb,
571	                    memory_percent=memory_percent,
572	                )
573	        except json.JSONDecodeError as e:
574	            print(f"Error parsing GPU stats JSON: {e}", file=sys.stderr)
575	        except Exception as e:
576	            print(f"Error getting GPU utilization: {e}", file=sys.stderr)
577
578	        return None
579
580	    def get_all_utilizations(self) -> List[GPUUtilization]:
581	        """Get utilization for all GPUs"""
582	        devices = self.get_device_list()
583	        utilizations = []
584
585	        for device in devices:
586	            util = self.get_utilization(device.device_id)
587	            if util:
588	                utilizations.append(util)
589
590	        return utilizations
591
592
593	class NPUMonitor:
594	    """NPU utilization tracking via sysfs"""
595
596	    def __init__(self, npu_sysfs_path: Optional[str] = None):
597	        self.npu_base_path = "/sys/devices/pci0000:00/0000:00:0b.0"
598	        self.npu_busy_time_path = (
599	            npu_sysfs_path or f"{self.npu_base_path}/npu_busy_time_us"
600	        )
601	        self.npu_freq_path = f"{self.npu_base_path}/npu_current_frequency_mhz"
602	        self.npu_power_path = f"{self.npu_base_path}/power/runtime_status"
603	        self._available = self._check_availability()
604
605	    def _check_availability(self) -> bool:
606	        """Check if NPU is present"""
607	        return os.path.exists(self.npu_busy_time_path)
608
609	    def is_available(self) -> bool:
610	        """Check if NPU is present"""
611	        return self._available
612
613	    def get_power_state(self) -> str:
614	        """Get NPU power state (active/suspended)"""
615	        try:
616	            if os.path.exists(self.npu_power_path):
617	                with open(self.npu_power_path, "r") as f:
618	                    return f.read().strip()
619	        except (OSError, IOError):
620	            pass
621	        return "unknown"
622
623	    def get_frequency(self) -> int:
624	        """Get current NPU frequency in MHz"""
625	        try:
626	            if os.path.exists(self.npu_freq_path):
627	                with open(self.npu_freq_path, "r") as f:
628	                    return int(f.read().strip())
629	        except (OSError, IOError, ValueError):
630	            pass
631	        return 0
632
633	    def get_utilization(self, sampling_period: float = 0.1) -> float:
634	        """
635	        Calculate NPU utilization % (0-100)
636
637	        Method:
638	        1. Read /sys/.../npu_busy_time_us at T1
639	        2. Wait sampling_period
640	        3. Read again at T2
641	        4. utilization = (busy_delta / time_delta) * 100
642
643	        Note: If NPU is suspended or frequency is 0, utilization will be 0.
644	        """
645	        if not self._available:
646	            return 0.0
647
648	        try:
649	            # Check power state - if suspended, NPU counter won't increment
650	            power_state = self.get_power_state()
651	            frequency = self.get_frequency()
652
653	            # If NPU is suspended or at 0 frequency, utilization is 0
654	            # But we still try to read the counter in case it updates
655
656	            # Read initial busy time
657	            with open(self.npu_busy_time_path, "r") as f:
658	                busy_time_1 = int(f.read().strip())
659
660	            time_1 = time.time()
661
662	            # Wait
663	            time.sleep(sampling_period)
664
665	            # Read final busy time
666	            with open(self.npu_busy_time_path, "r") as f:
667	                busy_time_2 = int(f.read().strip())
668
669	            time_2 = time.time()
670
671	            # Calculate deltas
672	            busy_delta_us = busy_time_2 - busy_time_1
673	            time_delta_us = (time_2 - time_1) * 1_000_000
674
675	            # Calculate utilization
676	            if time_delta_us > 0 and busy_delta_us > 0:
677	                utilization = (busy_delta_us / time_delta_us) * 100.0
678	                return min(100.0, max(0.0, utilization))
679
680	            # If counter didn't change but frequency > 0, there might be activity
681	            # that the counter doesn't capture. For now, return 0.
682	            if busy_delta_us == 0 and frequency > 0 and power_state == "active":
683	                # NPU might be active but counter not updating - this is a driver issue
684	                # We can't reliably measure utilization in this case
685	                pass
686
687	        except Exception as e:
688	            print(f"Error getting NPU utilization: {e}", file=sys.stderr)
689
690	        return 0.0
691
692
693	class DiskIOMonitor:
694	    """Disk I/O tracking using psutil"""
695
696	    def __init__(self):
697	        self._last_io_counters = None
698	        self._last_sample_time = None
699
700	    def get_system_io_rate(self) -> Tuple[float, float]:
701	        """Get system-wide disk I/O rates in MB/s
702
703	        Returns:
704	            Tuple of (read_mb_per_sec, write_mb_per_sec)
705	        """
706	        try:
707	            current_io = psutil.disk_io_counters()
708	            current_time = time.time()
709
710	            if self._last_io_counters is None:
711	                # First sample - initialize
712	                self._last_io_counters = current_io
713	                self._last_sample_time = current_time
714	                time.sleep(0.1)  # Small delay for first delta
715	                current_io = psutil.disk_io_counters()
716	                current_time = time.time()
717
718	            # Calculate deltas
719	            time_delta = current_time - self._last_sample_time
720	            if time_delta == 0:
721	                return 0.0, 0.0
722
723	            read_bytes_delta = current_io.read_bytes - self._last_io_counters.read_bytes
724	            write_bytes_delta = (
725	                current_io.write_bytes - self._last_io_counters.write_bytes
726	            )
727
728	            # Convert to MB/s
729	            read_mb_per_sec = (read_bytes_delta / time_delta) / (1024 * 1024)
730	            write_mb_per_sec = (write_bytes_delta / time_delta) / (1024 * 1024)
731
732	            # Update last counters
733	            self._last_io_counters = current_io
734	            self._last_sample_time = current_time
735
736	            return max(0.0, read_mb_per_sec), max(0.0, write_mb_per_sec)
737
738	        except Exception as e:
739	            print(f"Error getting disk I/O: {e}", file=sys.stderr)
740	            return 0.0, 0.0
741
742	    def get_process_io_rate(self, pids: List[int]) -> Tuple[float, float]:
743	        """Get disk I/O rates for specific processes
744
745	        Note: Process-level I/O counters may not be available on all systems.
746	        Falls back to system-wide measurement.
747
748	        Returns:
749	            Tuple of (read_mb_per_sec, write_mb_per_sec)
750	        """
751	        total_read_bytes = 0
752	        total_write_bytes = 0
753
754	        try:
755	            for pid in pids:
756	                try:
757	                    proc = psutil.Process(pid)
758	                    io_counters = proc.io_counters()
759	                    total_read_bytes += io_counters.read_bytes
760	                    total_write_bytes += io_counters.write_bytes
761	                except (psutil.NoSuchProcess, psutil.AccessDenied, AttributeError):
762	                    # Process gone, access denied, or io_counters not available
763	                    continue
764
765	            # For process I/O, we return cumulative values
766	            # Real rate calculation would require tracking over time
767	            # For now, use system-wide rates as approximation
768	            return self.get_system_io_rate()
769
770	        except Exception as e:
771	            print(f"Error getting process I/O: {e}", file=sys.stderr)
772	            return 0.0, 0.0
773
774
775	# ============================================================================
776	# Profiling Orchestrator
777	# ============================================================================
778
779
780	class ProfilingOrchestrator:
781	    """Manages application profiling lifecycle"""
782
783	    def __init__(self):
784	        self.cpu_monitor = CPUMonitor()
785	        self.gpu_monitor = GPUMonitor()
786	        self.npu_monitor = NPUMonitor()
787	        self.memory_monitor = MemoryMonitor()
788	        self.disk_monitor = DiskIOMonitor()
789	        self.sessions: Dict[str, ProfilingSession] = {}
790	        self._subprocess_handles: Dict[str, subprocess.Popen] = {}
791	        self._baseline_warnings: List[str] = []
792	        self._baseline_printed: bool = False
793
794	    def find_pids_by_name(self, process_name: str) -> List[int]:
795	        """Find all PIDs matching a process name (partial match)"""
796	        matching_pids = []
797
798	        try:
799	            for proc in psutil.process_iter(["pid", "name", "cmdline"]):
800	                try:
801	                    proc_info = proc.info
802	                    proc_name = proc_info["name"] or ""
803	                    cmdline = " ".join(proc_info["cmdline"] or [])
804
805	                    # Check if process name matches (case-insensitive, partial match)
806	                    if (
807	                        process_name.lower() in proc_name.lower()
808	                        or process_name.lower() in cmdline.lower()
809	                    ):
810	                        matching_pids.append(proc_info["pid"])
811
812	                except (
813	                    psutil.NoSuchProcess,
814	                    psutil.AccessDenied,
815	                    psutil.ZombieProcess,
816	                ):
817	                    continue
818
819	        except Exception as e:
820	            print(f"Error finding processes by name: {e}")
821
822	        return matching_pids
823
824	    def create_session(self, session_id: Optional[str] = None) -> ProfilingSession:
825	        """Initialize new profiling session"""
826	        if session_id is None:
827	            session_id = str(uuid.uuid4())
828
829	        session = ProfilingSession(
830	            session_id=session_id, start_time=time.time(), status="initializing"
831	        )
832
833	        self.sessions[session_id] = session
834	        return session
835
836	    def establish_baseline(
837	        self, session_id: str, duration: float = 5.0
838	    ) -> PlatformBaseline:
839	        """
840	        Capture system idle state
841
842	        Process:
843	        1. Wait for system to stabilize
844	        2. Sample CPU/GPU/NPU/Memory for duration
845	        3. Calculate average utilization
846	        4. Store as baseline
847	        """
848	        print(f"Establishing baseline for {duration} seconds...")
849
850	        samples = {"cpu": [], "memory": [], "gpu": [], "npu": [], "disk": []}
851
852	        # Collect samples
853	        start_time = time.time()
854	        while time.time() - start_time < duration:
855	            # CPU
856	            cpu_percent = self.cpu_monitor.get_overall_utilization(interval=0.5)
857	            samples["cpu"].append(cpu_percent)
858
859	            # Memory
860	            mem = self.memory_monitor.get_system_memory()
861	            samples["memory"].append(mem)
862
863	            # GPU
864	            gpu_utils = self.gpu_monitor.get_all_utilizations()
865	            samples["gpu"].append(gpu_utils)
866
867	            # NPU
868	            npu_percent = self.npu_monitor.get_utilization(sampling_period=0.1)
869	            samples["npu"].append(npu_percent)
870
871	            # Disk I/O
872	            disk_read, disk_write = self.disk_monitor.get_system_io_rate()
873	            samples["disk"].append((disk_read, disk_write))
874
875	            time.sleep(0.5)
876
877	        # Calculate averages
878	        avg_cpu = sum(samples["cpu"]) / len(samples["cpu"]) if samples["cpu"] else 0.0
879	        avg_npu = sum(samples["npu"]) / len(samples["npu"]) if samples["npu"] else 0.0
880
881	        # Average disk I/O
882	        if samples["disk"]:
883	            avg_disk_read = sum(d[0] for d in samples["disk"]) / len(samples["disk"])
884	            avg_disk_write = sum(d[1] for d in samples["disk"]) / len(samples["disk"])
885	        else:
886	            avg_disk_read = 0.0
887	            avg_disk_write = 0.0
888
889	        # Average memory
890	        if samples["memory"]:
891	            avg_mem_used = sum(m.used_mb for m in samples["memory"]) / len(
892	                samples["memory"]
893	            )
894	            avg_mem_percent = sum(m.percent for m in samples["memory"]) / len(
895	                samples["memory"]
896	            )
897	            mem_total = samples["memory"][0].total_mb
898	        else:
899	            avg_mem_used = 0.0
900	            avg_mem_percent = 0.0
901	            mem_total = 0.0
902
903	        # Average GPU
904	        gpu_baselines = []
905	        if samples["gpu"] and samples["gpu"][0]:
906	            for device_idx in range(len(samples["gpu"][0])):
907	                device_samples = [
908	                    sample[device_idx]
909	                    for sample in samples["gpu"]
910	                    if len(sample) > device_idx
911	                ]
912	                if device_samples:
913	                    avg_compute = sum(s.compute_percent for s in device_samples) / len(
914	                        device_samples
915	                    )
916	                    avg_mem_used = sum(s.memory_used_mb for s in device_samples) / len(
917	                        device_samples
918	                    )
919	                    avg_mem_percent = sum(
920	                        s.memory_percent for s in device_samples
921	                    ) / len(device_samples)
922
923	                    gpu_baselines.append(
924	                        GPUBaseline(
925	                            device_id=device_samples[0].device_id,
926	                            name=device_samples[0].name,
927	                            compute_percent=avg_compute,
928	                            memory_used_mb=avg_mem_used,
929	                            memory_total_mb=device_samples[0].memory_total_mb,
930	                            memory_percent=avg_mem_percent,
931	                        )
932	                    )
933
934	        cpu_info = self.cpu_monitor.get_cpu_info()
935
936	        baseline = PlatformBaseline(
937	            timestamp=time.time(),
938	            cpu_percent=avg_cpu,
939	            cpu_info=cpu_info,
940	            memory_total_mb=mem_total,
941	            memory_used_mb=avg_mem_used,
942	            memory_percent=avg_mem_percent,
943	            gpu_devices=gpu_baselines,
944	            npu_available=self.npu_monitor.is_available(),
945	            npu_percent=avg_npu,
946	            disk_read_mb_per_sec=avg_disk_read,
947	            disk_write_mb_per_sec=avg_disk_write,
948	        )
949
950	        # Store in session
951	        if session_id in self.sessions:
952	            self.sessions[session_id].baseline = baseline
953
954	        print(f"Baseline established:")
955	        print(f"  CPU: {avg_cpu:.1f}%")
956	        print(f"  Memory: {avg_mem_used:.0f} MB ({avg_mem_percent:.1f}%)")
957	        print(f"  GPU: {len(gpu_baselines)} device(s)")
958	        print(f"  NPU: {avg_npu:.1f}% (available: {self.npu_monitor.is_available()})")
959	        print(
960	            f"  Disk I/O: {avg_disk_read:.1f} MB/s read, {avg_disk_write:.1f} MB/s write"
961	        )
962
963	        # Check for high baseline (>80%) and warn user
964	        baseline_warnings = []
965	        if avg_cpu > 80.0:
966	            warning = (
967	                f"⚠  WARNING: CPU baseline is {avg_cpu:.1f}% (>80%) - system not idle!"
968	            )
969	            print(f"\n{warning}")
970	            baseline_warnings.append(f"CPU baseline: {avg_cpu:.1f}%")
971
972	        if gpu_baselines:
973	            for gpu in gpu_baselines:
974	                if gpu.compute_percent > 80.0:
975	                    warning = f"⚠  WARNING: GPU baseline is {gpu.compute_percent:.1f}% (>80%) - GPU already in use!"
976	                    print(f"\n{warning}")
977	                    baseline_warnings.append(
978	                        f"GPU baseline: {gpu.compute_percent:.1f}%"
979	                    )
980
981	        if avg_npu > 80.0:
982	            warning = f"⚠  WARNING: NPU baseline is {avg_npu:.1f}% (>80%) - NPU already in use!"
983	            print(f"\n{warning}")
984	            baseline_warnings.append(f"NPU baseline: {avg_npu:.1f}%")
985
986	        if baseline_warnings:
987	            print("\n⚠  High baseline detected! Results may be inaccurate.")
988	            print("   Recommendation: Stop other applications and re-run profiling.\n")
989
990	        # Store warnings for later analysis
991	        self._baseline_warnings = baseline_warnings
992
993	        return baseline
994
995	    def launch_application(
996	        self, session_id: str, app_name: str, command: str
997	    ) -> ApplicationProfile:
998	        """
999	        Launch application and start tracking
1000
1001	        Process:
1002	        1. Execute command in subprocess
1003	        2. Capture parent PID
1004	        3. Track all child PIDs
1005	        4. Return ApplicationProfile
1006	        """
1007	        app_id = f"{session_id}_{app_name}"
1008
1009	        print(f"Launching application '{app_name}': {command}")
1010
1011	        try:
1012	            # Parse command string into list to prevent command injection
1013	            # Using shlex.split() properly handles quoted arguments and spaces
1014	            command_list = shlex.split(command)
1015
1016	            # Launch process - using DEVNULL to prevent pipe buffer filling
1017	            # SECURITY: Using command list without shell=True prevents OS command injection
1018	            process = subprocess.Popen(
1019	                command_list,
1020	                stdout=subprocess.DEVNULL,
1021	                stderr=subprocess.PIPE,
1022	                preexec_fn=os.setsid,
1023	            )
1024
1025	            # Store process handle
1026	            self._subprocess_handles[app_id] = process
1027
1028	            time.sleep(1.5)  # Wait for process to start
1029
1030	            poll_result = process.poll()
1031	            if poll_result is not None:
1032	                # Process has already exited
1033	                stdout, stderr = process.communicate(timeout=1)
1034	                error_output = (
1035	                    stderr.decode("utf-8") if stderr else stdout.decode("utf-8")
1036	                )
1037
1038	                error_msg = f"Application exited immediately with code {poll_result}"
1039	                if error_output:
1040	                    error_msg += f"\nError output:\n{error_output}"
1041
1042	                print(f"ERROR: {error_msg}", file=sys.stderr)
1043
1044	                # Clean up
1045	                if app_id in self._subprocess_handles:
1046	                    del self._subprocess_handles[app_id]
1047
1048	                return ApplicationProfile(
1049	                    app_id=app_id,
1050	                    name=app_name,
1051	                    command=command,
1052	                    status="failed",
1053	                    error_message=error_msg,
1054	                )
1055
1056	            try:
1057	                proc = psutil.Process(process.pid)
1058	                if not proc.is_running():
1059	                    raise psutil.NoSuchProcess(process.pid)
1060	                process_name = proc.name()
1061	            except psutil.NoSuchProcess:
1062	                error_msg = f"Process {process.pid} not found after launch - application may have crashed"
1063	                print(f"ERROR: {error_msg}", file=sys.stderr)
1064
1065	                return ApplicationProfile(
1066	                    app_id=app_id,
1067	                    name=app_name,
1068	                    command=command,
1069	                    status="failed",
1070	                    error_message=error_msg,
1071	                )
1072	            except Exception as e:
1073	                process_name = "unknown"
1074	                print(f"Warning: Could not get process name: {e}")
1075
1076	            pids = self._get_process_tree_pids(process.pid)
1077
1078	            if not pids or len(pids) == 0:
1079	                error_msg = "No valid process IDs found after launch"
1080	                print(f"ERROR: {error_msg}", file=sys.stderr)
1081
1082	                return ApplicationProfile(
1083	                    app_id=app_id,
1084	                    name=app_name,
1085	                    command=command,
1086	                    status="failed",
1087	                    error_message=error_msg,
1088	                )
1089
1090	            app_profile = ApplicationProfile(
1091	                app_id=app_id,
1092	                name=app_name,
1093	                command=command,
1094	                process_name=process_name,
1095	                pids=pids,
1096	                root_pid=process.pid,
1097	                start_time=time.time(),
1098	                status="running",
1099	            )
1100
1101	            # Add to session
1102	            if session_id in self.sessions:
1103	                self.sessions[session_id].applications[app_name] = app_profile
1104
1105	            print(f"Application launched with PIDs: {pids}")
1106
1107	            return app_profile
1108
1109	        except Exception as e:
1110	            error_msg = f"Failed to launch application: {e}"
1111	            print(error_msg, file=sys.stderr)
1112
1113	            app_profile = ApplicationProfile(
1114	                app_id=app_id,
1115	                name=app_name,
1116	                command=command,
1117	                status="failed",
1118	                error_message=error_msg,
1119	            )
1120
1121	            return app_profile
1122
1123	    def attach_to_process(
1124	        self, session_id: str, app_name: str, pid: int
1125	    ) -> ApplicationProfile:
1126	        """Attach to an existing process for profiling"""
1127	        session = self.sessions.get(session_id)
1128	        if not session:
1129	            raise ValueError(f"Session {session_id} not found")
1130
1131	        app_id = f"{session_id}_{app_name}"
1132
1133	        # Verify process exists
1134	        try:
1135	            proc = psutil.Process(pid)
1136	            command = " ".join(proc.cmdline())
1137	            process_name = proc.name()
1138	            print(f"Attaching to process {pid}: {command}")
1139	        except psutil.NoSuchProcess:
1140	            error_msg = f"Process {pid} not found"
1141	            print(error_msg, file=sys.stderr)
1142	            return ApplicationProfile(
1143	                app_id=app_id,
1144	                name=app_name,
1145	                command=f"PID {pid}",
1146	                status="failed",
1147	                error_message=error_msg,
1148	            )
1149	        except psutil.AccessDenied:
1150	            error_msg = f"Access denied to process {pid}. Try running with sudo."
1151	            print(error_msg, file=sys.stderr)
1152	            return ApplicationProfile(
1153	                app_id=app_id,
1154	                name=app_name,
1155	                command=f"PID {pid}",
1156	                status="failed",
1157	                error_message=error_msg,
1158	            )
1159
1160	        # Get all PIDs in process tree
1161	        pids = self._get_process_tree_pids(pid)
1162
1163	        # Collect initial process tree metrics
1164	        process_tree = self._collect_process_tree_metrics(pid)
1165
1166	        # Create application profile
1167	        app_profile = ApplicationProfile(
1168	            app_id=app_id,
1169	            name=app_name,
1170	            process_name=process_name,
1171	            command=command,
1172	            pids=pids,
1173	            root_pid=pid,
1174	            process_tree=process_tree,
1175	            max_process_count=len(process_tree),
1176	            start_time=time.time(),
1177	            status="running",
1178	        )
1179
1180	        session.applications[app_name] = app_profile
1181	        print(f"Attached to process tree with {len(pids)} process(es):")
1182	        print(f"  Root PID: {pid}")
1183	        if len(pids) > 1:
1184	            print(f"  Child PIDs: {pids[1:]}")
1185
1186	        return app_profile
1187
1188	    def _get_process_tree_pids(self, parent_pid: int) -> List[int]:
1189	        """Get all PIDs in process tree"""
1190	        pids = [parent_pid]
1191
1192	        try:
1193	            parent = psutil.Process(parent_pid)
1194	            children = parent.children(recursive=True)
1195	            for child in children:
1196	                pids.append(child.pid)
1197	        except (psutil.NoSuchProcess, psutil.AccessDenied):
1198	            pass
1199
1200	        return pids
1201
1202	    def _collect_process_tree_metrics(
1203	        self, parent_pid: int
1204	    ) -> Dict[int, ProcessMetrics]:
1205	        """Collect detailed metrics for all processes in tree
1206
1207	        Returns:
1208	            Dict mapping PID to ProcessMetrics
1209	        """
1210	        metrics = {}
1211
1212	        # Get logical cores for CPU percent normalization
1213	        logical_cores = psutil.cpu_count(logical=True) or 1
1214
1215	        try:
1216	            parent = psutil.Process(parent_pid)
1217
1218	            # Add parent process
1219	            metrics[parent_pid] = self._get_process_metrics(parent, logical_cores)
1220
1221	            # Add all children recursively
1222	            children = parent.children(recursive=True)
1223	            for child in children:
1224	                try:
1225	                    child_metrics = self._get_process_metrics(child, logical_cores)
1226	                    metrics[child.pid] = child_metrics
1227	                except (psutil.NoSuchProcess, psutil.AccessDenied):
1228	                    continue
1229
1230	        except (psutil.NoSuchProcess, psutil.AccessDenied):
1231	            pass
1232
1233	        return metrics
1234
1235	    def _get_process_metrics(
1236	        self, proc: psutil.Process, logical_cores: int = 1
1237	    ) -> ProcessMetrics:
1238	        """Extract metrics from a psutil Process object
1239
1240	        Args:
1241	            proc: psutil Process object
1242	            logical_cores: Number of logical CPU cores for normalization
1243	        """
1244	        try:
1245	            mem_info = proc.memory_info()
1246	            cmdline = " ".join(proc.cmdline()) if proc.cmdline() else ""
1247
1248	            # Get direct children PIDs
1249	            children_pids = [c.pid for c in proc.children(recursive=False)]
1250
1251	            # Get raw CPU percent and normalize it (divide by logical cores)
1252	            raw_cpu_percent = proc.cpu_percent(interval=0.1)
1253	            normalized_cpu_percent = raw_cpu_percent / logical_cores
1254
1255	            return ProcessMetrics(
1256	                pid=proc.pid,
1257	                ppid=proc.ppid(),
1258	                name=proc.name(),
1259	                cmdline=cmdline[:100],  # Truncate long commands
1260	                cpu_percent=normalized_cpu_percent,
1261	                memory_mb=mem_info.rss / (1024 * 1024),
1262	                memory_percent=proc.memory_percent(),
1263	                num_threads=proc.num_threads(),
1264	                status=proc.status(),
1265	                create_time=proc.create_time(),
1266	                children_pids=children_pids,
1267	            )
1268	        except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess) as e:
1269	            # Return minimal metrics for problematic processes
1270	            return ProcessMetrics(
1271	                pid=proc.pid,
1272	                ppid=0,
1273	                name="<unavailable>",
1274	                cmdline="",
1275	                status="error",
1276	            )
1277
1278	    def _update_process_tree_stats(
1279	        self, app_profile: ApplicationProfile, current_tree: Dict[int, ProcessMetrics]
1280	    ) -> None:
1281	        """Update process tree statistics (spawns, exits, max count)"""
1282	        old_pids = set(app_profile.process_tree.keys())
1283	        new_pids = set(current_tree.keys())
1284
1285	        # Detect new spawns
1286	        spawned_pids = new_pids - old_pids
1287	        app_profile.process_spawns += len(spawned_pids)
1288
1289	        # Detect exits
1290	        exited_pids = old_pids - new_pids
1291	        app_profile.process_exits += len(exited_pids)
1292
1293	        # Update max process count
1294	        app_profile.max_process_count = max(
1295	            app_profile.max_process_count, len(current_tree)
1296	        )
1297
1298	        # Log significant changes
1299	        if spawned_pids and len(spawned_pids) <= 5:
1300	            for pid in spawned_pids:
1301	                if pid in current_tree:
1302	                    proc = current_tree[pid]
1303	                    print(f"  [+] Process spawned: PID {pid} ({proc.name})")
1304	        elif spawned_pids:
1305	            print(f"  [+] {len(spawned_pids)} new processes spawned")
1306
1307	        if exited_pids and len(exited_pids) <= 5:
1308	            for pid in exited_pids:
1309	                if pid in app_profile.process_tree:
1310	                    proc = app_profile.process_tree[pid]
1311	                    print(f"  [-] Process exited: PID {pid} ({proc.name})")
1312	        elif exited_pids:
1313	            print(f"  [-] {len(exited_pids)} processes exited")
1314
1315	    def monitor_application(
1316	        self,
1317	        session_id: str,
1318	        app_name: str,
1319	        duration: float = 60.0,
1320	        sample_interval: float = 1.0,
1321	    ) -> ApplicationProfile:
1322	        """
1323	        Monitor application resource usage
1324
1325	        Process:
1326	        1. Sample every sample_interval seconds
1327	        2. Calculate deltas from baseline
1328	        3. Attribute resources to app PIDs
1329	        4. Update ApplicationProfile
1330	        """
1331	        session = self.sessions.get(session_id)
1332	        if not session:
1333	            raise ValueError(f"Session {session_id} not found")
1334
1335	        app_profile = session.applications.get(app_name)
1336	        if not app_profile:
1337	            raise ValueError(f"Application {app_name} not found in session")
1338
1339	        baseline = session.baseline
1340	        if not baseline:
1341	            raise ValueError("Baseline not established")
1342
1343	        print(f"Monitoring application '{app_name}' for {duration} seconds...")
1344	        print(f"  Process tree tracking: enabled")
1345
1346	        samples = []
1347	        start_time = time.time()
1348	        sample_count = 0
1349
1350	        while time.time() - start_time < duration:
1351	            sample_count += 1
1352
1353	            # Update process tree (detects spawns/exits)
1354	            if app_profile.pids:
1355	                root_pid = app_profile.pids[0]
1356
1357	                # Collect detailed metrics for entire process tree
1358	                current_tree = self._collect_process_tree_metrics(root_pid)
1359
1360	                # Update process tree statistics
1361	                self._update_process_tree_stats(app_profile, current_tree)
1362
1363	                # Update stored process tree
1364	                app_profile.process_tree = current_tree
1365
1366	                # Update PIDs list
1367	                app_profile.pids = list(current_tree.keys())
1368
1369	            # Calculate current deltas
1370	            delta = self.calculate_deltas(baseline, app_profile.pids)
1371	            samples.append(delta)
1372
1373	            # Check if process is still running
1374	            if app_profile.pids:
1375	                try:
1376	                    psutil.Process(app_profile.pids[0])
1377	                except psutil.NoSuchProcess:
1378	                    print(f"Application process terminated")
1379	                    break
1380
1381	            time.sleep(sample_interval)
1382
1383	        # Get logical core count for normalization
1384	        logical_cores = baseline.cpu_info.logical_threads
1385	        app_profile.logical_cores = logical_cores
1386
1387	        # Calculate averages and peaks
1388	        if samples:
1389	            # Averages - normalize CPU percent to 0-100% scale
1390	            raw_cpu_percent = sum(s.cpu_percent for s in samples) / len(samples)
1391	            app_profile.cpu_percent_normalized = (
1392	                raw_cpu_percent / logical_cores
1393	                if logical_cores > 0
1394	                else raw_cpu_percent
1395	            )
1396	            # Make cpu_percent same as normalized for consistency
1397	            app_profile.cpu_percent = app_profile.cpu_percent_normalized
1398	            app_profile.memory_mb = sum(s.memory_mb for s in samples) / len(samples)
1399	            app_profile.memory_percent = sum(s.memory_percent for s in samples) / len(
1400	                samples
1401	            )
1402	            app_profile.gpu_compute_percent = sum(
1403	                s.gpu_compute_percent for s in samples
1404	            ) / len(samples)
1405	            app_profile.gpu_memory_mb = sum(s.gpu_memory_mb for s in samples) / len(
1406	                samples
1407	            )
1408	            app_profile.gpu_memory_percent = sum(
1409	                s.gpu_memory_percent for s in samples
1410	            ) / len(samples)
1411	            app_profile.npu_percent = sum(s.npu_percent for s in samples) / len(samples)
1412	            app_profile.disk_read_mb_per_sec = sum(
1413	                s.disk_read_mb_per_sec for s in samples
1414	            ) / len(samples)
1415	            app_profile.disk_write_mb_per_sec = sum(
1416	                s.disk_write_mb_per_sec for s in samples
1417	            ) / len(samples)
1418	            app_profile.disk_io_percent = sum(s.disk_io_percent for s in samples) / len(
1419	                samples
1420	            )
1421
1422	            # Peaks (maximum values) - normalize CPU percent to 0-100% scale
1423	            raw_peak_cpu = max(s.cpu_percent for s in samples)
1424	            app_profile.peak_cpu_percent_normalized = (
1425	                raw_peak_cpu / logical_cores if logical_cores > 0 else raw_peak_cpu
1426	            )
1427	            # Make peak_cpu_percent same as normalized for consistency
1428	            app_profile.peak_cpu_percent = app_profile.peak_cpu_percent_normalized
1429	            app_profile.peak_memory_mb = max(s.memory_mb for s in samples)
1430	            app_profile.peak_gpu_compute_percent = max(
1431	                s.gpu_compute_percent for s in samples
1432	            )
1433	            app_profile.peak_gpu_memory_mb = max(s.gpu_memory_mb for s in samples)
1434	            app_profile.peak_npu_percent = max(s.npu_percent for s in samples)
1435	            app_profile.peak_disk_read_mb_per_sec = max(
1436	                s.disk_read_mb_per_sec for s in samples
1437	            )
1438	            app_profile.peak_disk_write_mb_per_sec = max(
1439	                s.disk_write_mb_per_sec for s in samples
1440	            )
1441	            app_profile.peak_disk_io_percent = max(s.disk_io_percent for s in samples)
1442
1443	        app_profile.duration = time.time() - app_profile.start_time
1444	        app_profile.status = "completed"
1445
1446	        print(f"\nMonitoring completed:")
1447	        print(f"  Process Tree Summary:")
1448	        print(f"    Root PID: {app_profile.root_pid}")
1449	        print(f"    Final process count: {len(app_profile.process_tree)}")
1450	        print(f"    Peak process count: {app_profile.max_process_count}")
1451	        print(f"    Total spawns: {app_profile.process_spawns}")
1452	        print(f"    Total exits: {app_profile.process_exits}")
1453	        print(f"")
1454	        print(f"  Resource Utilization:")
1455	        print(
1456	            f"    CPU ({logical_cores} cores): {app_profile.cpu_percent_normalized:.1f}% (avg), {app_profile.peak_cpu_percent_normalized:.1f}% (peak)"
1457	        )
1458	        print(
1459	            f"    [Total: {app_profile.cpu_percent:.1f}% avg, {app_profile.peak_cpu_percent:.1f}% peak across all cores]"
1460	        )
1461	        print(
1462	            f"  Memory: {app_profile.memory_mb:.0f} MB (avg), {app_profile.peak_memory_mb:.0f} MB (peak)"
1463	        )
1464	        print(
1465	            f"  GPU Compute: {app_profile.gpu_compute_percent:.1f}% (avg), {app_profile.peak_gpu_compute_percent:.1f}% (peak)"
1466	        )
1467	        print(
1468	            f"  GPU Memory: {app_profile.gpu_memory_mb:.0f} MB (avg), {app_profile.peak_gpu_memory_mb:.0f} MB (peak)"
1469	        )
1470	        print(
1471	            f"  NPU: {app_profile.npu_percent:.1f}% (avg), {app_profile.peak_npu_percent:.1f}% (peak)"
1472	        )
1473	        print(
1474	            f"  Disk I/O: {app_profile.disk_read_mb_per_sec:.1f}/{app_profile.disk_write_mb_per_sec:.1f} MB/s (avg read/write)"
1475	        )
1476	        print(
1477	            f"    [Peak: {app_profile.peak_disk_read_mb_per_sec:.1f}/{app_profile.peak_disk_write_mb_per_sec:.1f} MB/s]"
1478	        )
1479
1480	        # Print process tree details
1481	        self._print_process_tree(app_profile)
1482
1483	        # NPU diagnostics - warn if NPU appears inactive
1484	        if app_profile.npu_percent == 0.0 and self.npu_monitor.is_available():
1485	            npu_state = self.npu_monitor.get_power_state()
1486	            npu_freq = self.npu_monitor.get_frequency()
1487	            print(f"  NPU Status: power={npu_state}, freq={npu_freq}MHz")
1488	            if npu_state == "suspended" or npu_freq == 0:
1489	                print(
1490	                    f"  ⚠ NPU appears inactive (power state: {npu_state}, freq: {npu_freq}MHz)"
1491	                )
1492	                print(
1493	                    f"     This may indicate: 1) App not using NPU, 2) NPU driver issue, 3) OpenVINO using CPU fallback"
1494	                )
1495
1496	        return app_profile
1497
1498	    def _print_process_tree(self, app_profile: ApplicationProfile) -> None:
1499	        """Print hierarchical process tree with resource usage"""
1500	        if not app_profile.process_tree or len(app_profile.process_tree) <= 1:
1501	            return
1502
1503	        # Sort processes by CPU usage
1504	        sorted_procs = sorted(
1505	            app_profile.process_tree.values(),
1506	            key=lambda p: p.cpu_percent,
1507	            reverse=True,
1508	        )
1509
1510	        # Show top processes (limit to 5 for simplicity)
1511	        print(f"\n  Process Tree ({len(app_profile.process_tree)} processes):")
1512	        for i, proc in enumerate(sorted_procs[:5]):
1513	            print(
1514	                f"    {proc.pid:<8} {proc.name:<20} {proc.cpu_percent:>6.1f}%  {proc.memory_mb:>7.1f} MB"
1515	            )
1516
1517	        if len(sorted_procs) > 5:
1518	            print(f"    ... and {len(sorted_procs) - 5} more processes")
1519
1520	    def calculate_deltas(
1521	        self, baseline: PlatformBaseline, app_pids: List[int]
1522	    ) -> ApplicationProfile:
1523	        """
1524	        Calculate resource attribution for application
1525
1526	        Method:
1527	        - CPU: Sum CPU% of all app PIDs
1528	        - Memory: Sum RSS of all app PIDs
1529	        - GPU: Total GPU% - baseline GPU%
1530	        - NPU: Total NPU% - baseline NPU%
1531	        """
1532	        delta = ApplicationProfile(app_id="temp", name="temp", command="")
1533
1534	        # CPU - sum of all PIDs
1535	        cpu_percents = self.cpu_monitor.get_per_process_utilization(app_pids)
1536	        delta.cpu_percent = sum(cpu_percents.values())
1537
1538	        # Memory - sum of all PIDs
1539	        mem_mb, mem_percent = self.memory_monitor.get_process_tree_memory(app_pids)
1540	        delta.memory_mb = mem_mb
1541	        delta.memory_percent = mem_percent
1542
1543	        # GPU - delta from baseline
1544	        gpu_utils = self.gpu_monitor.get_all_utilizations()
1545	        if gpu_utils and baseline.gpu_devices:
1546	            current_gpu = gpu_utils[0]
1547	            baseline_gpu = baseline.gpu_devices[0]
1548
1549	            delta.gpu_compute_percent = max(
1550	                0.0, current_gpu.compute_percent - baseline_gpu.compute_percent
1551	            )
1552	            delta.gpu_memory_mb = max(
1553	                0.0, current_gpu.memory_used_mb - baseline_gpu.memory_used_mb
1554	            )
1555	            delta.gpu_memory_percent = max(
1556	                0.0, current_gpu.memory_percent - baseline_gpu.memory_percent
1557	            )
1558
1559	        # NPU - delta from baseline
1560	        current_npu = self.npu_monitor.get_utilization(sampling_period=0.1)
1561	        delta.npu_percent = max(0.0, current_npu - baseline.npu_percent)
1562
1563	        # Disk I/O - delta from baseline
1564	        current_read, current_write = self.disk_monitor.get_system_io_rate()
1565	        delta.disk_read_mb_per_sec = max(
1566	            0.0, current_read - baseline.disk_read_mb_per_sec
1567	        )
1568	        delta.disk_write_mb_per_sec = max(
1569	            0.0, current_write - baseline.disk_write_mb_per_sec
1570	        )
1571	        # Calculate I/O percentage (assuming 500 MB/s as 100% - typical SSD)
1572	        total_io = delta.disk_read_mb_per_sec + delta.disk_write_mb_per_sec
1573	        delta.disk_io_percent = min(100.0, (total_io / 500.0) * 100.0)
1574
1575	        # Transfer baseline warnings if any
1576	        if hasattr(self, "_baseline_warnings"):
1577	            delta.baseline_warnings = self._baseline_warnings.copy()
1578
1579	        # Debug: Print baseline comparison (first sample only)
1580	        if not hasattr(self, "_baseline_printed"):
1581	            # Get current system CPU for comparison
1582	            current_system_cpu = self.cpu_monitor.get_overall_utilization(interval=0.1)
1583
1584	            print(f"\n[Baseline Debug]")
1585	            print(
1586	                f"  CPU: baseline={baseline.cpu_percent:.1f}%, system_current={current_system_cpu:.1f}%, app_pids={delta.cpu_percent:.1f}%"
1587	            )
1588	            print(
1589	                f"  NPU: baseline={baseline.npu_percent:.1f}%, current={current_npu:.1f}%, delta={delta.npu_percent:.1f}%"
1590	            )
1591	            if gpu_utils and baseline.gpu_devices:
1592	                print(
1593	                    f"  GPU: baseline={baseline_gpu.compute_percent:.1f}%, current={current_gpu.compute_percent:.1f}%, delta={delta.gpu_compute_percent:.1f}%"
1594	                )
1595	            self._baseline_printed = True
1596
1597	        return delta
1598
1599	    def stop_application(self, session_id: str, app_name: str):
1600	        """Stop a running application"""
1601	        app_id = f"{session_id}_{app_name}"
1602
1603	        if app_id in self._subprocess_handles:
1604	            process = self._subprocess_handles[app_id]
1605	            try:
1606	                # Kill process group
1607	                os.killpg(os.getpgid(process.pid), signal.SIGTERM)
1608	                process.wait(timeout=5)
1609	            except:
1610	                try:
1611	                    os.killpg(os.getpgid(process.pid), signal.SIGKILL)
1612	                except:
1613	                    pass
1614
1615	            del self._subprocess_handles[app_id]
1616	            print(f"Application '{app_name}' stopped")
1617
1618	    def prepare_profile_data(
1619	        self,
1620	        session: ProfilingSession,
1621	        profile: ApplicationProfile,
1622	        baseline: PlatformBaseline,
1623	        analysis: "BottleneckAnalysis",
1624	    ) -> Dict[str, Any]:
1625	        """Prepare profiling results as dictionary for JSON serialization
1626
1627	        This method is useful for FastAPI endpoints that need to return JSON directly.
1628
1629	        Args:
1630	            session: Profiling session object
1631	            profile: Application profile with metrics
1632	            baseline: Platform baseline measurements
1633	            analysis: Bottleneck analysis results
1634
1635	        Returns:
1636	            Dictionary containing all profiling data, ready for JSON serialization
1637
1638	        Example (FastAPI):
1639	            @app.get("/profile")
1640	            async def get_profile():
1641	                orchestrator = ProfilingOrchestrator()
1642	                # ... run profiling ...
1643	                data = orchestrator.prepare_profile_data(session, profile, baseline, analysis)
1644	                return data  # FastAPI auto-converts to JSON response
1645	        """
1646	        # Convert profile to dictionary
1647	        app_data = asdict(profile)
1648
1649	        # If GPU compute is 0.0%, ensure GPU metrics are 0.0 (GPU not in use)
1650	        if (
1651	            profile.gpu_compute_percent == 0.0
1652	            and profile.peak_gpu_compute_percent == 0.0
1653	        ):
1654	            app_data["gpu_compute_percent"] = 0.0
1655	            app_data["gpu_memory_mb"] = 0.0
1656	            app_data["gpu_memory_percent"] = 0.0
1657	            app_data["peak_gpu_compute_percent"] = 0.0
1658	            app_data["peak_gpu_memory_mb"] = 0.0
1659
1660	        return {
1661	            "session_id": session.session_id,
1662	            "timestamp": datetime.fromtimestamp(time.time()).isoformat(),
1663	            "baseline": asdict(baseline),
1664	            "application": app_data,
1665	            "bottleneck_analysis": asdict(analysis),
1666	        }
1667
1668	    def save_profile_to_json(
1669	        self,
1670	        filepath: str,
1671	        session: ProfilingSession,
1672	        profile: ApplicationProfile,
1673	        baseline: PlatformBaseline,
1674	        analysis: "BottleneckAnalysis",
1675	    ) -> None:
1676	        """Save profiling results to JSON file
1677
1678	        Args:
1679	            filepath: Output JSON file path
1680	            session: Profiling session object
1681	            profile: Application profile with metrics
1682	            baseline: Platform baseline measurements
1683	            analysis: Bottleneck analysis results
1684	        """
1685	        output_data = self.prepare_profile_data(session, profile, baseline, analysis)
1686
1687	        with open(filepath, "w") as f:
1688	            json.dump(output_data, f, indent=2)
1689
1690	        print(f"\n✓ Profile saved to: {filepath}")
1691
1692
1693	# ============================================================================
1694	# CLI Interface
1695	# ============================================================================
1696
1697
1698	def print_banner():
1699	    """Print tool banner"""
1700	    print("╔" + "═" * 78 + "╗")
1701	    print("║" + " " * 20 + "PLATFORM PROFILER v1.0" + " " * 36 + "║")
1702	    print("║" + " " * 15 + "CPU/GPU/NPU/Memory Utilization Analysis" + " " * 24 + "║")
1703	    print("╚" + "═" * 78 + "╝")
1704	    print()
1705
1706
1707	def run_profile(args):
1708	    """Run profiling command"""
1709	    # Validate arguments
1710	    if not args.app and not args.pid:
1711	        print("ERROR: Either --app or --pid must be specified.")
1712	        print("\nOptions:")
1713	        print("  --app 'command'  : Launch and profile a new application")
1714	        print("  --pid 12345      : Attach to and profile an existing process")
1715	        print(
1716	            "\nExample 1: python3 platform_profiler.py profile --app 'python inference.py' --name 'my_app'"
1717	        )
1718	        print(
1719	            "Example 2: python3 platform_profiler.py profile --pid 12345 --name 'my_app'"
1720	        )
1721	        return
1722
1723	    if args.app and args.pid:
1724	        print("ERROR: Cannot specify both --app and --pid. Choose one.")
1725	        return
1726
1727	    print_banner()
1728	    print(f"Profiling Application: {args.name}")
1729	    print("=" * 80)
1730
1731	    orchestrator = ProfilingOrchestrator()
1732
1733	    # Create session
1734	    session = orchestrator.create_session()
1735
1736	    # Establish baseline
1737	    print(f"\nEstablishing baseline ({args.baseline}s)...")
1738	    baseline = orchestrator.establish_baseline(
1739	        session.session_id, duration=args.baseline
1740	    )
1741
1742	    # Profile application
1743	    if args.pid:
1744	        print(f"\nAttaching to PID: {args.pid}")
1745	        app = orchestrator.attach_to_process(session.session_id, args.name, args.pid)
1746	    else:
1747	        # SECURITY: Validate command string to prevent command injection (taint barrier)
1748	        try:
1749	            validated_command = validate_command_string(args.app)
1750	        except ValueError as e:
1751	            print(f"\nERROR: Invalid command: {e}")
1752	            return
1753
1754	        # Validate file paths in the command (universal approach)
1755	        command_parts = validated_command.strip().split()
1756
1757	        # Check each argument that looks like a file path
1758	        for arg in command_parts:
1759	            # Skip flags/options (start with -)
1760	            if arg.startswith("-"):
1761	                continue
1762
1763	            # Check if argument looks like a file path (has extension or path separators)
1764	            if "/" in arg or "\\" in arg or ("." in arg and not arg.startswith(".")):
1765	                # SECURITY: Validate file path to prevent path manipulation (taint barrier)
1766	                try:
1767	                    file_path = validate_file_path(arg)
1768	                except ValueError as e:
1769	                    print(f"\nERROR: Invalid file path '{arg}': {e}")
1770	                    return
1771
1772	                # Validate file exists
1773	                if not file_path.exists():
1774	                    print(f"\nERROR: File not found: {arg}")
1775	                    print(f"  Resolved path: {file_path}")
1776	                    print(f"  Current directory: {Path.cwd()}")
1777	                    print(f"\nPlease check the file path and try again.")
1778	                    return
1779
1780	                if not file_path.is_file():
1781	                    print(f"\nERROR: Path is not a file: {arg}")
1782	                    print(f"  Resolved path: {file_path}")
1783	                    print(f"\nPlease provide a valid file path.")
1784	                    return
1785
1786	                print(f"\n✓ File validation passed: {file_path}")
1787
1788	        print(f"\nLaunching: {validated_command}")
1789	        # Pass validated command (taint removed by validate_command_string)
1790	        app = orchestrator.launch_application(
1791	            session.session_id, args.name, validated_command
1792	        )
1793
1794	    if app.status == "running":
1795	        profile = orchestrator.monitor_application(
1796	            session.session_id, args.name, duration=args.duration
1797	        )
1798
1799	        # Only stop if we launched it (not if we attached to existing PID)
1800	        if args.app:
1801	            orchestrator.stop_application(session.session_id, args.name)
1802	        else:
1803	            print(f"Note: Process {args.pid} is still running (attached mode)")
1804
1805	        # Show results
1806	        print("\n" + "=" * 80)
1807	        print("PROFILING RESULTS")
1808	        print("=" * 80)
1809	        print(f"\nApplication: {args.name}")
1810	        print(f"Duration: {profile.duration:.1f}s")
1811	        print(f"\n{'Resource':<20} {'Average':<20} {'Peak':<20}")
1812	        print("-" * 80)
1813	        print(
1814	            f"{'CPU (normalized)':<20} {profile.cpu_percent_normalized:>6.1f}% {profile.peak_cpu_percent_normalized:>20.1f}%"
1815	        )
1816	        print(
1817	            f"{'  ({profile.logical_cores} cores)':<20} [{profile.cpu_percent:>5.1f}% total] [{profile.peak_cpu_percent:>14.1f}% total]"
1818	        )
1819	        print(
1820	            f"{'Memory':<20} {profile.memory_mb:>10.0f} MB {profile.peak_memory_mb:>15.0f} MB"
1821	        )
1822	        print(
1823	            f"{'GPU Compute':<20} {profile.gpu_compute_percent:>6.1f}% {profile.peak_gpu_compute_percent:>20.1f}%"
1824	        )
1825	        print(
1826	            f"{'GPU Memory':<20} {profile.gpu_memory_mb:>10.0f} MB {profile.peak_gpu_memory_mb:>15.0f} MB"
1827	        )
1828	        print(
1829	            f"{'NPU':<20} {profile.npu_percent:>6.1f}% {profile.peak_npu_percent:>20.1f}%"
1830	        )
1831	        print(
1832	            f"{'Disk Read':<20} {profile.disk_read_mb_per_sec:>8.1f} MB/s {profile.peak_disk_read_mb_per_sec:>16.1f} MB/s"
1833	        )
1834	        print(
1835	            f"{'Disk Write':<20} {profile.disk_write_mb_per_sec:>8.1f} MB/s {profile.peak_disk_write_mb_per_sec:>16.1f} MB/s"
1836	        )
1837
1838	        # Perform bottleneck analysis
1839	        analyzer = BottleneckAnalyzer()
1840	        analysis = analyzer.analyze(profile, baseline)
1841
1842	        # Display bottleneck analysis
1843	        analyzer.print_analysis(analysis)
1844
1845	        # Save to file if requested
1846	        if args.output:
1847	            orchestrator.save_profile_to_json(
1848	                args.output, session, profile, baseline, analysis
1849	            )
1850
1851	    else:
1852	        print(f"\n✗ Failed to launch application: {app.error_message}")
1853
1854
1855	def run_status(args):
1856	    """Show current platform status"""
1857	    orchestrator = ProfilingOrchestrator()
1858
1859	    print("╔" + "═" * 78 + "╗")
1860	    print("║" + " " * 28 + "PLATFORM STATUS" + " " * 35 + "║")
1861	    print("╚" + "═" * 78 + "╝")
1862	    print()
1863
1864	    # CPU
1865	    cpu_info = orchestrator.cpu_monitor.get_cpu_info()
1866	    cpu_util = orchestrator.cpu_monitor.get_overall_utilization()
1867	    print(f"CPU: {cpu_info.model}")
1868	    print(
1869	        f"  Cores: {cpu_info.physical_cores} physical, {cpu_info.logical_threads} logical"
1870	    )
1871	    print(f"  Utilization: {cpu_util:.1f}%")
1872	    print()
1873
1874	    # Memory
1875	    mem = orchestrator.memory_monitor.get_system_memory()
1876	    print(f"Memory:")
1877	    print(f"  Total: {mem.total_mb:,.0f} MB ({mem.total_mb/1024:.1f} GB)")
1878	    print(f"  Used: {mem.used_mb:,.0f} MB ({mem.percent:.1f}%)")
1879	    print(f"  Available: {mem.available_mb:,.0f} MB ({100-mem.percent:.1f}%)")
1880	    print()
1881
1882	    # GPU
1883	    gpus = orchestrator.gpu_monitor.get_all_utilizations()
1884	    if gpus:
1885	        for gpu in gpus:
1886	            print(f"GPU {gpu.device_id}: {gpu.name}")
1887	            print(f"  Compute: {gpu.compute_percent:.1f}%")
1888	            print(
1889	                f"  Memory: {gpu.memory_used_mb:.0f} MB / {gpu.memory_total_mb:.0f} MB ({gpu.memory_percent:.1f}%)"
1890	            )
1891	            print()
1892	    else:
1893	        print("GPU: Not available")
1894	        print()
1895
1896	    # NPU
1897	    if orchestrator.npu_monitor.is_available():
1898	        npu_util = orchestrator.npu_monitor.get_utilization()
1899	        print(f"NPU: Available")
1900	        print(f"  Utilization: {npu_util:.1f}%")
1901	    else:
1902	        print("NPU: Not available")
1903	    print()
1904
1905
1906	# ============================================================================
1907	# FastAPI Interface
1908	# ============================================================================
1909
1910	if FASTAPI_AVAILABLE:
1911	    app = FastAPI(title="Platform Profiler API", version="1.0")
1912
1913	    app.add_middleware(
1914	        CORSMiddleware,
1915	        allow_origins=["http://127.0.0.1:8080", "http://localhost:8080"],
1916	        allow_credentials=True,
1917	        allow_methods=["*"],
1918	        allow_headers=["*"],
1919	    )
1920
1921	    # Global orchestrator instance for API
1922	    api_orchestrator = ProfilingOrchestrator()
1923	    api_session = None
1924	    api_baseline = None
1925
1926	    @app.get("/api/status")
1927	    async def get_platform_status():
1928	        """Get current platform status (CPU, GPU, NPU, Memory)"""
1929	        print("=== /api/status endpoint called ===")
1930	        try:
1931	            # Call orchestrator to get platform status
1932	            cpu_info = api_orchestrator.cpu_monitor.get_cpu_info()
1933	            cpu_util = api_orchestrator.cpu_monitor.get_overall_utilization()
1934	            mem = api_orchestrator.memory_monitor.get_system_memory()
1935	            gpus = api_orchestrator.gpu_monitor.get_all_utilizations()
1936	            npu_util = api_orchestrator.npu_monitor.get_utilization()
1937	            npu_available = api_orchestrator.npu_monitor.is_available()
1938
1939	            response = {
1940	                "cpu": {
1941	                    "model": cpu_info.model,
1942	                    "physical_cores": cpu_info.physical_cores,
1943	                    "logical_threads": cpu_info.logical_threads,
1944	                    "frequency_mhz": round(cpu_info.frequency_mhz, 2),
1945	                    "utilization_percent": round(cpu_util, 2),
1946	                },
1947	                "memory": {
1948	                    "total_mb": round(mem.total_mb, 2),
1949	                    "used_mb": round(mem.used_mb, 2),
1950	                    "available_mb": round(mem.available_mb, 2),
1951	                    "percent": round(mem.percent, 2),
1952	                },
1953	                "npu": {
1954	                    "available": npu_available,
1955	                    "utilization_percent": round(npu_util, 2) if npu_available else 0.0,
1956	                },
1957	            }
1958
1959	            # Add GPU metrics if available
1960	            if gpus:
1961	                if len(gpus) == 1:
1962	                    gpu = gpus[0]
1963	                    response["gpu"] = {
1964	                        "device_id": gpu.device_id,
1965	                        "name": gpu.name,
1966	                        "compute_percent": round(gpu.compute_percent, 2),
1967	                        "memory_used_mb": round(gpu.memory_used_mb, 2),
1968	                        "memory_total_mb": round(gpu.memory_total_mb, 2),
1969	                        "memory_percent": round(gpu.memory_percent, 2),
1970	                    }
1971	                else:
1972	                    response["gpus"] = []
1973	                    for gpu in gpus:
1974	                        gpu_data = {
1975	                            "device_id": gpu.device_id,
1976	                            "name": gpu.name,
1977	                            "compute_percent": round(gpu.compute_percent, 2),
1978	                            "memory_used_mb": round(gpu.memory_used_mb, 2),
1979	                            "memory_total_mb": round(gpu.memory_total_mb, 2),
1980	                            "memory_percent": round(gpu.memory_percent, 2),
1981	                        }
1982	                        response["gpus"].append(gpu_data)
1983	            else:
1984	                response["gpu"] = None
1985
1986	            return response
1987
1988	        except Exception as e:
1989	            print(f"Error in get_platform_status: {e}")
1990	            raise HTTPException(status_code=500, detail=str(e))
1991
1992	    @app.post("/api/profile-start")
1993	    async def start_profiling(pid: Optional[int] = None, app_name: str = "api_app"):
1994	        """Start profiling session (attach to PID or prepare for monitoring)"""
1995	        global api_session, api_baseline
1996	        print(
1997	            f"=== /api/profile/start endpoint called (pid={pid}, name={app_name}) ==="
1998	        )
1999
2000	        try:
2001	            # Create new session
2002	            api_session = api_orchestrator.create_session()
2003
2004	            # Establish baseline
2005	            api_baseline = api_orchestrator.establish_baseline(
2006	                api_session.session_id, duration=3.0
2007	            )
2008
2009	            response = {
2010	                "session_id": api_session.session_id,
2011	                "status": "baseline_established",
2012	                "baseline": {
2013	                    "cpu_percent": round(api_baseline.cpu_percent, 2),
2014	                    "memory_used_mb": round(api_baseline.memory_used_mb, 2),
2015	                    "npu_percent": round(api_baseline.npu_percent, 2),
2016	                },
2017	            }
2018
2019	            # If PID provided, attach to it
2020	            if pid:
2021	                app_profile = api_orchestrator.attach_to_process(
2022	                    api_session.session_id, app_name, pid
2023	                )
2024	                try:
2025	                    proc = psutil.Process(pid)
2026	                    actual_process_name = proc.name()
2027	                    actual_cmdline = " ".join(proc.cmdline())
2028	                except:
2029	                    actual_process_name = "unknown"
2030	                    actual_cmdline = ""
2031
2032	                response["status"] = "profiling_started"
2033	                response["app_name"] = app_name
2034	                response["pid"] = pid
2035	                response["process_name"] = actual_process_name
2036	                response["cmdline"] = actual_cmdline
2037
2038	            return response
2039
2040	        except Exception as e:
2041	            print(f"Error in start_profiling: {e}")
2042	            raise HTTPException(status_code=500, detail=str(e))
2043
2044	    @app.get("/api/profile-monitor")
2045	    async def monitor_profile(app_name: str = "api_app", duration: float = 30.0):
2046	        """Monitor application and return profiling results"""
2047	        global api_session, api_baseline
2048	        print(
2049	            f"=== /api/profile/monitor endpoint called (name={app_name}, duration={duration}) ==="
2050	        )
2051
2052	        if not api_session or not api_baseline:
2053	            raise HTTPException(
2054	                status_code=400,
2055	                detail="No active session. Call /api/profile/start first",
2056	            )
2057
2058	        try:
2059	            # Monitor the application
2060	            profile = api_orchestrator.monitor_application(
2061	                api_session.session_id, app_name, duration=duration
2062	            )
2063
2064	            # Perform bottleneck analysis
2065	            analyzer = BottleneckAnalyzer()
2066	            analysis = analyzer.analyze(profile, api_baseline)
2067
2068	            # Prepare response data using prepare_profile_data
2069	            data = api_orchestrator.prepare_profile_data(
2070	                api_session, profile, api_baseline, analysis
2071	            )
2072
2073	            return data
2074
2075	        except Exception as e:
2076	            print(f"Error in monitor_profile: {e}")
2077	            raise HTTPException(status_code=500, detail=str(e))
2078
2079	    @app.get("/api/profile-quick")
2080	    async def quick_profile(pid: int, duration: float = 30.0):
2081	        """Quick profile: start, monitor, and return results in one call"""
2082	        print(
2083	            f"=== /api/profile/quick endpoint called (pid={pid}, duration={duration}) ==="
2084	        )
2085
2086	        try:
2087	            # Create session and establish baseline
2088	            session = api_orchestrator.create_session()
2089	            baseline = api_orchestrator.establish_baseline(
2090	                session.session_id, duration=3.0
2091	            )
2092
2093	            # Attach to process
2094	            app_name = f"pid_{pid}"
2095	            profile = api_orchestrator.attach_to_process(
2096	                session.session_id, app_name, pid
2097	            )
2098
2099	            if profile.status == "failed":
2100	                raise HTTPException(status_code=400, detail=profile.error_message)
2101
2102	            # Monitor
2103	            profile = api_orchestrator.monitor_application(
2104	                session.session_id, app_name, duration=duration
2105	            )
2106
2107	            # Analyze
2108	            analyzer = BottleneckAnalyzer()
2109	            analysis = analyzer.analyze(profile, baseline)
2110
2111	            # Return data using prepare_profile_data
2112	            data = api_orchestrator.prepare_profile_data(
2113	                session, profile, baseline, analysis
2114	            )
2115	            return data
2116
2117	        except HTTPException:
2118	            raise
2119	        except Exception as e:
2120	            print(f"Error in quick_profile: {e}")
2121	            raise HTTPException(status_code=500, detail=str(e))
2122
2123	    @app.post("/api/profile-batch")
2124	    async def batch_profile(request: dict):
2125	        """Batch profile with user input param"""
2126	        print("=== PYTHON BACKEND DEBUG ===")
2127	        print(f"Request received: {request}")
2128	        print(f"Request type: {type(request)}")
2129
2130	        pid = request.get("pid")
2131	        # Command to launch (e.g., "python3 /path/to/script.py")
2132	        app_path = request.get("app_path")
2133	        app_name = request.get("app_name", "unknown")
2134	        selection_type = request.get("selection_type", "pid")
2135	        duration = request.get("duration", 30.0)
2136
2137	        print(f"Extracted values:", flush=True)
2138	        print(f"  selection_type: '{selection_type}'")
2139	        print(f"  pid: {pid}")
2140	        print(f"  app_path: '{app_path}'")
2141	        print(f"  app_name: '{app_name}'")
2142	        print(f"  duration: {duration}")
2143
2144	        # Use existing PID
2145	        if selection_type == "pid":
2146	            if not pid:
2147	                raise HTTPException(
2148	                    status_code=400, detail="PID is required for 'pid' mode"
2149	                )
2150
2151	            print(f"[PID MODE] Attaching to PID: {pid}")
2152
2153	            try:
2154	                # Create session and establish baseline
2155	                session = api_orchestrator.create_session()
2156	                baseline = api_orchestrator.establish_baseline(
2157	                    session.session_id, duration=3.0
2158	                )
2159
2160	                # Attach to existing process
2161	                if app_name == "unknown":
2162	                    app_name = f"pid_{pid}"
2163
2164	                profile = api_orchestrator.attach_to_process(
2165	                    session.session_id, app_name, pid
2166	                )
2167
2168	                if profile.status == "failed":
2169	                    raise HTTPException(status_code=400, detail=profile.error_message)
2170
2171	                # Monitor
2172	                profile = api_orchestrator.monitor_application(
2173	                    session.session_id, app_name, duration=duration
2174	                )
2175
2176	                # Analyze
2177	                analyzer = BottleneckAnalyzer()
2178	                analysis = analyzer.analyze(profile, baseline)
2179
2180	                # Return data
2181	                data = api_orchestrator.prepare_profile_data(
2182	                    session, profile, baseline, analysis
2183	                )
2184	                return data
2185
2186	            except HTTPException:
2187	                raise
2188	            except Exception as e:
2189	                print(f"[PID MODE] Error: {e}")
2190	                raise HTTPException(status_code=500, detail=str(e))
2191
2192	        # Launch new application by path
2193	        elif selection_type == "appname":
2194	            if not app_path or app_path.strip() == "":
2195	                raise HTTPException(
2196	                    status_code=400,
2197	                    detail="Application path/command is required when using 'appname' selection type",
2198	                )
2199
2200	            print(f"[APPPATH MODE] Launching application: {app_path}")
2201
2202	            try:
2203	                # Validate file paths in the command (universal approach)
2204	                command_parts = app_path.strip().split()
2205	                validated_file = None
2206
2207	                # Check each argument that looks like a file path
2208	                for arg in command_parts:
2209	                    # Skip flags/options (start with -)
2210	                    if arg.startswith("-"):
2211	                        continue
2212
2213	                    # Check if argument looks like a file path (has extension or path separators)
2214	                    if (
2215	                        "/" in arg
2216	                        or "\\" in arg
2217	                        or ("." in arg and not arg.startswith("."))
2218	                    ):
2219	                        file_path = Path(arg)
2220
2221	                        # Handle relative paths - make them absolute from current working directory
2222	                        if not file_path.is_absolute():
2223	                            file_path = Path.cwd() / file_path
2224
2225	                        # Validate file exists
2226	                        if not file_path.exists():
2227	                            error_msg = (
2228	                                f"File not found: {arg} (resolved to: {file_path})"
2229	                            )
2230	                            print(f"[APPPATH MODE] ERROR: {error_msg}")
2231	                            raise HTTPException(
2232	                                status_code=400,
2233	                                detail={
2234	                                    "status": "failed",
2235	                                    "error": error_msg,
2236	                                    "message": "Please check the file path and try again.",
2237	                                    "current_dir": str(Path.cwd()),
2238	                                },
2239	                            )
2240
2241	                        if not file_path.is_file():
2242	                            error_msg = f"Path is not a file: {arg}"
2243	                            print(f"[APPPATH MODE] ERROR: {error_msg}")
2244	                            raise HTTPException(
2245	                                status_code=400,
2246	                                detail={
2247	                                    "status": "failed",
2248	                                    "error": error_msg,
2249	                                    "message": "Please provide a valid file path.",
2250	                                },
2251	                            )
2252
2253	                        print(f"[APPPATH MODE] File validation passed: {file_path}")
2254	                        if not validated_file:
2255	                            validated_file = file_path
2256
2257	                # Create session and establish baseline
2258	                session = api_orchestrator.create_session()
2259	                baseline = api_orchestrator.establish_baseline(
2260	                    session.session_id, duration=3.0
2261	                )
2262
2263	                # Generate app name from command if not provided
2264	                if app_name == "unknown":
2265	                    # Extract name from validated file or first command part
2266	                    if validated_file:
2267	                        app_name = validated_file.stem
2268	                    elif len(command_parts) > 0:
2269	                        app_name = Path(command_parts[0]).stem
2270	                    else:
2271	                        app_name = "unknown_app"
2272
2273	                print(f"[APPPATH MODE] App name: {app_name}")
2274
2275	                # Launch the application
2276	                profile = api_orchestrator.launch_application(
2277	                    session.session_id, app_name, app_path.strip()
2278	                )
2279
2280	                if profile.status == "failed":
2281	                    raise HTTPException(
2282	                        status_code=400,
2283	                        detail={
2284	                            "status": "failed",
2285	                            "error": profile.error_message,
2286	                            "message": "Application failed to launch or exited immediately.",
2287	                        },
2288	                    )
2289
2290	                print(
2291	                    f"[APPPATH MODE] Application launched with PID: {profile.root_pid}"
2292	                )
2293
2294	                # Monitor the launched application
2295	                profile = api_orchestrator.monitor_application(
2296	                    session.session_id, app_name, duration=duration
2297	                )
2298
2299	                print(f"[APPPATH MODE] Monitoring complete")
2300	                print(f"  process_name: '{profile.process_name}'")
2301	                print(f"  command: '{profile.command}'")
2302
2303	                # Stop the application after profiling
2304	                api_orchestrator.stop_application(session.session_id, app_name)
2305	                print(f"[APPPATH MODE] Application stopped")
2306
2307	                # Analyze
2308	                analyzer = BottleneckAnalyzer()
2309	                analysis = analyzer.analyze(profile, baseline)
2310
2311	                # Return data
2312	                data = api_orchestrator.prepare_profile_data(
2313	                    session, profile, baseline, analysis
2314	                )
2315
2316	                print(f"[APPPATH MODE] Profiling complete!")
2317	                return data
2318
2319	            except HTTPException:
2320	                raise
2321	            except Exception as e:
2322	                print(f"[APPPATH MODE] Error: {e}")
2323	                import traceback
2324
2325	                traceback.print_exc()
2326	                raise HTTPException(status_code=500, detail=str(e))
2327
2328	        else:
2329	            raise HTTPException(
2330	                status_code=400,
2331	                detail=f"Invalid selection_type: {selection_type}. Must be 'pid' or 'appname'",
2332	            )
2333
2334
2335	def parse_api_arguments():
2336	    """Parse arguments for API mode"""
2337	    import argparse
2338
2339	    parser = argparse.ArgumentParser(description="Platform Profiler API Service")
2340	    parser.add_argument(
2341	        "--port",
2342	        type=int,
2343	        default=6240,
2344	        help="Port to run the FastAPI server on (default: 6240)",
2345	    )
2346	    parser.add_argument(
2347	        "--host",
2348	        type=str,
2349	        default="127.0.0.1",
2350	        help="Host to bind service to (default: 127.0.0.1)",
2351	    )
2352	    parser.add_argument(
2353	        "--reload", action="store_true", help="Enable auto reload for development"
2354	    )
2355
2356	    return parser.parse_args()
2357
2358
2359	def main():
2360	    """Main CLI entry point"""
2361	    import argparse
2362
2363	    parser = argparse.ArgumentParser(
2364	        description="Platform Profiler - Application Resource Utilization Analysis",
2365	        formatter_class=argparse.RawDescriptionHelpFormatter,
2366	        epilog="""
2367	EXAMPLES:
2368	  # Check platform status
2369	  %(prog)s status
2370
2371	  # Profile your application
2372	  %(prog)s profile --app "python inference.py" --name "my_app" --duration 30
2373
2374	For more info: https://github.com/intel/...
2375	        """,
2376	    )
2377
2378	    subparsers = parser.add_subparsers(dest="command", help="Command to execute")
2379
2380	    # Status command
2381	    status_parser = subparsers.add_parser("status", help="Show current platform status")
2382
2383	    # Profile command
2384	    profile_parser = subparsers.add_parser("profile", help="Profile application")
2385	    profile_parser.add_argument(
2386	        "--app", type=str, help="Application command to execute"
2387	    )
2388	    profile_parser.add_argument(
2389	        "--pid", type=int, help="PID of running process to profile"
2390	    )
2391	    profile_parser.add_argument(
2392	        "--name", type=str, default="app", help="Application name (default: app)"
2393	    )
2394	    profile_parser.add_argument(
2395	        "--baseline",
2396	        type=float,
2397	        default=5.0,
2398	        help="Baseline duration in seconds (default: 5)",
2399	    )
2400	    profile_parser.add_argument(
2401	        "--duration",
2402	        type=float,
2403	        default=30.0,
2404	        help="Profile duration in seconds (default: 30)",
2405	    )
2406	    profile_parser.add_argument(
2407	        "--output", type=str, help="Output JSON file (optional)"
2408	    )
2409
2410	    args = parser.parse_args()
2411
2412	    if not args.command:
2413	        print_banner()
2414	        parser.print_help()
2415	        return
2416
2417	    # SECURITY: Validate command is one of the allowed subcommands (taint barrier)
2418	    # This acts as a taint barrier for static analysis tools
2419	    allowed_commands = ["profile", "status"]
2420	    if args.command not in allowed_commands:
2421	        print(f"ERROR: Invalid command '{args.command}'")
2422	        return
2423
2424	    # Execute validated command
2425	    # Note: Additional validation occurs inside each function for command-specific fields
2426	    if args.command == "profile":
2427	        # SECURITY: Break taint chain by validating individual fields
2428	        # Extract and validate each field to prevent taint propagation
2429	        validated_args = type(
2430	            "obj",
2431	            (object,),
2432	            {
2433	                "command": str(args.command),  # Already validated via whitelist
2434	                "app": args.app if args.app is None else str(args.app),
2435	                "pid": args.pid if args.pid is None else int(args.pid),
2436	                "name": args.name if args.name is None else str(args.name),
2437	                "baseline": float(args.baseline) if hasattr(args, "baseline") else 5.0,
2438	                "duration": float(args.duration) if hasattr(args, "duration") else 30.0,
2439	                "output": (
2440	                    args.output
2441	                    if not hasattr(args, "output") or args.output is None
2442	                    else str(args.output)
2443	                ),
2444	            },
2445	        )()
2446	        run_profile(validated_args)
2447	    elif args.command == "status":
2448	        run_status(args)
2449
2450
2451	if __name__ == "__main__":
2452	    # Check if running in API mode
2453	    if len(sys.argv) > 1 and sys.argv[1] == "api":
2454	        if not FASTAPI_AVAILABLE:
2455	            print("ERROR: FastAPI not available!", file=sys.stderr)
2456	            print("Install with: pip install fastapi uvicorn", file=sys.stderr)
2457	            sys.exit(1)
2458
2459	        # Remove 'api' from argv so parse_api_arguments works correctly
2460	        # sys.argv.pop(1)
2461	        sys.argv = [sys.argv[0]] + sys.argv[
2462	            2:
2463	        ]  # Keep script name, remove 'api', keep rest
2464	        args = parse_api_arguments()
2465
2466	        print(f"Starting Platform Profiler API on {args.host}:{args.port}")
2467	        print(f"API endpoints:")
2468	        print(f"  GET  http://{args.host}:{args.port}/api/status")
2469	        print(f"  POST http://{args.host}:{args.port}/api/profile-start")
2470	        print(f"  GET  http://{args.host}:{args.port}/api/profile-monitor")
2471	        print(
2472	            f"  GET  http://{args.host}:{args.port}/api/profile/quick?pid=<PID>&duration=<seconds>"
2473	        )
2474	        print(f"\nDocs: http://{args.host}:{args.port}/docs")
2475
2476	        uvicorn.run(app, host=args.host, port=args.port, reload=args.reload)
2477	    else:
2478	        # Run CLI mode
2479	        main()
2480

#!/usr/bin/env python3

# Copyright (C) 2025 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

"""
Platform Profiler - Application Resource Utilization Analysis
==============================================================
QUICK START:
    # Check platform status
    python3 main.py status

    # Profile your application (supports Python, Node.js, binaries, etc.)
    python3 main.py profile --app "python3 customer_sample_app.py" --name "my_app"

USAGE EXAMPLES:
    # Python application
    python3 main.py profile \\
        --app "python3 customer_sample_app.py" \\
        --name "sample_app" \\
        --duration 30 \\
        --output profile.json

    # Node.js application
    python3 main.py profile --app "node server.js" --name "node_app" --duration 60

    # Binary/executable
    python3 main.py profile --app "./my_program --arg1 --arg2" --name "my_program"

    # Attach to existing process
    python3 main.py profile --pid 12345 --name "existing_app"

For full documentation, run: python3 main.py --help
"""

import time
import subprocess
import sys
import json
import os
import re
import uuid
import shlex
from typing import Dict, List, Optional, Tuple, Any
from dataclasses import dataclass, asdict, field
from pathlib import Path
from datetime import datetime
import signal

# FastAPI imports
try:
    from fastapi import FastAPI, HTTPException
    from fastapi.middleware.cors import CORSMiddleware
    import uvicorn

    FASTAPI_AVAILABLE = True
except ImportError:
    FASTAPI_AVAILABLE = False

# Import bottleneck analyzer from separate module
from bottleneck_analyzer import BottleneckAnalyzer, Bottleneck, BottleneckAnalysis

# Try to import psutil, provide helpful error if missing
try:
    import psutil
except ImportError:
    print("ERROR: psutil module not found!", file=sys.stderr)
    print("", file=sys.stderr)
    print("Please install it using ONE of these methods:", file=sys.stderr)
    print("  1. pip install psutil", file=sys.stderr)
    print("  2. sudo apt-get install python3-psutil", file=sys.stderr)
    print(
        "  3. Create venv: python3 -m venv venv && source venv/bin/activate && pip install psutil",
        file=sys.stderr,
    )
    sys.exit(1)

# ============================================================================
# Security - Input Validation
# ============================================================================

def validate_command_string(command: str) -> str:
    """
    Validate and sanitize command string for security.

    This function serves as a taint barrier for static analysis tools.
    It ensures the command string doesn't contain dangerous patterns.

    Args:
        command: Command string to validate

    Returns:
        Validated command string (same as input if valid)

    Raises:
        ValueError: If command contains suspicious patterns
    """
    if not command or not command.strip():
        raise ValueError("Command string cannot be empty")

    # Check for null bytes (could terminate string unexpectedly)
    if "\x00" in command:
        raise ValueError("Command contains null byte")

    # Ensure command starts with an executable name (no leading special chars)
    first_char = command.strip()[0]
    if first_char in (";", "|", "&", ">", "<", "$", "`"):
        raise ValueError(f"Command cannot start with shell metacharacter: {first_char}")

    # Return the validated command (taint is now removed for static analysis)
    return command

def validate_file_path(path_str: str) -> Path:
    """
    Validate and resolve file path for security.

    This function serves as a taint barrier for static analysis tools.

    Args:
        path_str: Path string to validate

    Returns:
        Validated Path object

    Raises:
        ValueError: If path is invalid or suspicious
    """
    if not path_str or not path_str.strip():
        raise ValueError("Path cannot be empty")

    # Check for null bytes
    if "\x00" in path_str:
        raise ValueError("Path contains null byte")

    # Convert to Path object
    path = Path(path_str)

    # Make absolute if relative
    if not path.is_absolute():
        path = Path.cwd() / path

    # Resolve to canonical path (removes .. and symlinks)
    try:
        path = path.resolve()
    except (OSError, RuntimeError) as e:
        raise ValueError(f"Cannot resolve path: {e}")

    # Return validated path (taint is now removed for static analysis)
    return path

# ============================================================================
# Data Models
# ============================================================================

@dataclass
class CPUInfo:
    """CPU specifications"""

    model: str
    physical_cores: int
    logical_threads: int
    frequency_mhz: float

@dataclass
class GPUDevice:
    """GPU device information"""

    device_id: int
    name: str
    pci_device_id: str
    memory_total_mb: float

@dataclass
class GPUUtilization:
    """GPU utilization metrics"""

    device_id: int
    name: str
    compute_percent: float
    memory_used_mb: float
    memory_total_mb: float
    memory_percent: float

@dataclass
class SystemMemory:
    """System memory information"""

    total_mb: float
    used_mb: float
    available_mb: float
    percent: float

@dataclass
class ProcessMemory:
    """Per-process memory metrics"""

    pid: int
    rss_mb: float
    vms_mb: float
    percent: float

@dataclass
class ProcessMetrics:
    """Resource metrics for a single process in the tree"""

    pid: int
    ppid: int  # Parent PID
    name: str
    cmdline: str
    cpu_percent: float = 0.0
    memory_mb: float = 0.0
    memory_percent: float = 0.0
    num_threads: int = 0
    status: str = "running"  # running, sleeping, zombie, etc.
    create_time: float = 0.0
    children_pids: List[int] = field(default_factory=list)

@dataclass
class GPUBaseline:
    """GPU baseline state"""

    device_id: int
    name: str
    compute_percent: float
    memory_used_mb: float
    memory_total_mb: float
    memory_percent: float

@dataclass
class PlatformBaseline:
    """System state when idle (no applications running)"""

    timestamp: float

    # CPU baseline
    cpu_percent: float
    cpu_info: CPUInfo

    # Memory baseline
    memory_total_mb: float
    memory_used_mb: float
    memory_percent: float

    # GPU baseline (per device)
    gpu_devices: List[GPUBaseline]

    # NPU baseline
    npu_available: bool
    npu_percent: float

    # Disk I/O baseline
    disk_read_mb_per_sec: float = 0.0
    disk_write_mb_per_sec: float = 0.0

@dataclass
class ApplicationProfile:
    app_id: str
    name: str
    command: str
    pids: List[int] = field(default_factory=list)

    process_name: str = "unknown"

    # Process tree tracking
    process_tree: Dict[int, ProcessMetrics] = field(default_factory=dict)
    root_pid: int = 0  # Main process PID
    max_process_count: int = 0  # Peak number of processes
    process_spawns: int = 0  # Total child processes spawned
    process_exits: int = 0  # Total processes that exited

    # Resource utilization (deltas from baseline) - AVERAGES
    cpu_percent: float = 0.0  # Normalized CPU 0-100% (avg per core)
    cpu_percent_normalized: float = 0.0  # Same as cpu_percent (kept for compatibility)
    memory_mb: float = 0.0
    memory_percent: float = 0.0

    gpu_compute_percent: float = 0.0
    gpu_memory_mb: float = 0.0
    gpu_memory_percent: float = 0.0

    npu_percent: float = 0.0

    # Disk I/O
    disk_read_mb_per_sec: float = 0.0
    disk_write_mb_per_sec: float = 0.0
    disk_io_percent: float = 0.0  # Combined I/O activity percentage

    # Peak utilization (maximum observed values)
    peak_cpu_percent: float = 0.0  # Normalized CPU peak 0-100%
    peak_cpu_percent_normalized: float = (
        0.0  # Same as peak_cpu_percent (kept for compatibility)
    )
    peak_memory_mb: float = 0.0
    peak_gpu_compute_percent: float = 0.0
    peak_gpu_memory_mb: float = 0.0
    peak_npu_percent: float = 0.0
    peak_disk_read_mb_per_sec: float = 0.0
    peak_disk_write_mb_per_sec: float = 0.0
    peak_disk_io_percent: float = 0.0

    # System info
    logical_cores: int = 0  # Number of logical CPU cores

    # Metadata
    start_time: float = 0.0
    duration: float = 0.0
    status: str = "initializing"  # initializing, running, completed, failed
    error_message: Optional[str] = None

    # Baseline quality warnings
    baseline_warnings: List[str] = field(default_factory=list)

@dataclass
class ProfilingSession:
    """Manages multi-application profiling"""

    session_id: str
    applications: Dict[str, ApplicationProfile] = field(default_factory=dict)
    baseline: Optional[PlatformBaseline] = None
    start_time: float = 0.0
    status: str = "initializing"  # initializing, profiling, completed

# ============================================================================
# Resource Monitors
# ============================================================================

class CPUMonitor:
    """CPU utilization tracking"""

    def __init__(self):
        self._last_cpu_times = None

    def get_overall_utilization(self, interval: float = 1.0) -> float:
        """Get system-wide CPU % (0-100)"""
        try:
            return psutil.cpu_percent(interval=interval)
        except Exception as e:
            print(f"Error getting CPU utilization: {e}", file=sys.stderr)
            return 0.0

    def get_per_process_utilization(self, pids: List[int]) -> Dict[int, float]:
        """Get CPU % for specific PIDs"""
        result = {}
        for pid in pids:
            try:
                proc = psutil.Process(pid)
                cpu_percent = proc.cpu_percent(interval=0.1)
                result[pid] = cpu_percent
            except (psutil.NoSuchProcess, psutil.AccessDenied):
                result[pid] = 0.0
        return result

    def get_cpu_info(self) -> CPUInfo:
        """Get CPU specs (cores, threads, model)"""
        try:
            # Get model name
            model = "Unknown CPU"
            try:
                with open("/proc/cpuinfo", "r") as f:
                    for line in f:
                        if "model name" in line:
                            model = line.split(":")[1].strip()
                            break
            except (OSError, IOError):
                pass

            # Get core counts
            physical_cores = psutil.cpu_count(logical=False) or 1
            logical_threads = psutil.cpu_count(logical=True) or 1

            # Get frequency
            freq = psutil.cpu_freq()
            frequency_mhz = freq.current if freq else 0.0

            return CPUInfo(
                model=model,
                physical_cores=physical_cores,
                logical_threads=logical_threads,
                frequency_mhz=frequency_mhz,
            )
        except Exception as e:
            print(f"Error getting CPU info: {e}", file=sys.stderr)
            return CPUInfo(
                model="Unknown", physical_cores=1, logical_threads=1, frequency_mhz=0.0
            )

class MemoryMonitor:
    """System and per-process memory tracking"""

    def get_system_memory(self) -> SystemMemory:
        """Get system memory information"""
        try:
            mem = psutil.virtual_memory()
            return SystemMemory(
                total_mb=mem.total / (1024 * 1024),
                used_mb=mem.used / (1024 * 1024),
                available_mb=mem.available / (1024 * 1024),
                percent=mem.percent,
            )
        except Exception as e:
            print(f"Error getting system memory: {e}", file=sys.stderr)
            return SystemMemory(total_mb=0, used_mb=0, available_mb=0, percent=0)

    def get_process_memory(self, pid: int) -> Optional[ProcessMemory]:
        """Get memory for specific process"""
        try:
            proc = psutil.Process(pid)
            mem_info = proc.memory_info()
            mem_percent = proc.memory_percent()

            return ProcessMemory(
                pid=pid,
                rss_mb=mem_info.rss / (1024 * 1024),
                vms_mb=mem_info.vms / (1024 * 1024),
                percent=mem_percent,
            )
        except (psutil.NoSuchProcess, psutil.AccessDenied):
            return None

    def get_process_tree_memory(self, pids: List[int]) -> Tuple[float, float]:
        """Sum memory for all PIDs in tree

        Returns:
            Tuple of (total_mb, total_percent)
        """
        total_mb = 0.0
        total_percent = 0.0

        for pid in pids:
            mem = self.get_process_memory(pid)
            if mem:
                total_mb += mem.rss_mb
                total_percent += mem.percent

        return total_mb, total_percent

class GPUMonitor:
    """GPU utilization tracking using xpumcli"""

    def __init__(self):
        self._has_xpumcli = self._check_xpumcli()

    def _check_xpumcli(self) -> bool:
        """Check if xpumcli is available"""
        try:
            result = subprocess.run(
                ["which", "xpumcli"], capture_output=True, timeout=2
            )
            return result.returncode == 0
        except (OSError, subprocess.TimeoutExpired):
            return False

    def get_device_list(self) -> List[GPUDevice]:
        """List all available GPUs using xpumcli discovery"""
        if not self._has_xpumcli:
            return []

        devices = []
        try:
            # Use JSON output for easier parsing
            result = subprocess.run(
                ["xpumcli", "discovery", "-j"],
                capture_output=True,
                text=True,
                timeout=5,
            )

            if result.returncode == 0:
                data = json.loads(result.stdout)
                device_list = data.get("device_list", [])

                for device_info in device_list:
                    if device_info.get("device_type") == "GPU":
                        device_id = device_info.get("device_id", 0)
                        name = device_info.get("device_name", "Unknown GPU")
                        pci_id = device_info.get("pci_device_id", "")

                        # Get memory size (will get from stats later if not available here)
                        memory_total_mb = 0.0

                        devices.append(
                            GPUDevice(
                                device_id=device_id,
                                name=name,
                                pci_device_id=pci_id,
                                memory_total_mb=memory_total_mb,
                            )
                        )
        except json.JSONDecodeError as e:
            print(f"Error parsing GPU discovery JSON: {e}", file=sys.stderr)
        except Exception as e:
            print(f"Error getting GPU device list: {e}", file=sys.stderr)

        return devices

    def get_utilization(self, device_id: int = 0) -> Optional[GPUUtilization]:
        """Get GPU utilization for specific device using xpumcli stats"""
        if not self._has_xpumcli:
            return None

        try:
            # Use JSON output for easier parsing
            result = subprocess.run(
                ["xpumcli", "stats", "-d", str(device_id), "-j"],
                capture_output=True,
                text=True,
                timeout=5,
            )

            if result.returncode == 0:
                data = json.loads(result.stdout)

                compute_percent = 0.0
                memory_used_mb = 0.0
                memory_total_mb = 65536.0  # Default 64GB, will update if available
                name = "Unknown GPU"

                # Parse device_level metrics
                device_level = data.get("device_level", [])
                for metric in device_level:
                    metrics_type = metric.get("metrics_type", "")
                    value = metric.get("value", 0.0)

                    # Compute utilization
                    if "COMPUTE_ALL_UTILIZATION" in metrics_type:
                        compute_percent = value
                    # Memory used (in MiB)
                    elif "MEMORY_USED" in metrics_type:
                        memory_used_mb = value
                    # Memory utilization (percentage)
                    elif "MEMORY_UTILIZATION" in metrics_type:
                        # We have percentage, can calculate total
                        if value > 0 and memory_used_mb > 0:
                            memory_total_mb = (memory_used_mb / value) * 100

                # Get device name from discovery
                devices = self.get_device_list()
                if devices and len(devices) > device_id:
                    name = devices[device_id].name

                memory_percent = (
                    (memory_used_mb / memory_total_mb * 100)
                    if memory_total_mb > 0
                    else 0.0
                )

                return GPUUtilization(
                    device_id=device_id,
                    name=name,
                    compute_percent=compute_percent,
                    memory_used_mb=memory_used_mb,
                    memory_total_mb=memory_total_mb,
                    memory_percent=memory_percent,
                )
        except json.JSONDecodeError as e:
            print(f"Error parsing GPU stats JSON: {e}", file=sys.stderr)
        except Exception as e:
            print(f"Error getting GPU utilization: {e}", file=sys.stderr)

        return None

    def get_all_utilizations(self) -> List[GPUUtilization]:
        """Get utilization for all GPUs"""
        devices = self.get_device_list()
        utilizations = []

        for device in devices:
            util = self.get_utilization(device.device_id)
            if util:
                utilizations.append(util)

        return utilizations

class NPUMonitor:
    """NPU utilization tracking via sysfs"""

    def __init__(self, npu_sysfs_path: Optional[str] = None):
        self.npu_base_path = "/sys/devices/pci0000:00/0000:00:0b.0"
        self.npu_busy_time_path = (
            npu_sysfs_path or f"{self.npu_base_path}/npu_busy_time_us"
        )
        self.npu_freq_path = f"{self.npu_base_path}/npu_current_frequency_mhz"
        self.npu_power_path = f"{self.npu_base_path}/power/runtime_status"
        self._available = self._check_availability()

    def _check_availability(self) -> bool:
        """Check if NPU is present"""
        return os.path.exists(self.npu_busy_time_path)

    def is_available(self) -> bool:
        """Check if NPU is present"""
        return self._available

    def get_power_state(self) -> str:
        """Get NPU power state (active/suspended)"""
        try:
            if os.path.exists(self.npu_power_path):
                with open(self.npu_power_path, "r") as f:
                    return f.read().strip()
        except (OSError, IOError):
            pass
        return "unknown"

    def get_frequency(self) -> int:
        """Get current NPU frequency in MHz"""
        try:
            if os.path.exists(self.npu_freq_path):
                with open(self.npu_freq_path, "r") as f:
                    return int(f.read().strip())
        except (OSError, IOError, ValueError):
            pass
        return 0

    def get_utilization(self, sampling_period: float = 0.1) -> float:
        """
        Calculate NPU utilization % (0-100)

        Method:
        1. Read /sys/.../npu_busy_time_us at T1
        2. Wait sampling_period
        3. Read again at T2
        4. utilization = (busy_delta / time_delta) * 100

        Note: If NPU is suspended or frequency is 0, utilization will be 0.
        """
        if not self._available:
            return 0.0

        try:
            # Check power state - if suspended, NPU counter won't increment
            power_state = self.get_power_state()
            frequency = self.get_frequency()

            # If NPU is suspended or at 0 frequency, utilization is 0
            # But we still try to read the counter in case it updates

            # Read initial busy time
            with open(self.npu_busy_time_path, "r") as f:
                busy_time_1 = int(f.read().strip())

            time_1 = time.time()

            # Wait
            time.sleep(sampling_period)

            # Read final busy time
            with open(self.npu_busy_time_path, "r") as f:
                busy_time_2 = int(f.read().strip())

            time_2 = time.time()

            # Calculate deltas
            busy_delta_us = busy_time_2 - busy_time_1
            time_delta_us = (time_2 - time_1) * 1_000_000

            # Calculate utilization
            if time_delta_us > 0 and busy_delta_us > 0:
                utilization = (busy_delta_us / time_delta_us) * 100.0
                return min(100.0, max(0.0, utilization))

            # If counter didn't change but frequency > 0, there might be activity
            # that the counter doesn't capture. For now, return 0.
            if busy_delta_us == 0 and frequency > 0 and power_state == "active":
                # NPU might be active but counter not updating - this is a driver issue
                # We can't reliably measure utilization in this case
                pass

        except Exception as e:
            print(f"Error getting NPU utilization: {e}", file=sys.stderr)

        return 0.0

class DiskIOMonitor:
    """Disk I/O tracking using psutil"""

    def __init__(self):
        self._last_io_counters = None
        self._last_sample_time = None

    def get_system_io_rate(self) -> Tuple[float, float]:
        """Get system-wide disk I/O rates in MB/s

        Returns:
            Tuple of (read_mb_per_sec, write_mb_per_sec)
        """
        try:
            current_io = psutil.disk_io_counters()
            current_time = time.time()

            if self._last_io_counters is None:
                # First sample - initialize
                self._last_io_counters = current_io
                self._last_sample_time = current_time
                time.sleep(0.1)  # Small delay for first delta
                current_io = psutil.disk_io_counters()
                current_time = time.time()

            # Calculate deltas
            time_delta = current_time - self._last_sample_time
            if time_delta == 0:
                return 0.0, 0.0

            read_bytes_delta = current_io.read_bytes - self._last_io_counters.read_bytes
            write_bytes_delta = (
                current_io.write_bytes - self._last_io_counters.write_bytes
            )

            # Convert to MB/s
            read_mb_per_sec = (read_bytes_delta / time_delta) / (1024 * 1024)
            write_mb_per_sec = (write_bytes_delta / time_delta) / (1024 * 1024)

            # Update last counters
            self._last_io_counters = current_io
            self._last_sample_time = current_time

            return max(0.0, read_mb_per_sec), max(0.0, write_mb_per_sec)

        except Exception as e:
            print(f"Error getting disk I/O: {e}", file=sys.stderr)
            return 0.0, 0.0

    def get_process_io_rate(self, pids: List[int]) -> Tuple[float, float]:
        """Get disk I/O rates for specific processes

        Note: Process-level I/O counters may not be available on all systems.
        Falls back to system-wide measurement.

        Returns:
            Tuple of (read_mb_per_sec, write_mb_per_sec)
        """
        total_read_bytes = 0
        total_write_bytes = 0

        try:
            for pid in pids:
                try:
                    proc = psutil.Process(pid)
                    io_counters = proc.io_counters()
                    total_read_bytes += io_counters.read_bytes
                    total_write_bytes += io_counters.write_bytes
                except (psutil.NoSuchProcess, psutil.AccessDenied, AttributeError):
                    # Process gone, access denied, or io_counters not available
                    continue

            # For process I/O, we return cumulative values
            # Real rate calculation would require tracking over time
            # For now, use system-wide rates as approximation
            return self.get_system_io_rate()

        except Exception as e:
            print(f"Error getting process I/O: {e}", file=sys.stderr)
            return 0.0, 0.0

# ============================================================================
# Profiling Orchestrator
# ============================================================================

class ProfilingOrchestrator:
    """Manages application profiling lifecycle"""

    def __init__(self):
        self.cpu_monitor = CPUMonitor()
        self.gpu_monitor = GPUMonitor()
        self.npu_monitor = NPUMonitor()
        self.memory_monitor = MemoryMonitor()
        self.disk_monitor = DiskIOMonitor()
        self.sessions: Dict[str, ProfilingSession] = {}
        self._subprocess_handles: Dict[str, subprocess.Popen] = {}
        self._baseline_warnings: List[str] = []
        self._baseline_printed: bool = False

    def find_pids_by_name(self, process_name: str) -> List[int]:
        """Find all PIDs matching a process name (partial match)"""
        matching_pids = []

        try:
            for proc in psutil.process_iter(["pid", "name", "cmdline"]):
                try:
                    proc_info = proc.info
                    proc_name = proc_info["name"] or ""
                    cmdline = " ".join(proc_info["cmdline"] or [])

                    # Check if process name matches (case-insensitive, partial match)
                    if (
                        process_name.lower() in proc_name.lower()
                        or process_name.lower() in cmdline.lower()
                    ):
                        matching_pids.append(proc_info["pid"])

                except (
                    psutil.NoSuchProcess,
                    psutil.AccessDenied,
                    psutil.ZombieProcess,
                ):
                    continue

        except Exception as e:
            print(f"Error finding processes by name: {e}")

        return matching_pids

    def create_session(self, session_id: Optional[str] = None) -> ProfilingSession:
        """Initialize new profiling session"""
        if session_id is None:
            session_id = str(uuid.uuid4())

        session = ProfilingSession(
            session_id=session_id, start_time=time.time(), status="initializing"
        )

        self.sessions[session_id] = session
        return session

    def establish_baseline(
        self, session_id: str, duration: float = 5.0
    ) -> PlatformBaseline:
        """
        Capture system idle state

        Process:
        1. Wait for system to stabilize
        2. Sample CPU/GPU/NPU/Memory for duration
        3. Calculate average utilization
        4. Store as baseline
        """
        print(f"Establishing baseline for {duration} seconds...")

        samples = {"cpu": [], "memory": [], "gpu": [], "npu": [], "disk": []}

        # Collect samples
        start_time = time.time()
        while time.time() - start_time < duration:
            # CPU
            cpu_percent = self.cpu_monitor.get_overall_utilization(interval=0.5)
            samples["cpu"].append(cpu_percent)

            # Memory
            mem = self.memory_monitor.get_system_memory()
            samples["memory"].append(mem)

            # GPU
            gpu_utils = self.gpu_monitor.get_all_utilizations()
            samples["gpu"].append(gpu_utils)

            # NPU
            npu_percent = self.npu_monitor.get_utilization(sampling_period=0.1)
            samples["npu"].append(npu_percent)

            # Disk I/O
            disk_read, disk_write = self.disk_monitor.get_system_io_rate()
            samples["disk"].append((disk_read, disk_write))

            time.sleep(0.5)

        # Calculate averages
        avg_cpu = sum(samples["cpu"]) / len(samples["cpu"]) if samples["cpu"] else 0.0
        avg_npu = sum(samples["npu"]) / len(samples["npu"]) if samples["npu"] else 0.0

        # Average disk I/O
        if samples["disk"]:
            avg_disk_read = sum(d[0] for d in samples["disk"]) / len(samples["disk"])
            avg_disk_write = sum(d[1] for d in samples["disk"]) / len(samples["disk"])
        else:
            avg_disk_read = 0.0
            avg_disk_write = 0.0

        # Average memory
        if samples["memory"]:
            avg_mem_used = sum(m.used_mb for m in samples["memory"]) / len(
                samples["memory"]
            )
            avg_mem_percent = sum(m.percent for m in samples["memory"]) / len(
                samples["memory"]
            )
            mem_total = samples["memory"][0].total_mb
        else:
            avg_mem_used = 0.0
            avg_mem_percent = 0.0
            mem_total = 0.0

        # Average GPU
        gpu_baselines = []
        if samples["gpu"] and samples["gpu"][0]:
            for device_idx in range(len(samples["gpu"][0])):
                device_samples = [
                    sample[device_idx]
                    for sample in samples["gpu"]
                    if len(sample) > device_idx
                ]
                if device_samples:
                    avg_compute = sum(s.compute_percent for s in device_samples) / len(
                        device_samples
                    )
                    avg_mem_used = sum(s.memory_used_mb for s in device_samples) / len(
                        device_samples
                    )
                    avg_mem_percent = sum(
                        s.memory_percent for s in device_samples
                    ) / len(device_samples)

                    gpu_baselines.append(
                        GPUBaseline(
                            device_id=device_samples[0].device_id,
                            name=device_samples[0].name,
                            compute_percent=avg_compute,
                            memory_used_mb=avg_mem_used,
                            memory_total_mb=device_samples[0].memory_total_mb,
                            memory_percent=avg_mem_percent,
                        )
                    )

        cpu_info = self.cpu_monitor.get_cpu_info()

        baseline = PlatformBaseline(
            timestamp=time.time(),
            cpu_percent=avg_cpu,
            cpu_info=cpu_info,
            memory_total_mb=mem_total,
            memory_used_mb=avg_mem_used,
            memory_percent=avg_mem_percent,
            gpu_devices=gpu_baselines,
            npu_available=self.npu_monitor.is_available(),
            npu_percent=avg_npu,
            disk_read_mb_per_sec=avg_disk_read,
            disk_write_mb_per_sec=avg_disk_write,
        )

        # Store in session
        if session_id in self.sessions:
            self.sessions[session_id].baseline = baseline

        print(f"Baseline established:")
        print(f"  CPU: {avg_cpu:.1f}%")
        print(f"  Memory: {avg_mem_used:.0f} MB ({avg_mem_percent:.1f}%)")
        print(f"  GPU: {len(gpu_baselines)} device(s)")
        print(f"  NPU: {avg_npu:.1f}% (available: {self.npu_monitor.is_available()})")
        print(
            f"  Disk I/O: {avg_disk_read:.1f} MB/s read, {avg_disk_write:.1f} MB/s write"
        )

        # Check for high baseline (>80%) and warn user
        baseline_warnings = []
        if avg_cpu > 80.0:
            warning = (
                f"⚠  WARNING: CPU baseline is {avg_cpu:.1f}% (>80%) - system not idle!"
            )
            print(f"\n{warning}")
            baseline_warnings.append(f"CPU baseline: {avg_cpu:.1f}%")

        if gpu_baselines:
            for gpu in gpu_baselines:
                if gpu.compute_percent > 80.0:
                    warning = f"⚠  WARNING: GPU baseline is {gpu.compute_percent:.1f}% (>80%) - GPU already in use!"
                    print(f"\n{warning}")
                    baseline_warnings.append(
                        f"GPU baseline: {gpu.compute_percent:.1f}%"
                    )

        if avg_npu > 80.0:
            warning = f"⚠  WARNING: NPU baseline is {avg_npu:.1f}% (>80%) - NPU already in use!"
            print(f"\n{warning}")
            baseline_warnings.append(f"NPU baseline: {avg_npu:.1f}%")

        if baseline_warnings:
            print("\n⚠  High baseline detected! Results may be inaccurate.")
            print("   Recommendation: Stop other applications and re-run profiling.\n")

        # Store warnings for later analysis
        self._baseline_warnings = baseline_warnings

        return baseline

    def launch_application(
        self, session_id: str, app_name: str, command: str
    ) -> ApplicationProfile:
        """
        Launch application and start tracking

        Process:
        1. Execute command in subprocess
        2. Capture parent PID
        3. Track all child PIDs
        4. Return ApplicationProfile
        """
        app_id = f"{session_id}_{app_name}"

        print(f"Launching application '{app_name}': {command}")

        try:
            # Parse command string into list to prevent command injection
            # Using shlex.split() properly handles quoted arguments and spaces
            command_list = shlex.split(command)

            # Launch process - using DEVNULL to prevent pipe buffer filling
            # SECURITY: Using command list without shell=True prevents OS command injection
            process = subprocess.Popen(
                command_list,
                stdout=subprocess.DEVNULL,
                stderr=subprocess.PIPE,
                preexec_fn=os.setsid,
            )

            # Store process handle
            self._subprocess_handles[app_id] = process

            time.sleep(1.5)  # Wait for process to start

            poll_result = process.poll()
            if poll_result is not None:
                # Process has already exited
                stdout, stderr = process.communicate(timeout=1)
                error_output = (
                    stderr.decode("utf-8") if stderr else stdout.decode("utf-8")
                )

                error_msg = f"Application exited immediately with code {poll_result}"
                if error_output:
                    error_msg += f"\nError output:\n{error_output}"

                print(f"ERROR: {error_msg}", file=sys.stderr)

                # Clean up
                if app_id in self._subprocess_handles:
                    del self._subprocess_handles[app_id]

                return ApplicationProfile(
                    app_id=app_id,
                    name=app_name,
                    command=command,
                    status="failed",
                    error_message=error_msg,
                )

            try:
                proc = psutil.Process(process.pid)
                if not proc.is_running():
                    raise psutil.NoSuchProcess(process.pid)
                process_name = proc.name()
            except psutil.NoSuchProcess:
                error_msg = f"Process {process.pid} not found after launch - application may have crashed"
                print(f"ERROR: {error_msg}", file=sys.stderr)

                return ApplicationProfile(
                    app_id=app_id,
                    name=app_name,
                    command=command,
                    status="failed",
                    error_message=error_msg,
                )
            except Exception as e:
                process_name = "unknown"
                print(f"Warning: Could not get process name: {e}")

            pids = self._get_process_tree_pids(process.pid)

            if not pids or len(pids) == 0:
                error_msg = "No valid process IDs found after launch"
                print(f"ERROR: {error_msg}", file=sys.stderr)

                return ApplicationProfile(
                    app_id=app_id,
                    name=app_name,
                    command=command,
                    status="failed",
                    error_message=error_msg,
                )

            app_profile = ApplicationProfile(
                app_id=app_id,
                name=app_name,
                command=command,
                process_name=process_name,
                pids=pids,
                root_pid=process.pid,
                start_time=time.time(),
                status="running",
            )

            # Add to session
            if session_id in self.sessions:
                self.sessions[session_id].applications[app_name] = app_profile

            print(f"Application launched with PIDs: {pids}")

            return app_profile

        except Exception as e:
            error_msg = f"Failed to launch application: {e}"
            print(error_msg, file=sys.stderr)

            app_profile = ApplicationProfile(
                app_id=app_id,
                name=app_name,
                command=command,
                status="failed",
                error_message=error_msg,
            )

            return app_profile

    def attach_to_process(
        self, session_id: str, app_name: str, pid: int
    ) -> ApplicationProfile:
        """Attach to an existing process for profiling"""
        session = self.sessions.get(session_id)
        if not session:
            raise ValueError(f"Session {session_id} not found")

        app_id = f"{session_id}_{app_name}"

        # Verify process exists
        try:
            proc = psutil.Process(pid)
            command = " ".join(proc.cmdline())
            process_name = proc.name()
            print(f"Attaching to process {pid}: {command}")
        except psutil.NoSuchProcess:
            error_msg = f"Process {pid} not found"
            print(error_msg, file=sys.stderr)
            return ApplicationProfile(
                app_id=app_id,
                name=app_name,
                command=f"PID {pid}",
                status="failed",
                error_message=error_msg,
            )
        except psutil.AccessDenied:
            error_msg = f"Access denied to process {pid}. Try running with sudo."
            print(error_msg, file=sys.stderr)
            return ApplicationProfile(
                app_id=app_id,
                name=app_name,
                command=f"PID {pid}",
                status="failed",
                error_message=error_msg,
            )

        # Get all PIDs in process tree
        pids = self._get_process_tree_pids(pid)

        # Collect initial process tree metrics
        process_tree = self._collect_process_tree_metrics(pid)

        # Create application profile
        app_profile = ApplicationProfile(
            app_id=app_id,
            name=app_name,
            process_name=process_name,
            command=command,
            pids=pids,
            root_pid=pid,
            process_tree=process_tree,
            max_process_count=len(process_tree),
            start_time=time.time(),
            status="running",
        )

        session.applications[app_name] = app_profile
        print(f"Attached to process tree with {len(pids)} process(es):")
        print(f"  Root PID: {pid}")
        if len(pids) > 1:
            print(f"  Child PIDs: {pids[1:]}")

        return app_profile

    def _get_process_tree_pids(self, parent_pid: int) -> List[int]:
        """Get all PIDs in process tree"""
        pids = [parent_pid]

        try:
            parent = psutil.Process(parent_pid)
            children = parent.children(recursive=True)
            for child in children:
                pids.append(child.pid)
        except (psutil.NoSuchProcess, psutil.AccessDenied):
            pass

        return pids

    def _collect_process_tree_metrics(
        self, parent_pid: int
    ) -> Dict[int, ProcessMetrics]:
        """Collect detailed metrics for all processes in tree

        Returns:
            Dict mapping PID to ProcessMetrics
        """
        metrics = {}

        # Get logical cores for CPU percent normalization
        logical_cores = psutil.cpu_count(logical=True) or 1

        try:
            parent = psutil.Process(parent_pid)

            # Add parent process
            metrics[parent_pid] = self._get_process_metrics(parent, logical_cores)

            # Add all children recursively
            children = parent.children(recursive=True)
            for child in children:
                try:
                    child_metrics = self._get_process_metrics(child, logical_cores)
                    metrics[child.pid] = child_metrics
                except (psutil.NoSuchProcess, psutil.AccessDenied):
                    continue

        except (psutil.NoSuchProcess, psutil.AccessDenied):
            pass

        return metrics

    def _get_process_metrics(
        self, proc: psutil.Process, logical_cores: int = 1
    ) -> ProcessMetrics:
        """Extract metrics from a psutil Process object

        Args:
            proc: psutil Process object
            logical_cores: Number of logical CPU cores for normalization
        """
        try:
            mem_info = proc.memory_info()
            cmdline = " ".join(proc.cmdline()) if proc.cmdline() else ""

            # Get direct children PIDs
            children_pids = [c.pid for c in proc.children(recursive=False)]

            # Get raw CPU percent and normalize it (divide by logical cores)
            raw_cpu_percent = proc.cpu_percent(interval=0.1)
            normalized_cpu_percent = raw_cpu_percent / logical_cores

            return ProcessMetrics(
                pid=proc.pid,
                ppid=proc.ppid(),
                name=proc.name(),
                cmdline=cmdline[:100],  # Truncate long commands
                cpu_percent=normalized_cpu_percent,
                memory_mb=mem_info.rss / (1024 * 1024),
                memory_percent=proc.memory_percent(),
                num_threads=proc.num_threads(),
                status=proc.status(),
                create_time=proc.create_time(),
                children_pids=children_pids,
            )
        except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess) as e:
            # Return minimal metrics for problematic processes
            return ProcessMetrics(
                pid=proc.pid,
                ppid=0,
                name="<unavailable>",
                cmdline="",
                status="error",
            )

    def _update_process_tree_stats(
        self, app_profile: ApplicationProfile, current_tree: Dict[int, ProcessMetrics]
    ) -> None:
        """Update process tree statistics (spawns, exits, max count)"""
        old_pids = set(app_profile.process_tree.keys())
        new_pids = set(current_tree.keys())

        # Detect new spawns
        spawned_pids = new_pids - old_pids
        app_profile.process_spawns += len(spawned_pids)

        # Detect exits
        exited_pids = old_pids - new_pids
        app_profile.process_exits += len(exited_pids)

        # Update max process count
        app_profile.max_process_count = max(
            app_profile.max_process_count, len(current_tree)
        )

        # Log significant changes
        if spawned_pids and len(spawned_pids) <= 5:
            for pid in spawned_pids:
                if pid in current_tree:
                    proc = current_tree[pid]
                    print(f"  [+] Process spawned: PID {pid} ({proc.name})")
        elif spawned_pids:
            print(f"  [+] {len(spawned_pids)} new processes spawned")

        if exited_pids and len(exited_pids) <= 5:
            for pid in exited_pids:
                if pid in app_profile.process_tree:
                    proc = app_profile.process_tree[pid]
                    print(f"  [-] Process exited: PID {pid} ({proc.name})")
        elif exited_pids:
            print(f"  [-] {len(exited_pids)} processes exited")

    def monitor_application(
        self,
        session_id: str,
        app_name: str,
        duration: float = 60.0,
        sample_interval: float = 1.0,
    ) -> ApplicationProfile:
        """
        Monitor application resource usage

        Process:
        1. Sample every sample_interval seconds
        2. Calculate deltas from baseline
        3. Attribute resources to app PIDs
        4. Update ApplicationProfile
        """
        session = self.sessions.get(session_id)
        if not session:
            raise ValueError(f"Session {session_id} not found")

        app_profile = session.applications.get(app_name)
        if not app_profile:
            raise ValueError(f"Application {app_name} not found in session")

        baseline = session.baseline
        if not baseline:
            raise ValueError("Baseline not established")

        print(f"Monitoring application '{app_name}' for {duration} seconds...")
        print(f"  Process tree tracking: enabled")

        samples = []
        start_time = time.time()
        sample_count = 0

        while time.time() - start_time < duration:
            sample_count += 1

            # Update process tree (detects spawns/exits)
            if app_profile.pids:
                root_pid = app_profile.pids[0]

                # Collect detailed metrics for entire process tree
                current_tree = self._collect_process_tree_metrics(root_pid)

                # Update process tree statistics
                self._update_process_tree_stats(app_profile, current_tree)

                # Update stored process tree
                app_profile.process_tree = current_tree

                # Update PIDs list
                app_profile.pids = list(current_tree.keys())

            # Calculate current deltas
            delta = self.calculate_deltas(baseline, app_profile.pids)
            samples.append(delta)

            # Check if process is still running
            if app_profile.pids:
                try:
                    psutil.Process(app_profile.pids[0])
                except psutil.NoSuchProcess:
                    print(f"Application process terminated")
                    break

            time.sleep(sample_interval)

        # Get logical core count for normalization
        logical_cores = baseline.cpu_info.logical_threads
        app_profile.logical_cores = logical_cores

        # Calculate averages and peaks
        if samples:
            # Averages - normalize CPU percent to 0-100% scale
            raw_cpu_percent = sum(s.cpu_percent for s in samples) / len(samples)
            app_profile.cpu_percent_normalized = (
                raw_cpu_percent / logical_cores
                if logical_cores > 0
                else raw_cpu_percent
            )
            # Make cpu_percent same as normalized for consistency
            app_profile.cpu_percent = app_profile.cpu_percent_normalized
            app_profile.memory_mb = sum(s.memory_mb for s in samples) / len(samples)
            app_profile.memory_percent = sum(s.memory_percent for s in samples) / len(
                samples
            )
            app_profile.gpu_compute_percent = sum(
                s.gpu_compute_percent for s in samples
            ) / len(samples)
            app_profile.gpu_memory_mb = sum(s.gpu_memory_mb for s in samples) / len(
                samples
            )
            app_profile.gpu_memory_percent = sum(
                s.gpu_memory_percent for s in samples
            ) / len(samples)
            app_profile.npu_percent = sum(s.npu_percent for s in samples) / len(samples)
            app_profile.disk_read_mb_per_sec = sum(
                s.disk_read_mb_per_sec for s in samples
            ) / len(samples)
            app_profile.disk_write_mb_per_sec = sum(
                s.disk_write_mb_per_sec for s in samples
            ) / len(samples)
            app_profile.disk_io_percent = sum(s.disk_io_percent for s in samples) / len(
                samples
            )

            # Peaks (maximum values) - normalize CPU percent to 0-100% scale
            raw_peak_cpu = max(s.cpu_percent for s in samples)
            app_profile.peak_cpu_percent_normalized = (
                raw_peak_cpu / logical_cores if logical_cores > 0 else raw_peak_cpu
            )
            # Make peak_cpu_percent same as normalized for consistency
            app_profile.peak_cpu_percent = app_profile.peak_cpu_percent_normalized
            app_profile.peak_memory_mb = max(s.memory_mb for s in samples)
            app_profile.peak_gpu_compute_percent = max(
                s.gpu_compute_percent for s in samples
            )
            app_profile.peak_gpu_memory_mb = max(s.gpu_memory_mb for s in samples)
            app_profile.peak_npu_percent = max(s.npu_percent for s in samples)
            app_profile.peak_disk_read_mb_per_sec = max(
                s.disk_read_mb_per_sec for s in samples
            )
            app_profile.peak_disk_write_mb_per_sec = max(
                s.disk_write_mb_per_sec for s in samples
            )
            app_profile.peak_disk_io_percent = max(s.disk_io_percent for s in samples)

        app_profile.duration = time.time() - app_profile.start_time
        app_profile.status = "completed"

        print(f"\nMonitoring completed:")
        print(f"  Process Tree Summary:")
        print(f"    Root PID: {app_profile.root_pid}")
        print(f"    Final process count: {len(app_profile.process_tree)}")
        print(f"    Peak process count: {app_profile.max_process_count}")
        print(f"    Total spawns: {app_profile.process_spawns}")
        print(f"    Total exits: {app_profile.process_exits}")
        print(f"")
        print(f"  Resource Utilization:")
        print(
            f"    CPU ({logical_cores} cores): {app_profile.cpu_percent_normalized:.1f}% (avg), {app_profile.peak_cpu_percent_normalized:.1f}% (peak)"
        )
        print(
            f"    [Total: {app_profile.cpu_percent:.1f}% avg, {app_profile.peak_cpu_percent:.1f}% peak across all cores]"
        )
        print(
            f"  Memory: {app_profile.memory_mb:.0f} MB (avg), {app_profile.peak_memory_mb:.0f} MB (peak)"
        )
        print(
            f"  GPU Compute: {app_profile.gpu_compute_percent:.1f}% (avg), {app_profile.peak_gpu_compute_percent:.1f}% (peak)"
        )
        print(
            f"  GPU Memory: {app_profile.gpu_memory_mb:.0f} MB (avg), {app_profile.peak_gpu_memory_mb:.0f} MB (peak)"
        )
        print(
            f"  NPU: {app_profile.npu_percent:.1f}% (avg), {app_profile.peak_npu_percent:.1f}% (peak)"
        )
        print(
            f"  Disk I/O: {app_profile.disk_read_mb_per_sec:.1f}/{app_profile.disk_write_mb_per_sec:.1f} MB/s (avg read/write)"
        )
        print(
            f"    [Peak: {app_profile.peak_disk_read_mb_per_sec:.1f}/{app_profile.peak_disk_write_mb_per_sec:.1f} MB/s]"
        )

        # Print process tree details
        self._print_process_tree(app_profile)

        # NPU diagnostics - warn if NPU appears inactive
        if app_profile.npu_percent == 0.0 and self.npu_monitor.is_available():
            npu_state = self.npu_monitor.get_power_state()
            npu_freq = self.npu_monitor.get_frequency()
            print(f"  NPU Status: power={npu_state}, freq={npu_freq}MHz")
            if npu_state == "suspended" or npu_freq == 0:
                print(
                    f"  ⚠ NPU appears inactive (power state: {npu_state}, freq: {npu_freq}MHz)"
                )
                print(
                    f"     This may indicate: 1) App not using NPU, 2) NPU driver issue, 3) OpenVINO using CPU fallback"
                )

        return app_profile

    def _print_process_tree(self, app_profile: ApplicationProfile) -> None:
        """Print hierarchical process tree with resource usage"""
        if not app_profile.process_tree or len(app_profile.process_tree) <= 1:
            return

        # Sort processes by CPU usage
        sorted_procs = sorted(
            app_profile.process_tree.values(),
            key=lambda p: p.cpu_percent,
            reverse=True,
        )

        # Show top processes (limit to 5 for simplicity)
        print(f"\n  Process Tree ({len(app_profile.process_tree)} processes):")
        for i, proc in enumerate(sorted_procs[:5]):
            print(
                f"    {proc.pid:<8} {proc.name:<20} {proc.cpu_percent:>6.1f}%  {proc.memory_mb:>7.1f} MB"
            )

        if len(sorted_procs) > 5:
            print(f"    ... and {len(sorted_procs) - 5} more processes")

    def calculate_deltas(
        self, baseline: PlatformBaseline, app_pids: List[int]
    ) -> ApplicationProfile:
        """
        Calculate resource attribution for application

        Method:
        - CPU: Sum CPU% of all app PIDs
        - Memory: Sum RSS of all app PIDs
        - GPU: Total GPU% - baseline GPU%
        - NPU: Total NPU% - baseline NPU%
        """
        delta = ApplicationProfile(app_id="temp", name="temp", command="")

        # CPU - sum of all PIDs
        cpu_percents = self.cpu_monitor.get_per_process_utilization(app_pids)
        delta.cpu_percent = sum(cpu_percents.values())

        # Memory - sum of all PIDs
        mem_mb, mem_percent = self.memory_monitor.get_process_tree_memory(app_pids)
        delta.memory_mb = mem_mb
        delta.memory_percent = mem_percent

        # GPU - delta from baseline
        gpu_utils = self.gpu_monitor.get_all_utilizations()
        if gpu_utils and baseline.gpu_devices:
            current_gpu = gpu_utils[0]
            baseline_gpu = baseline.gpu_devices[0]

            delta.gpu_compute_percent = max(
                0.0, current_gpu.compute_percent - baseline_gpu.compute_percent
            )
            delta.gpu_memory_mb = max(
                0.0, current_gpu.memory_used_mb - baseline_gpu.memory_used_mb
            )
            delta.gpu_memory_percent = max(
                0.0, current_gpu.memory_percent - baseline_gpu.memory_percent
            )

        # NPU - delta from baseline
        current_npu = self.npu_monitor.get_utilization(sampling_period=0.1)
        delta.npu_percent = max(0.0, current_npu - baseline.npu_percent)

        # Disk I/O - delta from baseline
        current_read, current_write = self.disk_monitor.get_system_io_rate()
        delta.disk_read_mb_per_sec = max(
            0.0, current_read - baseline.disk_read_mb_per_sec
        )
        delta.disk_write_mb_per_sec = max(
            0.0, current_write - baseline.disk_write_mb_per_sec
        )
        # Calculate I/O percentage (assuming 500 MB/s as 100% - typical SSD)
        total_io = delta.disk_read_mb_per_sec + delta.disk_write_mb_per_sec
        delta.disk_io_percent = min(100.0, (total_io / 500.0) * 100.0)

        # Transfer baseline warnings if any
        if hasattr(self, "_baseline_warnings"):
            delta.baseline_warnings = self._baseline_warnings.copy()

        # Debug: Print baseline comparison (first sample only)
        if not hasattr(self, "_baseline_printed"):
            # Get current system CPU for comparison
            current_system_cpu = self.cpu_monitor.get_overall_utilization(interval=0.1)

            print(f"\n[Baseline Debug]")
            print(
                f"  CPU: baseline={baseline.cpu_percent:.1f}%, system_current={current_system_cpu:.1f}%, app_pids={delta.cpu_percent:.1f}%"
            )
            print(
                f"  NPU: baseline={baseline.npu_percent:.1f}%, current={current_npu:.1f}%, delta={delta.npu_percent:.1f}%"
            )
            if gpu_utils and baseline.gpu_devices:
                print(
                    f"  GPU: baseline={baseline_gpu.compute_percent:.1f}%, current={current_gpu.compute_percent:.1f}%, delta={delta.gpu_compute_percent:.1f}%"
                )
            self._baseline_printed = True

        return delta

    def stop_application(self, session_id: str, app_name: str):
        """Stop a running application"""
        app_id = f"{session_id}_{app_name}"

        if app_id in self._subprocess_handles:
            process = self._subprocess_handles[app_id]
            try:
                # Kill process group
                os.killpg(os.getpgid(process.pid), signal.SIGTERM)
                process.wait(timeout=5)
            except:
                try:
                    os.killpg(os.getpgid(process.pid), signal.SIGKILL)
                except:
                    pass

            del self._subprocess_handles[app_id]
            print(f"Application '{app_name}' stopped")

    def prepare_profile_data(
        self,
        session: ProfilingSession,
        profile: ApplicationProfile,
        baseline: PlatformBaseline,
        analysis: "BottleneckAnalysis",
    ) -> Dict[str, Any]:
        """Prepare profiling results as dictionary for JSON serialization

        This method is useful for FastAPI endpoints that need to return JSON directly.

        Args:
            session: Profiling session object
            profile: Application profile with metrics
            baseline: Platform baseline measurements
            analysis: Bottleneck analysis results

        Returns:
            Dictionary containing all profiling data, ready for JSON serialization

        Example (FastAPI):
            @app.get("/profile")
            async def get_profile():
                orchestrator = ProfilingOrchestrator()
                # ... run profiling ...
                data = orchestrator.prepare_profile_data(session, profile, baseline, analysis)
                return data  # FastAPI auto-converts to JSON response
        """
        # Convert profile to dictionary
        app_data = asdict(profile)

        # If GPU compute is 0.0%, ensure GPU metrics are 0.0 (GPU not in use)
        if (
            profile.gpu_compute_percent == 0.0
            and profile.peak_gpu_compute_percent == 0.0
        ):
            app_data["gpu_compute_percent"] = 0.0
            app_data["gpu_memory_mb"] = 0.0
            app_data["gpu_memory_percent"] = 0.0
            app_data["peak_gpu_compute_percent"] = 0.0
            app_data["peak_gpu_memory_mb"] = 0.0

        return {
            "session_id": session.session_id,
            "timestamp": datetime.fromtimestamp(time.time()).isoformat(),
            "baseline": asdict(baseline),
            "application": app_data,
            "bottleneck_analysis": asdict(analysis),
        }

    def save_profile_to_json(
        self,
        filepath: str,
        session: ProfilingSession,
        profile: ApplicationProfile,
        baseline: PlatformBaseline,
        analysis: "BottleneckAnalysis",
    ) -> None:
        """Save profiling results to JSON file

        Args:
            filepath: Output JSON file path
            session: Profiling session object
            profile: Application profile with metrics
            baseline: Platform baseline measurements
            analysis: Bottleneck analysis results
        """
        output_data = self.prepare_profile_data(session, profile, baseline, analysis)

        with open(filepath, "w") as f:
            json.dump(output_data, f, indent=2)

        print(f"\n✓ Profile saved to: {filepath}")

# ============================================================================
# CLI Interface
# ============================================================================

def print_banner():
    """Print tool banner"""
    print("╔" + "═" * 78 + "╗")
    print("║" + " " * 20 + "PLATFORM PROFILER v1.0" + " " * 36 + "║")
    print("║" + " " * 15 + "CPU/GPU/NPU/Memory Utilization Analysis" + " " * 24 + "║")
    print("╚" + "═" * 78 + "╝")
    print()

def run_profile(args):
    """Run profiling command"""
    # Validate arguments
    if not args.app and not args.pid:
        print("ERROR: Either --app or --pid must be specified.")
        print("\nOptions:")
        print("  --app 'command'  : Launch and profile a new application")
        print("  --pid 12345      : Attach to and profile an existing process")
        print(
            "\nExample 1: python3 platform_profiler.py profile --app 'python inference.py' --name 'my_app'"
        )
        print(
            "Example 2: python3 platform_profiler.py profile --pid 12345 --name 'my_app'"
        )
        return

    if args.app and args.pid:
        print("ERROR: Cannot specify both --app and --pid. Choose one.")
        return

    print_banner()
    print(f"Profiling Application: {args.name}")
    print("=" * 80)

    orchestrator = ProfilingOrchestrator()

    # Create session
    session = orchestrator.create_session()

    # Establish baseline
    print(f"\nEstablishing baseline ({args.baseline}s)...")
    baseline = orchestrator.establish_baseline(
        session.session_id, duration=args.baseline
    )

    # Profile application
    if args.pid:
        print(f"\nAttaching to PID: {args.pid}")
        app = orchestrator.attach_to_process(session.session_id, args.name, args.pid)
    else:
        # SECURITY: Validate command string to prevent command injection (taint barrier)
        try:
            validated_command = validate_command_string(args.app)
        except ValueError as e:
            print(f"\nERROR: Invalid command: {e}")
            return

        # Validate file paths in the command (universal approach)
        command_parts = validated_command.strip().split()

        # Check each argument that looks like a file path
        for arg in command_parts:
            # Skip flags/options (start with -)
            if arg.startswith("-"):
                continue

            # Check if argument looks like a file path (has extension or path separators)
            if "/" in arg or "\\" in arg or ("." in arg and not arg.startswith(".")):
                # SECURITY: Validate file path to prevent path manipulation (taint barrier)
                try:
                    file_path = validate_file_path(arg)
                except ValueError as e:
                    print(f"\nERROR: Invalid file path '{arg}': {e}")
                    return

                # Validate file exists
                if not file_path.exists():
                    print(f"\nERROR: File not found: {arg}")
                    print(f"  Resolved path: {file_path}")
                    print(f"  Current directory: {Path.cwd()}")
                    print(f"\nPlease check the file path and try again.")
                    return

                if not file_path.is_file():
                    print(f"\nERROR: Path is not a file: {arg}")
                    print(f"  Resolved path: {file_path}")
                    print(f"\nPlease provide a valid file path.")
                    return

                print(f"\n✓ File validation passed: {file_path}")

        print(f"\nLaunching: {validated_command}")
        # Pass validated command (taint removed by validate_command_string)
        app = orchestrator.launch_application(
            session.session_id, args.name, validated_command
        )

    if app.status == "running":
        profile = orchestrator.monitor_application(
            session.session_id, args.name, duration=args.duration
        )

        # Only stop if we launched it (not if we attached to existing PID)
        if args.app:
            orchestrator.stop_application(session.session_id, args.name)
        else:
            print(f"Note: Process {args.pid} is still running (attached mode)")

        # Show results
        print("\n" + "=" * 80)
        print("PROFILING RESULTS")
        print("=" * 80)
        print(f"\nApplication: {args.name}")
        print(f"Duration: {profile.duration:.1f}s")
        print(f"\n{'Resource':<20} {'Average':<20} {'Peak':<20}")
        print("-" * 80)
        print(
            f"{'CPU (normalized)':<20} {profile.cpu_percent_normalized:>6.1f}% {profile.peak_cpu_percent_normalized:>20.1f}%"
        )
        print(
            f"{'  ({profile.logical_cores} cores)':<20} [{profile.cpu_percent:>5.1f}% total] [{profile.peak_cpu_percent:>14.1f}% total]"
        )
        print(
            f"{'Memory':<20} {profile.memory_mb:>10.0f} MB {profile.peak_memory_mb:>15.0f} MB"
        )
        print(
            f"{'GPU Compute':<20} {profile.gpu_compute_percent:>6.1f}% {profile.peak_gpu_compute_percent:>20.1f}%"
        )
        print(
            f"{'GPU Memory':<20} {profile.gpu_memory_mb:>10.0f} MB {profile.peak_gpu_memory_mb:>15.0f} MB"
        )
        print(
            f"{'NPU':<20} {profile.npu_percent:>6.1f}% {profile.peak_npu_percent:>20.1f}%"
        )
        print(
            f"{'Disk Read':<20} {profile.disk_read_mb_per_sec:>8.1f} MB/s {profile.peak_disk_read_mb_per_sec:>16.1f} MB/s"
        )
        print(
            f"{'Disk Write':<20} {profile.disk_write_mb_per_sec:>8.1f} MB/s {profile.peak_disk_write_mb_per_sec:>16.1f} MB/s"
        )

        # Perform bottleneck analysis
        analyzer = BottleneckAnalyzer()
        analysis = analyzer.analyze(profile, baseline)

        # Display bottleneck analysis
        analyzer.print_analysis(analysis)

        # Save to file if requested
        if args.output:
            orchestrator.save_profile_to_json(
                args.output, session, profile, baseline, analysis
            )

    else:
        print(f"\n✗ Failed to launch application: {app.error_message}")

def run_status(args):
    """Show current platform status"""
    orchestrator = ProfilingOrchestrator()

    print("╔" + "═" * 78 + "╗")
    print("║" + " " * 28 + "PLATFORM STATUS" + " " * 35 + "║")
    print("╚" + "═" * 78 + "╝")
    print()

    # CPU
    cpu_info = orchestrator.cpu_monitor.get_cpu_info()
    cpu_util = orchestrator.cpu_monitor.get_overall_utilization()
    print(f"CPU: {cpu_info.model}")
    print(
        f"  Cores: {cpu_info.physical_cores} physical, {cpu_info.logical_threads} logical"
    )
    print(f"  Utilization: {cpu_util:.1f}%")
    print()

    # Memory
    mem = orchestrator.memory_monitor.get_system_memory()
    print(f"Memory:")
    print(f"  Total: {mem.total_mb:,.0f} MB ({mem.total_mb/1024:.1f} GB)")
    print(f"  Used: {mem.used_mb:,.0f} MB ({mem.percent:.1f}%)")
    print(f"  Available: {mem.available_mb:,.0f} MB ({100-mem.percent:.1f}%)")
    print()

    # GPU
    gpus = orchestrator.gpu_monitor.get_all_utilizations()
    if gpus:
        for gpu in gpus:
            print(f"GPU {gpu.device_id}: {gpu.name}")
            print(f"  Compute: {gpu.compute_percent:.1f}%")
            print(
                f"  Memory: {gpu.memory_used_mb:.0f} MB / {gpu.memory_total_mb:.0f} MB ({gpu.memory_percent:.1f}%)"
            )
            print()
    else:
        print("GPU: Not available")
        print()

    # NPU
    if orchestrator.npu_monitor.is_available():
        npu_util = orchestrator.npu_monitor.get_utilization()
        print(f"NPU: Available")
        print(f"  Utilization: {npu_util:.1f}%")
    else:
        print("NPU: Not available")
    print()

# ============================================================================
# FastAPI Interface
# ============================================================================

if FASTAPI_AVAILABLE:
    app = FastAPI(title="Platform Profiler API", version="1.0")

    app.add_middleware(
        CORSMiddleware,
        allow_origins=["http://127.0.0.1:8080", "http://localhost:8080"],
        allow_credentials=True,
        allow_methods=["*"],
        allow_headers=["*"],
    )

    # Global orchestrator instance for API
    api_orchestrator = ProfilingOrchestrator()
    api_session = None
    api_baseline = None

    @app.get("/api/status")
    async def get_platform_status():
        """Get current platform status (CPU, GPU, NPU, Memory)"""
        print("=== /api/status endpoint called ===")
        try:
            # Call orchestrator to get platform status
            cpu_info = api_orchestrator.cpu_monitor.get_cpu_info()
            cpu_util = api_orchestrator.cpu_monitor.get_overall_utilization()
            mem = api_orchestrator.memory_monitor.get_system_memory()
            gpus = api_orchestrator.gpu_monitor.get_all_utilizations()
            npu_util = api_orchestrator.npu_monitor.get_utilization()
            npu_available = api_orchestrator.npu_monitor.is_available()

            response = {
                "cpu": {
                    "model": cpu_info.model,
                    "physical_cores": cpu_info.physical_cores,
                    "logical_threads": cpu_info.logical_threads,
                    "frequency_mhz": round(cpu_info.frequency_mhz, 2),
                    "utilization_percent": round(cpu_util, 2),
                },
                "memory": {
                    "total_mb": round(mem.total_mb, 2),
                    "used_mb": round(mem.used_mb, 2),
                    "available_mb": round(mem.available_mb, 2),
                    "percent": round(mem.percent, 2),
                },
                "npu": {
                    "available": npu_available,
                    "utilization_percent": round(npu_util, 2) if npu_available else 0.0,
                },
            }

            # Add GPU metrics if available
            if gpus:
                if len(gpus) == 1:
                    gpu = gpus[0]
                    response["gpu"] = {
                        "device_id": gpu.device_id,
                        "name": gpu.name,
                        "compute_percent": round(gpu.compute_percent, 2),
                        "memory_used_mb": round(gpu.memory_used_mb, 2),
                        "memory_total_mb": round(gpu.memory_total_mb, 2),
                        "memory_percent": round(gpu.memory_percent, 2),
                    }
                else:
                    response["gpus"] = []
                    for gpu in gpus:
                        gpu_data = {
                            "device_id": gpu.device_id,
                            "name": gpu.name,
                            "compute_percent": round(gpu.compute_percent, 2),
                            "memory_used_mb": round(gpu.memory_used_mb, 2),
                            "memory_total_mb": round(gpu.memory_total_mb, 2),
                            "memory_percent": round(gpu.memory_percent, 2),
                        }
                        response["gpus"].append(gpu_data)
            else:
                response["gpu"] = None

            return response

        except Exception as e:
            print(f"Error in get_platform_status: {e}")
            raise HTTPException(status_code=500, detail=str(e))

    @app.post("/api/profile-start")
    async def start_profiling(pid: Optional[int] = None, app_name: str = "api_app"):
        """Start profiling session (attach to PID or prepare for monitoring)"""
        global api_session, api_baseline
        print(
            f"=== /api/profile/start endpoint called (pid={pid}, name={app_name}) ==="
        )

        try:
            # Create new session
            api_session = api_orchestrator.create_session()

            # Establish baseline
            api_baseline = api_orchestrator.establish_baseline(
                api_session.session_id, duration=3.0
            )

            response = {
                "session_id": api_session.session_id,
                "status": "baseline_established",
                "baseline": {
                    "cpu_percent": round(api_baseline.cpu_percent, 2),
                    "memory_used_mb": round(api_baseline.memory_used_mb, 2),
                    "npu_percent": round(api_baseline.npu_percent, 2),
                },
            }

            # If PID provided, attach to it
            if pid:
                app_profile = api_orchestrator.attach_to_process(
                    api_session.session_id, app_name, pid
                )
                try:
                    proc = psutil.Process(pid)
                    actual_process_name = proc.name()
                    actual_cmdline = " ".join(proc.cmdline())
                except:
                    actual_process_name = "unknown"
                    actual_cmdline = ""

                response["status"] = "profiling_started"
                response["app_name"] = app_name
                response["pid"] = pid
                response["process_name"] = actual_process_name
                response["cmdline"] = actual_cmdline

            return response

        except Exception as e:
            print(f"Error in start_profiling: {e}")
            raise HTTPException(status_code=500, detail=str(e))

    @app.get("/api/profile-monitor")
    async def monitor_profile(app_name: str = "api_app", duration: float = 30.0):
        """Monitor application and return profiling results"""
        global api_session, api_baseline
        print(
            f"=== /api/profile/monitor endpoint called (name={app_name}, duration={duration}) ==="
        )

        if not api_session or not api_baseline:
            raise HTTPException(
                status_code=400,
                detail="No active session. Call /api/profile/start first",
            )

        try:
            # Monitor the application
            profile = api_orchestrator.monitor_application(
                api_session.session_id, app_name, duration=duration
            )

            # Perform bottleneck analysis
            analyzer = BottleneckAnalyzer()
            analysis = analyzer.analyze(profile, api_baseline)

            # Prepare response data using prepare_profile_data
            data = api_orchestrator.prepare_profile_data(
                api_session, profile, api_baseline, analysis
            )

            return data

        except Exception as e:
            print(f"Error in monitor_profile: {e}")
            raise HTTPException(status_code=500, detail=str(e))

    @app.get("/api/profile-quick")
    async def quick_profile(pid: int, duration: float = 30.0):
        """Quick profile: start, monitor, and return results in one call"""
        print(
            f"=== /api/profile/quick endpoint called (pid={pid}, duration={duration}) ==="
        )

        try:
            # Create session and establish baseline
            session = api_orchestrator.create_session()
            baseline = api_orchestrator.establish_baseline(
                session.session_id, duration=3.0
            )

            # Attach to process
            app_name = f"pid_{pid}"
            profile = api_orchestrator.attach_to_process(
                session.session_id, app_name, pid
            )

            if profile.status == "failed":
                raise HTTPException(status_code=400, detail=profile.error_message)

            # Monitor
            profile = api_orchestrator.monitor_application(
                session.session_id, app_name, duration=duration
            )

            # Analyze
            analyzer = BottleneckAnalyzer()
            analysis = analyzer.analyze(profile, baseline)

            # Return data using prepare_profile_data
            data = api_orchestrator.prepare_profile_data(
                session, profile, baseline, analysis
            )
            return data

        except HTTPException:
            raise
        except Exception as e:
            print(f"Error in quick_profile: {e}")
            raise HTTPException(status_code=500, detail=str(e))

    @app.post("/api/profile-batch")
    async def batch_profile(request: dict):
        """Batch profile with user input param"""
        print("=== PYTHON BACKEND DEBUG ===")
        print(f"Request received: {request}")
        print(f"Request type: {type(request)}")

        pid = request.get("pid")
        # Command to launch (e.g., "python3 /path/to/script.py")
        app_path = request.get("app_path")
        app_name = request.get("app_name", "unknown")
        selection_type = request.get("selection_type", "pid")
        duration = request.get("duration", 30.0)

        print(f"Extracted values:", flush=True)
        print(f"  selection_type: '{selection_type}'")
        print(f"  pid: {pid}")
        print(f"  app_path: '{app_path}'")
        print(f"  app_name: '{app_name}'")
        print(f"  duration: {duration}")

        # Use existing PID
        if selection_type == "pid":
            if not pid:
                raise HTTPException(
                    status_code=400, detail="PID is required for 'pid' mode"
                )

            print(f"[PID MODE] Attaching to PID: {pid}")

            try:
                # Create session and establish baseline
                session = api_orchestrator.create_session()
                baseline = api_orchestrator.establish_baseline(
                    session.session_id, duration=3.0
                )

                # Attach to existing process
                if app_name == "unknown":
                    app_name = f"pid_{pid}"

                profile = api_orchestrator.attach_to_process(
                    session.session_id, app_name, pid
                )

                if profile.status == "failed":
                    raise HTTPException(status_code=400, detail=profile.error_message)

                # Monitor
                profile = api_orchestrator.monitor_application(
                    session.session_id, app_name, duration=duration
                )

                # Analyze
                analyzer = BottleneckAnalyzer()
                analysis = analyzer.analyze(profile, baseline)

                # Return data
                data = api_orchestrator.prepare_profile_data(
                    session, profile, baseline, analysis
                )
                return data

            except HTTPException:
                raise
            except Exception as e:
                print(f"[PID MODE] Error: {e}")
                raise HTTPException(status_code=500, detail=str(e))

        # Launch new application by path
        elif selection_type == "appname":
            if not app_path or app_path.strip() == "":
                raise HTTPException(
                    status_code=400,
                    detail="Application path/command is required when using 'appname' selection type",
                )

            print(f"[APPPATH MODE] Launching application: {app_path}")

            try:
                # Validate file paths in the command (universal approach)
                command_parts = app_path.strip().split()
                validated_file = None

                # Check each argument that looks like a file path
                for arg in command_parts:
                    # Skip flags/options (start with -)
                    if arg.startswith("-"):
                        continue

                    # Check if argument looks like a file path (has extension or path separators)
                    if (
                        "/" in arg
                        or "\\" in arg
                        or ("." in arg and not arg.startswith("."))
                    ):
                        file_path = Path(arg)

                        # Handle relative paths - make them absolute from current working directory
                        if not file_path.is_absolute():
                            file_path = Path.cwd() / file_path

                        # Validate file exists
                        if not file_path.exists():
                            error_msg = (
                                f"File not found: {arg} (resolved to: {file_path})"
                            )
                            print(f"[APPPATH MODE] ERROR: {error_msg}")
                            raise HTTPException(
                                status_code=400,
                                detail={
                                    "status": "failed",
                                    "error": error_msg,
                                    "message": "Please check the file path and try again.",
                                    "current_dir": str(Path.cwd()),
                                },
                            )

                        if not file_path.is_file():
                            error_msg = f"Path is not a file: {arg}"
                            print(f"[APPPATH MODE] ERROR: {error_msg}")
                            raise HTTPException(
                                status_code=400,
                                detail={
                                    "status": "failed",
                                    "error": error_msg,
                                    "message": "Please provide a valid file path.",
                                },
                            )

                        print(f"[APPPATH MODE] File validation passed: {file_path}")
                        if not validated_file:
                            validated_file = file_path

                # Create session and establish baseline
                session = api_orchestrator.create_session()
                baseline = api_orchestrator.establish_baseline(
                    session.session_id, duration=3.0
                )

                # Generate app name from command if not provided
                if app_name == "unknown":
                    # Extract name from validated file or first command part
                    if validated_file:
                        app_name = validated_file.stem
                    elif len(command_parts) > 0:
                        app_name = Path(command_parts[0]).stem
                    else:
                        app_name = "unknown_app"

                print(f"[APPPATH MODE] App name: {app_name}")

                # Launch the application
                profile = api_orchestrator.launch_application(
                    session.session_id, app_name, app_path.strip()
                )

                if profile.status == "failed":
                    raise HTTPException(
                        status_code=400,
                        detail={
                            "status": "failed",
                            "error": profile.error_message,
                            "message": "Application failed to launch or exited immediately.",
                        },
                    )

                print(
                    f"[APPPATH MODE] Application launched with PID: {profile.root_pid}"
                )

                # Monitor the launched application
                profile = api_orchestrator.monitor_application(
                    session.session_id, app_name, duration=duration
                )

                print(f"[APPPATH MODE] Monitoring complete")
                print(f"  process_name: '{profile.process_name}'")
                print(f"  command: '{profile.command}'")

                # Stop the application after profiling
                api_orchestrator.stop_application(session.session_id, app_name)
                print(f"[APPPATH MODE] Application stopped")

                # Analyze
                analyzer = BottleneckAnalyzer()
                analysis = analyzer.analyze(profile, baseline)

                # Return data
                data = api_orchestrator.prepare_profile_data(
                    session, profile, baseline, analysis
                )

                print(f"[APPPATH MODE] Profiling complete!")
                return data

            except HTTPException:
                raise
            except Exception as e:
                print(f"[APPPATH MODE] Error: {e}")
                import traceback

                traceback.print_exc()
                raise HTTPException(status_code=500, detail=str(e))

        else:
            raise HTTPException(
                status_code=400,
                detail=f"Invalid selection_type: {selection_type}. Must be 'pid' or 'appname'",
            )

def parse_api_arguments():
    """Parse arguments for API mode"""
    import argparse

    parser = argparse.ArgumentParser(description="Platform Profiler API Service")
    parser.add_argument(
        "--port",
        type=int,
        default=6240,
        help="Port to run the FastAPI server on (default: 6240)",
    )
    parser.add_argument(
        "--host",
        type=str,
        default="127.0.0.1",
        help="Host to bind service to (default: 127.0.0.1)",
    )
    parser.add_argument(
        "--reload", action="store_true", help="Enable auto reload for development"
    )

    return parser.parse_args()

def main():
    """Main CLI entry point"""
    import argparse

    parser = argparse.ArgumentParser(
        description="Platform Profiler - Application Resource Utilization Analysis",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
EXAMPLES:
  # Check platform status
  %(prog)s status

  # Profile your application
  %(prog)s profile --app "python inference.py" --name "my_app" --duration 30

For more info: https://github.com/intel/...
        """,
    )

    subparsers = parser.add_subparsers(dest="command", help="Command to execute")

    # Status command
    status_parser = subparsers.add_parser("status", help="Show current platform status")

    # Profile command
    profile_parser = subparsers.add_parser("profile", help="Profile application")
    profile_parser.add_argument(
        "--app", type=str, help="Application command to execute"
    )
    profile_parser.add_argument(
        "--pid", type=int, help="PID of running process to profile"
    )
    profile_parser.add_argument(
        "--name", type=str, default="app", help="Application name (default: app)"
    )
    profile_parser.add_argument(
        "--baseline",
        type=float,
        default=5.0,
        help="Baseline duration in seconds (default: 5)",
    )
    profile_parser.add_argument(
        "--duration",
        type=float,
        default=30.0,
        help="Profile duration in seconds (default: 30)",
    )
    profile_parser.add_argument(
        "--output", type=str, help="Output JSON file (optional)"
    )

    args = parser.parse_args()

    if not args.command:
        print_banner()
        parser.print_help()
        return

    # SECURITY: Validate command is one of the allowed subcommands (taint barrier)
    # This acts as a taint barrier for static analysis tools
    allowed_commands = ["profile", "status"]
    if args.command not in allowed_commands:
        print(f"ERROR: Invalid command '{args.command}'")
        return

    # Execute validated command
    # Note: Additional validation occurs inside each function for command-specific fields
    if args.command == "profile":
        # SECURITY: Break taint chain by validating individual fields
        # Extract and validate each field to prevent taint propagation
        validated_args = type(
            "obj",
            (object,),
            {
                "command": str(args.command),  # Already validated via whitelist
                "app": args.app if args.app is None else str(args.app),
                "pid": args.pid if args.pid is None else int(args.pid),
                "name": args.name if args.name is None else str(args.name),
                "baseline": float(args.baseline) if hasattr(args, "baseline") else 5.0,
                "duration": float(args.duration) if hasattr(args, "duration") else 30.0,
                "output": (
                    args.output
                    if not hasattr(args, "output") or args.output is None
                    else str(args.output)
                ),
            },
        )()
        run_profile(validated_args)
    elif args.command == "status":
        run_status(args)

if __name__ == "__main__":
    # Check if running in API mode
    if len(sys.argv) > 1 and sys.argv[1] == "api":
        if not FASTAPI_AVAILABLE:
            print("ERROR: FastAPI not available!", file=sys.stderr)
            print("Install with: pip install fastapi uvicorn", file=sys.stderr)
            sys.exit(1)

        # Remove 'api' from argv so parse_api_arguments works correctly
        # sys.argv.pop(1)
        sys.argv = [sys.argv[0]] + sys.argv[
            2:
        ]  # Keep script name, remove 'api', keep rest
        args = parse_api_arguments()

        print(f"Starting Platform Profiler API on {args.host}:{args.port}")
        print(f"API endpoints:")
        print(f"  GET  http://{args.host}:{args.port}/api/status")
        print(f"  POST http://{args.host}:{args.port}/api/profile-start")
        print(f"  GET  http://{args.host}:{args.port}/api/profile-monitor")
        print(
            f"  GET  http://{args.host}:{args.port}/api/profile/quick?pid=<PID>&duration=<seconds>"
        )
        print(f"\nDocs: http://{args.host}:{args.port}/docs")

        uvicorn.run(app, host=args.host, port=args.port, reload=args.reload)
    else:
        # Run CLI mode
        main()