"""Bounded Linux/Docker memory observation; GET only, no service changes. Run alongside the named acceptance workload. Output is evidence, never an automatic declaration that a 2 GB deployment has passed its load acceptance. Requires Python 3 and read access to /proc and the local Docker socket. """ import argparse import http.client import json import os import re import socket import stat import threading import time MAX_RESPONSE = 4 * 1024 * 1024 MAX_FILE_BYTES = 65536 MAX_FILE_LINES = 256 CGROUP_ROOT = '/sys/fs/cgroup' CGROUP_FILES = ('memory.current', 'memory.peak', 'memory.events', 'memory.swap.current', 'memory.swap.peak', 'memory.swap.events', 'memory.max', 'memory.swap.max', 'cpu.stat') class CgroupUnavailable(ValueError): """Fixed reason codes only; never include private paths or raw content.""" class ObservationDeadlineExceeded(RuntimeError): pass def check_deadline(deadline): if deadline is not None and time.monotonic() >= deadline: raise ObservationDeadlineExceeded('Observation time bound exceeded') def read_bounded(path, expected_device=None, deadline=None): # Fixed production paths only; no CLI path or executable is accepted. check_deadline(deadline) descriptor = os.open(path, os.O_RDONLY | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_NONBLOCK', 0)) with os.fdopen(descriptor, 'rb') as handle: identity = os.fstat(handle.fileno()) if not stat.S_ISREG(identity.st_mode) or (expected_device is not None and identity.st_dev != expected_device): raise ValueError('Non-regular observation') raw = handle.read(MAX_FILE_BYTES + 1) check_deadline(deadline) if len(raw) > MAX_FILE_BYTES: raise ValueError('Observation file too large') text = raw.decode('ascii') if len(text.splitlines()) > MAX_FILE_LINES: raise ValueError('Too many observation lines') return text def parse_counter(text): if not re.fullmatch(r'(?:0|[1-9][0-9]{0,19})\n?', text): raise ValueError('Invalid counter') value = int(text) if value > 2 ** 64 - 1: raise ValueError('Counter overflow') return value def parse_events(text, required): lines = text.splitlines() if not 1 <= len(lines) <= 32: raise ValueError('Invalid event count') result = {} for line in lines: fields = line.split(' ') if len(fields) != 2 or not re.fullmatch(r'[a-z_]{1,64}', fields[0]) or fields[0] in result: raise ValueError('Invalid event') result[fields[0]] = parse_counter(fields[1]) if not set(required).issubset(result): raise ValueError('Incomplete events') return result def parse_limit(text): # Preserve unlimited vs zero vs unavailable; never substitute host RAM. return 'max' if text in ('max', 'max\n') else parse_counter(text) def parse_cpu_throttling(text): required = ('nr_periods', 'nr_throttled', 'throttled_usec') parsed = parse_events(text, required) # Kernel extensions are validated but do not expand the output whitelist. return {key: parsed[key] for key in required} def parse_memory_pressure(text): rows = text.splitlines() if len(rows) != 2: raise ValueError('Incomplete memory pressure') result = {} for line in rows: match = re.fullmatch(r'(some|full) avg10=([0-9]{1,3}\.[0-9]{2}) avg60=([0-9]{1,3}\.[0-9]{2}) avg300=([0-9]{1,3}\.[0-9]{2}) total=((?:0|[1-9][0-9]{0,19}))', line) if not match or match[1] in result: raise ValueError('Invalid memory pressure') for value in match.group(2, 3, 4): whole, fraction = value.split('.') if int(whole) * 100 + int(fraction) > 10000: raise ValueError('Invalid memory pressure percentage') result[match[1]] = {'avg10Percent': match[2], 'avg60Percent': match[3], 'avg300Percent': match[4], 'totalMicros': parse_counter(match[5])} if set(result) != {'some', 'full'}: raise ValueError('Incomplete memory pressure') return result def configured_memory_swap(inspect): # Docker MemorySwap is TOTAL memory+swap; 0 means daemon default, -1 unlimited. host = inspect.get('HostConfig') value = host.get('MemorySwap') if isinstance(host, dict) else None if type(value) is not int or not -1 <= value <= 2 ** 63 - 1: return None return value def resource_limits_stable(before, after): values = [] for item in (before, after): host = item.get('HostConfig') memory = host.get('Memory') if isinstance(host, dict) else None swap = configured_memory_swap(item) if type(memory) is not int or not 0 <= memory <= 2 ** 63 - 1 or swap is None: return None values.append((memory, swap)) return values[0] == values[1] def collect_host(deadline=None): memory = parse_host(read_bounded('/proc/meminfo', deadline=deadline), read_bounded('/proc/vmstat', deadline=deadline)) try: pressure = parse_memory_pressure(read_bounded('/proc/pressure/memory', deadline=deadline)) memory['memoryPressure'] = {'status': 'available', 'metrics': pressure, 'reason': None} except (OSError, ValueError, UnicodeError): memory['memoryPressure'] = {'status': 'unavailable', 'metrics': None, 'reason': 'missing-unreadable-or-malformed'} return memory def container_identity(inspect, identifier): state = inspect['State'] if (inspect.get('Id') != identifier or not re.fullmatch(r'[a-f0-9]{64}', identifier) or type(state.get('Pid')) is not int or not 0 < state['Pid'] <= 2 ** 31 - 1 or state.get('Running') is not True or type(state.get('OOMKilled')) is not bool or not isinstance(state.get('StartedAt'), str) or not state['StartedAt'] or type(inspect.get('RestartCount')) is not int or inspect['RestartCount'] < 0 or not isinstance(inspect.get('Image'), str) or not inspect['Image']): raise ValueError('Unavailable container identity') return (identifier, inspect['Image'], state['StartedAt'], state['Pid'], inspect['RestartCount'], state['OOMKilled']) def cgroup_location(pid, identifier, deadline=None): check_deadline(deadline) mounts = read_bounded('/proc/self/mountinfo', deadline=deadline).splitlines() # Namespace-relative/subtree mounts are deliberately unsupported: their # root cannot authenticate a host container's complete cgroup membership. matches = [line.split(' - ', 1) for line in mounts if ' - ' in line] if not any(left.split()[3:5] == ['/', CGROUP_ROOT] and right.split()[0] == 'cgroup2' for left, right in matches): raise CgroupUnavailable('unsupported-or-unavailable-host-cgroup2-mount') membership = read_bounded('/proc/' + str(pid) + '/cgroup', deadline=deadline) lines = membership.splitlines() if len(lines) != 1 or not lines[0].startswith('0::/'): raise CgroupUnavailable('unsupported-v1-hybrid-or-malformed-membership') relative = lines[0][3:] parts = relative.split('/')[1:] if (not parts or any(not p or p in ('.', '..') or not re.fullmatch(r'[A-Za-z0-9_.:-]{1,128}', p) for p in parts) or parts[-1] not in (identifier, 'docker-' + identifier + '.scope')): raise ValueError('Unbound cgroup membership') root = os.path.realpath(CGROUP_ROOT) path = os.path.realpath(CGROUP_ROOT + relative) if root != CGROUP_ROOT or path != root + relative or not path.startswith(root + os.sep): raise ValueError('Cgroup path escaped') return path, membership def process_generation(pid, deadline=None): check_deadline(deadline) raw = read_bounded('/proc/' + str(pid) + '/stat', deadline=deadline) prefix, suffix = raw.rsplit(')', 1) if prefix.split(' ', 1)[0] != str(pid) or ' (' not in prefix: raise ValueError('Invalid process identity') fields = suffix.split() if len(fields) < 20: raise ValueError('Incomplete process identity') start = parse_counter(fields[19]) root = os.stat('/proc/' + str(pid) + '/root') return start, root.st_dev, root.st_ino def unavailable_cgroup(reason): return {'version': 2, 'status': 'unavailable', 'reason': reason, 'metrics': {name: None for name in CGROUP_FILES}, 'unavailable': {name: reason for name in CGROUP_FILES}} def collect_container(identifier, deadline=None): check_deadline(deadline) before = docker_get('/containers/' + identifier + '/json', deadline) stats = docker_get('/containers/' + identifier + '/stats?stream=false&one-shot=true', deadline) check_deadline(deadline) result = project_container(before, stats, identifier) cgroup = unavailable_cgroup('identity-unavailable') try: identity = container_identity(before, identifier) pid = identity[3] generation = process_generation(pid, deadline) path, membership = cgroup_location(pid, identifier, deadline) directory = os.stat(path) if directory.st_dev != os.stat(CGROUP_ROOT).st_dev: raise ValueError('Cgroup filesystem changed') metrics, missing = {}, {} for name in CGROUP_FILES: check_deadline(deadline) try: candidate = path + '/' + name if os.path.realpath(candidate) != candidate: raise ValueError('Symlinked metric') raw = read_bounded(candidate, directory.st_dev, deadline=deadline) check_deadline(deadline) if name == 'memory.events': metrics[name] = parse_events(raw, ('low', 'high', 'max', 'oom', 'oom_kill')) elif name == 'memory.swap.events': metrics[name] = parse_events(raw, ('high', 'max', 'fail')) elif name in ('memory.max', 'memory.swap.max'): metrics[name] = parse_limit(raw) elif name == 'cpu.stat': metrics[name] = parse_cpu_throttling(raw) else: metrics[name] = parse_counter(raw) except (OSError, ValueError, UnicodeError): metrics[name] = None missing[name] = 'missing-unreadable-or-malformed' check_deadline(deadline) after_path, after_membership = cgroup_location(pid, identifier, deadline) after_directory = os.stat(path) if (process_generation(pid, deadline) != generation or after_path != path or after_membership != membership or (directory.st_dev, directory.st_ino) != (after_directory.st_dev, after_directory.st_ino)): cgroup = unavailable_cgroup('process-or-cgroup-changed') else: cgroup = {'version': 2, 'status': 'partial' if missing else 'available', 'reason': 'some-metrics-unavailable' if missing else None, 'metrics': metrics, 'unavailable': missing, 'peakScope': 'cgroup lifetime or since external reset; not an isolated workload peak', 'provenance': 'fixed host cgroup2 mount + exact Docker ID membership + process/root generation'} except CgroupUnavailable as error: cgroup = unavailable_cgroup(str(error)) except (OSError, ValueError, KeyError, IndexError, UnicodeError): cgroup = unavailable_cgroup('identity-process-or-cgroup-unavailable-or-unsupported') # Even failed cgroup observations are bracketed by a fresh Docker inspect. check_deadline(deadline) after = docker_get('/containers/' + identifier + '/json', deadline) check_deadline(deadline) try: stable = container_identity(before, identifier) == container_identity(after, identifier) except (ValueError, KeyError, TypeError): stable = False if not stable: cgroup = unavailable_cgroup('container-identity-changed-or-unavailable') limits_stable = resource_limits_stable(before, after) if stable else None if stable and limits_stable is not True and cgroup['status'] != 'unavailable': for key in ('memory.max', 'memory.swap.max'): cgroup['metrics'][key] = None cgroup['unavailable'][key] = 'container-resource-limits-changed-or-unavailable' cgroup['status'] = 'partial' cgroup['reason'] = 'some-metrics-unavailable' result.update({'resourceLimitsStableDuringSample': limits_stable, 'pid': before['State'].get('Pid'), 'identityStableDuringSample': stable, 'runningAfter': after.get('State', {}).get('Running'), 'oomKilledAfter': after.get('State', {}).get('OOMKilled'), 'cgroupV2': cgroup}) return result def docker_get(path, deadline=None): deadline = time.monotonic() + 5 if deadline is None else deadline check_deadline(deadline) connection = http.client.HTTPConnection('localhost', timeout=min(5, deadline - time.monotonic())) owned_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) connection.sock = owned_socket # A socket timeout alone permits an endless body that drips bytes. The # watchdog owns precisely this request's socket, including after HTTP's # Connection:close detaches it from HTTPConnection into HTTPResponse. def expire(): try: owned_socket.shutdown(socket.SHUT_RDWR) except OSError: pass finally: owned_socket.close() watchdog = threading.Timer(max(0, deadline - time.monotonic()), expire) watchdog.daemon = True watchdog_started = False response = None try: watchdog.start() watchdog_started = True owned_socket.settimeout(min(5, max(0.001, deadline - time.monotonic()))) check_deadline(deadline) owned_socket.connect('/var/run/docker.sock') check_deadline(deadline) connection.request('GET', '/v1.45' + path, headers={'Connection': 'close'}) response = connection.getresponse() raw = response.read(MAX_RESPONSE + 1) check_deadline(deadline) if response.status != 200 or len(raw) > MAX_RESPONSE: raise ValueError('Docker observation unavailable') result = json.loads(raw) check_deadline(deadline) return result except Exception: check_deadline(deadline) raise finally: watchdog.cancel() if watchdog_started: watchdog.join() try: if response is not None: response.close() finally: try: connection.close() finally: owned_socket.close() def nonnegative(value): if type(value) is not int or value < 0: raise ValueError('Invalid numeric observation') return value def parse_host(meminfo, vmstat): memory = {} for line in meminfo.splitlines(): fields = line.split() if fields and fields[0] in ('MemTotal:', 'MemAvailable:', 'SwapTotal:', 'SwapFree:'): if len(fields) != 3 or fields[2] != 'kB': raise ValueError('Invalid meminfo units') memory[fields[0][:-1] + 'Bytes'] = nonnegative(int(fields[1])) * 1024 if len(memory) != 4 or memory['MemAvailableBytes'] > memory['MemTotalBytes'] or memory['SwapFreeBytes'] > memory['SwapTotalBytes']: raise ValueError('Incomplete host memory observation') for line in vmstat.splitlines(): fields = line.split() if fields and fields[0] in ('pswpin', 'pswpout', 'oom_kill'): memory[fields[0]] = nonnegative(int(fields[1])) return memory def project_container(inspect, stats, identifier): if inspect.get('Id') != identifier or stats.get('id') != identifier: raise ValueError('Pinned container changed') state = inspect['State'] memory = stats['memory_stats'] usage = nonnegative(memory['usage']) inactive = memory.get('stats', {}).get('inactive_file') if inactive is not None: inactive = nonnegative(inactive) if inactive > usage: raise ValueError('Invalid cache observation') return { 'id': identifier, 'imageId': inspect['Image'], 'startedAt': state['StartedAt'], 'running': state['Running'] is True, 'oomKilled': state.get('OOMKilled'), 'restartCount': inspect.get('RestartCount'), 'usageIncludingCacheBytes': usage, 'inactiveFileBytes': inactive, 'usageMinusInactiveFileBytes': usage - inactive if inactive is not None else None, 'configuredLimitBytes': nonnegative(inspect['HostConfig']['Memory']), 'configuredMemorySwapBytes': configured_memory_swap(inspect), 'configuredMemorySwapStatus': 'available' if configured_memory_swap(inspect) is not None else 'unavailable', 'reportedLimitBytes': nonnegative(memory['limit']), 'swapUsageBytes': memory.get('swap_usage'), } def summarize(samples): if not samples: raise ValueError('No observations') baseline = {c['id']: (c['imageId'], c['startedAt'], c['restartCount'], c.get('pid'), c.get('oomKilled')) for c in samples[0]['containers']} stable = all({c['id']: (c['imageId'], c['startedAt'], c['restartCount'], c.get('pid'), c.get('oomKilled')) for c in sample['containers']} == baseline and all(c.get('identityStableDuringSample', True) for c in sample['containers']) for sample in samples) counters = {} for field in ('pswpin', 'pswpout', 'oom_kill'): first = samples[0]['host'].get(field) last = samples[-1]['host'].get(field) counters[field + 'Delta'] = last - first if first is not None and last is not None and last >= first else None return { 'samples': len(samples), 'elapsedMs': samples[-1]['elapsedMs'], 'hostMemTotalBytes': samples[0]['host']['MemTotalBytes'], 'minHostAvailableBytes': min(s['host']['MemAvailableBytes'] for s in samples), 'maxObservedContainerUsageIncludingCacheBytes': max(sum(c['usageIncludingCacheBytes'] for c in s['containers']) for s in samples), 'containerIdentityAndStartStable': stable, 'allObservedRunning': all(c['running'] is True and c.get('runningAfter') is True and c.get('identityStableDuringSample') is True for s in samples for c in s['containers']), 'anyObservedOomKilled': any(c['oomKilled'] is True or c.get('oomKilledAfter') is True for s in samples for c in s['containers']), 'allCgroupV2MetricsAvailable': all(s['containers'] and all(c.get('cgroupV2', {}).get('status') == 'available' for c in s['containers']) for s in samples), **counters, 'allResourceLimitsStableDuringSample': all(c.get('resourceLimitsStableDuringSample') is True for s in samples for c in s['containers']) and all(s['containers'] for s in samples), 'allHostMemoryPressureAvailable': all(s['host'].get('memoryPressure', {}).get('status') == 'available' for s in samples), 'acceptancePassed': False, 'scope': 'sampled evidence only; correlate with workload, limits, errors and latency; unsampled peaks are unknown', } def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--daemon-id', required=True) parser.add_argument('--container', action='append', required=True) parser.add_argument('--samples', type=int, default=30) parser.add_argument('--interval', type=float, default=1) parser.add_argument('--output', required=True) args = parser.parse_args() if os.name != 'posix' or not hasattr(socket, 'AF_UNIX'): parser.error('Run on the Linux Docker host') if not re.fullmatch(r'[A-Za-z0-9-]{8,128}', args.daemon_id) or not 1 <= len(args.container) <= 16 or len(set(args.container)) != len(args.container) or any(not re.fullmatch(r'[a-f0-9]{64}', c) for c in args.container): parser.error('Pin exact daemon ID and 1..16 distinct full container IDs') if not 1 <= args.samples <= 300 or not 0.2 <= args.interval <= 10 or args.samples * args.interval > 300: parser.error('Observation bound: 300 samples / 300 seconds') started = time.monotonic() deadline = started + 360 if docker_get('/info', deadline).get('ID') != args.daemon_id: raise ValueError('Pinned Docker daemon changed') samples = [] for index in range(args.samples): check_deadline(deadline) host = collect_host(deadline) containers = [] for identifier in args.container: check_deadline(deadline) containers.append(collect_container(identifier, deadline)) check_deadline(deadline) samples.append({'elapsedMs': round((time.monotonic() - started) * 1000), 'host': host, 'containers': containers}) if index + 1 < args.samples: time.sleep(min(args.interval, max(0, deadline - time.monotonic()))) check_deadline(deadline) if docker_get('/info', deadline).get('ID') != args.daemon_id: raise ValueError('Pinned Docker daemon changed') result = {'schemaVersion': 3, 'readOnlyDockerAndProc': True, 'serviceChanges': 0, 'summary': summarize(samples), 'observations': samples} encoded = json.dumps(result, indent=2) + '\n' check_deadline(deadline) # Refuse to replace an existing evidence file. Never dump Docker env/logs. with open(args.output, 'x', encoding='utf-8') as handle: check_deadline(deadline) handle.write(encoded) check_deadline(deadline) print(json.dumps(result['summary'])) if __name__ == '__main__': try: main() except Exception as error: print(json.dumps({'status': 'observation-refused', 'errorClass': type(error).__name__, 'serviceChanges': 0})) raise SystemExit(1)