Source code for synapse.lib.platforms.linux

import os
import logging
import resource
import contextlib
import ctypes as c

import synapse.exc as s_exc
import synapse.lib.const as s_const

logger = logging.getLogger(__name__)

import synapse.lib.platforms.common as s_pcommon

meminfo_total_fallback_log = False

[docs]def initHostInfo(): return { 'format': 'elf', 'platform': 'linux', 'hasmemlocking': True, # has mlock, and all the below related functions 'hassysctls': True, }
[docs]def getFileMappedRegion(filename): ''' Return a tuple of address and length of a particular file memory mapped into this process ''' # /proc/<pid>/maps has a bunch of entries that look like this: # 7fb5195fc000-7fb519ffc000 r--s 00000000 fd:01 5245137 /tmp/foo.lmdb/data.mdb filename = str(filename) largest = None with open(f'/proc/{os.getpid()}/maps') as maps: for line in maps: if len(line) < 50: continue if line.rstrip().endswith(filename): addrs = line.split(' ', 1)[0] start, end = addrs.split('-') start_addr = int(start, 16) end_addr = int(end, 16) memlen = end_addr - start_addr if largest is None or memlen > largest[1]: largest = (start_addr, memlen) if largest is None: raise s_exc.NoSuchFile(mesg=f'{filename} is not mapped into current process', path=filename) return largest
[docs]def getTotalMemory(): ''' Get the total amount of memory in the system. Notes: This attempts to get information from cgroup data before falling back to ``/proc/meminfo`` data. Returns: int: The number of bytes of memory available in the system. ''' # cgroup based checks should be reliable when running inside of memory # limited containers. This cgroupv1 based check is generally safe on # older systems using cgroupv1 still. # Reference # https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt fp = '/sys/fs/cgroup/memory/memory.limit_in_bytes' if os.path.isfile(fp): with open(fp) as f: valu = int(f.read()) # Skip a known value on cgroupsv1 where there has not been # a limit set, so we will resort to using /proc/meminfo instead. # We assume a 64 bit long in our platform. Mimic the linux kernel # behavior of using a rounded integer division of # LONG_MAX / PAGE_SIZE; which is later multiplied by PAGE_SIZE # For more see https://unix.stackexchange.com/q/420906 _ps = os.sysconf('SC_PAGESIZE') if valu != ((2 ** 63 - 1) // _ps) * _ps: return valu # A host (or container) using cgroupv2 with a max memory enabled will have # a memory.max file available. # Reference # https://www.kernel.org/doc/Documentation/cgroup-v2.txt fp = '/sys/fs/cgroup/memory.max' if os.path.isfile(fp): with open(fp) as f: valu = f.read() if valu.strip() != 'max': return int(valu) # /proc/meminfo is a fallback we can try to use in the event that # we find ourselves in a situation where there is not a memory cap. # if this happens inside of a container which does not have a maximum # memory cap, we could mis-represent the available memory. # Reference # https://www.kernel.org/doc/Documentation/filesystems/proc.txt fp = '/proc/meminfo' if os.path.isfile(fp): global meminfo_total_fallback_log if meminfo_total_fallback_log is False: # Only warn about this fallback one per process. logger.debug('Unable to use cgroup information to determine total memory, using /proc/meminfo') meminfo_total_fallback_log = True return getAvailableMemory() logger.warning('Unable to find max memory limit') # pragma: no cover return 0 # pragma: no cover
[docs]def getSysctls(): _sysctls = ( ('vm.swappiness', '/proc/sys/vm/swappiness', int), ('vm.dirty_expire_centisecs', '/proc/sys/vm/dirty_expire_centisecs', int), ('vm.dirty_writeback_centisecs', '/proc/sys/vm/dirty_writeback_centisecs', int), ('vm.dirty_background_ratio', '/proc/sys/vm/dirty_background_ratio', int), ('vm.dirty_ratio', '/proc/sys/vm/dirty_ratio', int), ) ret = {} for key, fp, func in _sysctls: if os.path.isfile(fp): with open(fp) as f: valu = f.read().strip() try: ret[key] = func(valu) except Exception: # pragma: no cover logger.exception(f'Error normalizing sysctl: {key} @ {fp}, valu={valu}') ret[key] = None else: # pragma: no cover logger.warning(f'Missing sysctl: {key} @ {fp}') ret[key] = None return ret
[docs]def getAvailableMemory(): ''' Returns the available memory of the system ''' # Prefer MemAvailable over MemFree. (MemAvailable is not available on older kernels) with open(f'/proc/meminfo') as f: for line in f: if line.startswith('MemFree'): free = int(line.split()[1]) * s_const.kibibyte elif line.startswith('MemAvailable'): return int(line.split()[1]) * s_const.kibibyte return free
[docs]def getMaxLockedMemory(): ''' Returns the maximum amount of memory this process can lock ''' # TODO: consider CAP_IPC_LOCK capability _, hard = resource.getrlimit(resource.RLIMIT_MEMLOCK) if hard == resource.RLIM_INFINITY: return 2**64 - 1 return hard
[docs]def getCurrentLockedMemory(): ''' Return the amount of memory this process has locked ''' # Look for lines like: 'Locked: 400 kB' and add them up sum = 0 with open(f'/proc/{os.getpid()}/smaps') as smaps: for line in smaps: if line.startswith('Locked:'): kb = int(line.split()[1]) sum += kb * s_const.kibibyte return sum
[docs]def maximizeMaxLockedMemory(): ''' Remove any discretionary (i.e. soft) limits ''' soft, hard = resource.getrlimit(resource.RLIMIT_MEMLOCK) if soft != hard: resource.setrlimit(resource.RLIMIT_MEMLOCK, (hard, hard))
libc = s_pcommon.getLibC() # int mlock(const void *addr, size_t len); _mlock = libc.mlock _mlock.restype = c.c_int _mlock.argtypes = [c.c_void_p, c.c_size_t]
[docs]def mlock(address, length): ''' Lock a chunk of memory to prevent it from being swapped out, raising an OSError on error ''' retn = _mlock(address, length) if not retn: return err = c.get_errno() raise OSError(err, os.strerror(err))
# int munlock(const void *addr, size_t len); _munlock = libc.munlock _munlock.restype = c.c_int _munlock.argtypes = [c.c_void_p, c.c_size_t]
[docs]def munlock(address, length): ''' Unlock a chunk of memory, raising an OSError on error ''' retn = _munlock(address, length) if not retn: return err = c.get_errno() raise OSError(err, os.strerror(err))
# void *mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset); _mmap = libc.mmap _mmap.restype = c.c_void_p _mmap.argtypes = [c.c_void_p, c.c_size_t, c.c_int, c.c_int, c.c_int, c.c_ulonglong] # int munmap(void *addr, size_t length); _munmap = libc.munmap _munmap.restype = c.c_int _munmap.argtypes = [c.c_void_p, c.c_size_t]
[docs]@contextlib.contextmanager def mmap(address, length, prot, flags, fd, offset): ''' A simple mmap context manager that releases the GIL while mapping and unmapping. It raises an OSError on error ''' baseaddr = _mmap(address, length, prot, flags, fd, offset) if baseaddr == -1: err = c.get_errno() raise OSError(err, os.strerror(err)) try: yield baseaddr finally: _munmap(baseaddr, length)