Source code for synapse.lib.stormtypes

import bz2
import copy
import gzip
import json
import time

import regex
import types
import base64
import pprint
import struct
import asyncio
import decimal
import inspect
import logging
import binascii
import datetime
import calendar
import functools
import contextlib
import collections

import synapse.exc as s_exc
import synapse.common as s_common
import synapse.telepath as s_telepath

import synapse.lib.coro as s_coro
import synapse.lib.node as s_node
import synapse.lib.time as s_time
import synapse.lib.cache as s_cache
import synapse.lib.const as s_const
import synapse.lib.queue as s_queue
import synapse.lib.scope as s_scope
import synapse.lib.msgpack as s_msgpack
import synapse.lib.trigger as s_trigger
import synapse.lib.urlhelp as s_urlhelp
import synapse.lib.version as s_version
import synapse.lib.stormctrl as s_stormctrl

logger = logging.getLogger(__name__)

AXON_MINVERS_PROXY = (2, 97, 0)
AXON_MINVERS_PROXYTRUE = (2, 192, 0)
AXON_MINVERS_SSLOPTS = '>=2.162.0'

[docs] class Undef: _storm_typename = 'undef'
[docs] async def stormrepr(self): return '$lib.undef'
undef = Undef()
[docs] def confirm(perm, gateiden=None): s_scope.get('runt').confirm(perm, gateiden=gateiden)
[docs] def allowed(perm, gateiden=None): return s_scope.get('runt').allowed(perm, gateiden=gateiden)
[docs] def confirmEasyPerm(item, perm, mesg=None): return s_scope.get('runt').confirmEasyPerm(item, perm, mesg=mesg)
[docs] def allowedEasyPerm(item, perm): return s_scope.get('runt').allowedEasyPerm(item, perm)
[docs] def strifyHttpArg(item, multi=False): if isinstance(item, (list, tuple)): return [(str(k), str(v)) for (k, v) in item] elif isinstance(item, dict): retn = {} for name, valu in item.items(): if isinstance(valu, (list, tuple)) and multi: retn[str(name)] = [str(v) for v in valu] else: retn[str(name)] = str(valu) return retn return item
[docs] async def resolveCoreProxyUrl(valu): ''' Resolve a proxy value to a proxy URL. Args: valu (str|None|bool): The proxy value. Returns: (str|None): A proxy URL string or None. ''' runt = s_scope.get('runt') match valu: case None: s_common.deprecated('Setting the HTTP proxy argument $lib.null', curv='2.192.0') await runt.snap.warnonce('Setting the HTTP proxy argument to $lib.null is deprecated. Use $lib.true instead.') return await runt.snap.core.getConfOpt('http:proxy') case True: return await runt.snap.core.getConfOpt('http:proxy') case False: runt.confirm(('storm', 'lib', 'inet', 'http', 'proxy')) return None case str(): runt.confirm(('storm', 'lib', 'inet', 'http', 'proxy')) return valu case _: raise s_exc.BadArg(mesg='HTTP proxy argument must be a string or bool.')
[docs] async def resolveAxonProxyArg(valu): ''' Resolve a proxy value to the kwarg to set for an Axon HTTP call. Args: valu (str|null|bool): The proxy value. Returns: tuple: A retn tuple where the proxy kwarg should not be set if ok=False, otherwise a proxy URL or None. ''' runt = s_scope.get('runt') axonvers = runt.snap.core.axoninfo['synapse']['version'] if axonvers < AXON_MINVERS_PROXY: await runt.snap.warnonce(f'Axon version does not support proxy argument: {axonvers} < {AXON_MINVERS_PROXY}') return False, None match valu: case None: s_common.deprecated('Setting the Storm HTTP proxy argument $lib.null', curv='2.192.0') await runt.snap.warnonce('Setting the Storm HTTP proxy argument to $lib.null is deprecated. Use $lib.true instead.') if axonvers >= AXON_MINVERS_PROXYTRUE: return True, True return True, None case True: if axonvers < AXON_MINVERS_PROXYTRUE: return True, None return True, True case False: runt.confirm(('storm', 'lib', 'inet', 'http', 'proxy')) return True, False case str(): runt.confirm(('storm', 'lib', 'inet', 'http', 'proxy')) return True, valu case _: raise s_exc.BadArg(mesg='HTTP proxy argument must be a string or bool.')
[docs] class StormTypesRegistry: # The following types are currently undefined. base_undefined_types = ( 'any', 'int', 'lib', # lib.import 'null', 'time', 'prim', 'undef', 'float', 'generator', ) undefined_types = set(base_undefined_types) known_types = set() rtypes = collections.defaultdict(set) # callable -> return types, populated on demand. def __init__(self): self._LIBREG = {} self._TYPREG = {}
[docs] def addStormLib(self, path, ctor): if path in self._LIBREG: raise Exception('cannot register a library twice') assert isinstance(path, tuple) self._LIBREG[path] = ctor
[docs] def delStormLib(self, path): if not self._LIBREG.pop(path, None): raise Exception('no such path!')
[docs] def addStormType(self, path, ctor): if path in self._TYPREG: raise Exception('cannot register a type twice') assert ctor._storm_typename is not None, f'path={path} ctor={ctor}' self._TYPREG[path] = ctor self.known_types.add(ctor._storm_typename) self.undefined_types.discard(ctor._storm_typename)
[docs] def delStormType(self, path): ctor = self._TYPREG.pop(path, None) if ctor is None: raise Exception('no such path!') self.known_types.discard(ctor._storm_typename) self.undefined_types.add(ctor._storm_typename)
[docs] def registerLib(self, ctor): '''Decorator to register a StormLib''' path = getattr(ctor, '_storm_lib_path', s_common.novalu) if path is s_common.novalu: raise Exception('no key!') self.addStormLib(path, ctor) for info in ctor._storm_locals: rtype = info.get('type') if isinstance(rtype, dict) and rtype.get('type') == 'function': if (fname := rtype.get('_funcname')) == '_storm_query': continue if (func := getattr(ctor, fname, None)) is not None: funcpath = '.'.join(('lib',) + ctor._storm_lib_path + (info['name'],)) func._storm_funcpath = f"${funcpath}" return ctor
[docs] def registerType(self, ctor): '''Decorator to register a StormPrim''' self.addStormType(ctor.__name__, ctor) for info in ctor._storm_locals: rtype = info.get('type') if isinstance(rtype, dict) and rtype.get('type') == 'function': fname = rtype.get('_funcname') if (func := getattr(ctor, fname, None)) is not None: func._storm_funcpath = f"{ctor._storm_typename}.{info['name']}" return ctor
[docs] def iterLibs(self): return list(self._LIBREG.items())
[docs] def iterTypes(self): return list(self._TYPREG.items())
def _validateInfo(self, obj, info, name): # Check the rtype of the info to see if its a dict; and if so, # validate it has the _funcname key pointing to the to a # callable on the obj, and that the documented arguments match # those of the callable. The _funcname key is removed. rtype = info.get('type') if isinstance(rtype, dict): rname = rtype.get('type') rname = copy.deepcopy(rname) # Make a mutable copy we're going to modify if isinstance(rname, tuple): rname = list(rname) isstor = False isfunc = False isgtor = False isctor = False if rname == 'ctor' or 'ctor' in rname: isctor = True if isinstance(rname, str): rname = '' if isinstance(rname, list): rname.remove('ctor') if isinstance(rname, dict): rname.pop('ctor', None) if rname == 'function' or 'function' in rname: isfunc = True if isinstance(rname, str): rname = '' if isinstance(rname, list): rname.remove('function') if isinstance(rname, dict): rname.pop('function', None) if rname == 'gtor' or 'gtor' in rname: isgtor = True if isinstance(rname, str): rname = '' if isinstance(rname, list): rname.remove('gtor') if isinstance(rname, dict): rname.pop('gtor', None) if rname == 'stor' or 'stor' in rname: if isinstance(rname, str): rname = '' if isinstance(rname, list): rname.remove('stor') if isinstance(rname, dict): rname.pop('stor', None) isstor = True invalid = (isgtor and isctor) or (isfunc and (isgtor or isctor)) if invalid: mesg = f'Dictionary represents invalid combination of ctors, gtors, locls, and stors [{name} {obj} {info.get("name")}] [{rtype}].' raise AssertionError(mesg) if rname: mesg = f'Dictionary return types represents a unknown rtype [{name} {obj} {info.get("name")}] [{rtype}] [{rname}].' raise AssertionError(mesg) if isfunc: self._validateFunction(obj, info, name) if isstor: self._validateStor(obj, info, name) if isctor: self._validateCtor(obj, info, name) if isgtor: self._validateGtor(obj, info, name) def _validateFunction(self, obj, info, name): rtype = info.get('type') funcname = rtype.get('_funcname') if funcname == '_storm_query': # Sentinel used for future validation of pure storm # functions defined in _storm_query data. return locl = getattr(obj, funcname, None) assert locl is not None, f'bad _funcname=[{funcname}] for {obj} {info.get("name")}' args = rtype.get('args', ()) callsig = getCallSig(locl) # Assert the callsigs match callsig_args = [str(v).split('=')[0] for v in callsig.parameters.values()] assert [d.get('name') for d in args] == callsig_args, f'args / callsig args mismatch for {funcname} {name} {obj} {args} {callsig_args}' # ensure default values are provided for parameter, argdef in zip(callsig.parameters.values(), args): pdef = parameter.default # defaults to inspect._empty for undefined default values. adef = argdef.get('default', inspect._empty) # Allow $lib.undef as a defined default to represent the undef constant. if pdef is undef: assert adef == '$lib.undef', \ f'Expected $lib.undef for default value {obj} {funcname}, defvals {pdef} != {adef} for {parameter}' else: assert pdef == adef, \ f'Default value mismatch for {obj} {funcname}, defvals {pdef} != {adef} for {parameter}' def _validateStor(self, obj, info, name): rtype = info.get('type') funcname = rtype.pop('_storfunc') locl = getattr(obj, funcname, None) assert locl is not None, f'bad _storfunc=[{funcname}] for {obj} {info.get("name")}' args = rtype.get('args') assert args is None, f'stors have no defined args funcname=[{funcname}] for {obj} {info.get("name")}' callsig = getCallSig(locl) # Assert the callsig for a stor has one argument callsig_args = [str(v).split('=')[0] for v in callsig.parameters.values()] assert len(callsig_args) == 1, f'stor funcs must only have one argument for {obj} {info.get("name")}' def _validateCtor(self, obj, info, name): rtype = info.get('type') funcname = rtype.pop('_ctorfunc') locl = getattr(obj, funcname, None) assert locl is not None, f'bad _ctorfunc=[{funcname}] for {obj} {info.get("name")}' args = rtype.get('args') assert args is None, f'ctors have no defined args funcname=[{funcname}] for {obj} {info.get("name")}' callsig = getCallSig(locl) # Assert the callsig for a ctor has one argument callsig_args = [str(v).split('=')[0] for v in callsig.parameters.values()] assert len(callsig_args) == 1, f'ctor funcs must only have one argument for {obj} {info.get("name")}' def _validateGtor(self, obj, info, name): rtype = info.get('type') funcname = rtype.pop('_gtorfunc') locl = getattr(obj, funcname, None) assert locl is not None, f'bad _gtorfunc=[{funcname}] for {obj} {info.get("name")}' args = rtype.get('args') assert args is None, f'gtors have no defined args funcname=[{funcname}] for {obj} {info.get("name")}' callsig = getCallSig(locl) # Assert the callsig for a stor has one argument callsig_args = [str(v).split('=')[0] for v in callsig.parameters.values()] assert len(callsig_args) == 0, f'gtor funcs must only have one argument for {obj} {info.get("name")}'
[docs] def getLibDocs(self, lib=None): # Ensure type docs are loaded/verified. _ = self.getTypeDocs() if lib is None: libs = self.iterLibs() libs.sort(key=lambda x: x[0]) else: libs = ((lib._storm_lib_path, lib),) docs = [] for (sname, slib) in libs: sname = slib.__class__.__name__ locs = [] tdoc = { 'desc': getDoc(slib, sname), 'locals': locs, 'path': ('lib',) + slib._storm_lib_path, 'deprecated': slib._storm_lib_deprecation, } for info in sorted(slib._storm_locals, key=lambda x: x.get('name')): info = s_msgpack.deepcopy(info) self._validateInfo(slib, info, sname) locs.append(info) docs.append(tdoc) for tdoc in docs: basepath = tdoc.get('path') assert basepath[0] == 'lib' locls = tdoc.get('locals') for info in locls: path = basepath + (info.get('name'),) ityp = info.get('type') if isinstance(ityp, str): self.rtypes[path].add(ityp) continue retv = ityp.get('returns') rtyp = retv.get('type') if isinstance(rtyp, (list, tuple)): [self.rtypes[path].add(r) for r in rtyp] continue self.rtypes[path].add(rtyp) for path, rtyps in self.rtypes.items(): for rtyp in rtyps: if rtyp not in self.known_types and rtyp not in self.undefined_types: # pragma: no cover raise s_exc.NoSuchType(mesg=f'The return type {rtyp} for {path} is unknown.', type=rtyp) return docs
[docs] def getTypeDocs(self, styp: str =None): if styp is None: types = self.iterTypes() types.sort(key=lambda x: x[1]._storm_typename) else: types = [(k, v) for (k, v) in self.iterTypes() if styp == v._storm_typename] docs = [] for (sname, styp) in types: locs = [] tdoc = { 'desc': getDoc(styp, sname), 'locals': locs, 'path': (styp._storm_typename,), } for info in sorted(styp._storm_locals, key=lambda x: x.get('name')): info = s_msgpack.deepcopy(info) self._validateInfo(styp, info, sname) locs.append(info) docs.append(tdoc) for tdoc in docs: basepath = tdoc.get('path') assert len(basepath) == 1 locls = tdoc.get('locals') for info in locls: path = basepath + (info.get('name'),) ityp = info.get('type') if isinstance(ityp, str): self.rtypes[path].add(ityp) continue retv = ityp.get('returns') rtyp = retv.get('type') if isinstance(rtyp, (list, tuple)): [self.rtypes[path].add(r) for r in rtyp] continue self.rtypes[path].add(rtyp) for path, rtyps in self.rtypes.items(): for rtyp in rtyps: if rtyp not in self.known_types and rtyp not in self.undefined_types: # pragma: no cover raise s_exc.NoSuchType(mesg=f'The return type {rtyp} for {path} is unknown.', type=rtyp) return docs
registry = StormTypesRegistry()
[docs] def getDoc(obj, errstr): '''Helper to get __doc__''' doc = getattr(obj, '__doc__') if doc is None: doc = f'No doc for {errstr}' logger.warning(doc) return doc
[docs] def getCallSig(func) -> inspect.Signature: '''Get the callsig of a function, stripping self if present.''' callsig = inspect.signature(func) params = list(callsig.parameters.values()) if params and params[0].name == 'self': callsig = callsig.replace(parameters=params[1:]) return callsig
[docs] def stormfunc(readonly=False): def wrap(f): f._storm_readonly = readonly return f return wrap
[docs] def intify(x): if isinstance(x, str): x = x.lower() if x == 'true': return 1 if x == 'false': return 0 try: return int(x, 0) except ValueError as e: mesg = f'Failed to make an integer from "{x}".' raise s_exc.BadCast(mesg=mesg) from e try: return int(x) except Exception as e: mesg = f'Failed to make an integer from "{x}".' raise s_exc.BadCast(mesg=mesg) from e
[docs] async def kwarg_format(_text, **kwargs): ''' Replaces instances curly-braced argument names in text with their values ''' for name, valu in kwargs.items(): temp = '{%s}' % (name,) _text = _text.replace(temp, await torepr(valu, usestr=True)) return _text
[docs] class StormType: ''' The base type for storm runtime value objects. ''' _storm_locals = () # type: Any # To be overridden for deref constants that need documentation _ismutable = True _storm_typename = 'unknown' def __init__(self, path=None): self.path = path # ctors take no arguments and are intended to return Prim objects. This must be sync. # These are intended for delayed Prim object construction until they are needed. self.ctors = {} # stors are setter functions which take a single value for setting. # These are intended to act similar to python @setter decorators. self.stors = {} # gtors are getter functions which are called without arguments. This must be async. # These are intended as to act similar to python @property decorators. self.gtors = {} # Locals are intended for storing callable functions and constants. self.locls = {}
[docs] def getObjLocals(self): ''' Get the default list of key-value pairs which may be added to the object ``.locls`` dictionary. Returns: dict: A key/value pairs. ''' return {}
async def _storm_copy(self): mesg = f'Type ({self._storm_typename}) does not support being copied!' raise s_exc.BadArg(mesg=mesg)
[docs] @stormfunc(readonly=True) async def setitem(self, name, valu): if not self.stors: mesg = f'{self.__class__.__name__} does not support assignment.' raise s_exc.StormRuntimeError(mesg=mesg) name = await tostr(name) stor = self.stors.get(name) if stor is None: mesg = f'Setting {name} is not supported on {self._storm_typename}.' raise s_exc.NoSuchName(name=name, mesg=mesg) if s_scope.get('runt').readonly and not getattr(stor, '_storm_readonly', False): mesg = f'Setting {name} on {self._storm_typename} is not marked readonly safe.' raise s_exc.IsReadOnly(mesg=mesg, name=name, valu=valu) await s_coro.ornot(stor, valu)
[docs] async def deref(self, name): name = await tostr(name) locl = self.locls.get(name, s_common.novalu) if locl is not s_common.novalu: return locl ctor = self.ctors.get(name) if ctor is not None: item = ctor(path=self.path) self.locls[name] = item return item valu = await self._derefGet(name) if valu is not s_common.novalu: return valu raise s_exc.NoSuchName(mesg=f'Cannot find name [{name}] on type {self._storm_typename}', name=name, styp=self.__class__.__name__)
async def _derefGet(self, name): gtor = self.gtors.get(name) if gtor is None: return s_common.novalu return await gtor()
[docs] def ismutable(self): return self._ismutable
[docs] class Lib(StormType): ''' A collection of storm methods under a name ''' _ismutable = False _storm_query = None _storm_typename = 'lib' _storm_lib_perms = () _storm_lib_deprecation = None def __init__(self, runt, name=()): StormType.__init__(self) self.runt = runt self.name = name self.auth = runt.snap.core.auth self.addLibFuncs()
[docs] def addLibFuncs(self): self.locls.update(self.getObjLocals())
[docs] async def initLibAsync(self): if self._storm_query is not None: query = await self.runt.snap.core.getStormQuery(self._storm_query) self.modrunt = await self.runt.getModRuntime(query) self.runt.onfini(self.modrunt) async for item in self.modrunt.execute(): await asyncio.sleep(0) # pragma: no cover for k, v in self.modrunt.vars.items(): # Annotate the name and lib onto the callable # so that it can be inspected later. if callable(v) and v.__name__ == 'realfunc': v._storm_runtime_lib = self v._storm_runtime_lib_func = k v._storm_funcpath = f'${".".join(("lib",) + self.name + (k,))}' self.locls[k] = v
[docs] async def stormrepr(self): if '__module__' in self.locls: return f'Imported Module {".".join(self.name)}' return f'Library ${".".join(("lib",) + self.name)}'
[docs] async def deref(self, name): name = await tostr(name) if name.startswith('__'): raise s_exc.StormRuntimeError(mesg=f'Cannot dereference private value [{name}]', name=name) try: return await StormType.deref(self, name) except s_exc.NoSuchName: pass path = self.name + (name,) slib = self.runt.snap.core.getStormLib(path) if slib is None: raise s_exc.NoSuchName(mesg=f'Cannot find name [{name}]', name=name) ctor = slib[2].get('ctor', Lib) libinst = ctor(self.runt, name=path) await libinst.initLibAsync() return libinst
[docs] async def dyncall(self, iden, todo, gatekeys=()): return await self.runt.dyncall(iden, todo, gatekeys=gatekeys)
[docs] async def dyniter(self, iden, todo, gatekeys=()): async for item in self.runt.dyniter(iden, todo, gatekeys=gatekeys): yield item
[docs] @registry.registerLib class LibPkg(Lib): ''' A Storm Library for interacting with Storm Packages. ''' _storm_locals = ( {'name': 'add', 'desc': 'Add a Storm Package to the Cortex.', 'type': {'type': 'function', '_funcname': '_libPkgAdd', 'args': ( {'name': 'pkgdef', 'type': 'dict', 'desc': 'A Storm Package definition.', }, {'name': 'verify', 'type': 'boolean', 'default': False, 'desc': 'Verify storm package signature.', }, ), 'returns': {'type': 'null', }}}, {'name': 'get', 'desc': 'Get a Storm Package from the Cortex.', 'type': {'type': 'function', '_funcname': '_libPkgGet', 'args': ( {'name': 'name', 'type': 'str', 'desc': 'A Storm Package name.', }, ), 'returns': {'type': 'dict', 'desc': 'The Storm package definition.', }}}, {'name': 'has', 'desc': 'Check if a Storm Package is available in the Cortex.', 'type': {'type': 'function', '_funcname': '_libPkgHas', 'args': ( {'name': 'name', 'type': 'str', 'desc': 'A Storm Package name to check for the existence of.', }, ), 'returns': {'type': 'boolean', 'desc': 'True if the package exists in the Cortex, False if it does not.', }}}, {'name': 'del', 'desc': 'Delete a Storm Package from the Cortex.', 'type': {'type': 'function', '_funcname': '_libPkgDel', 'args': ( {'name': 'name', 'type': 'str', 'desc': 'The name of the package to delete.', }, ), 'returns': {'type': 'null', }}}, {'name': 'list', 'desc': 'Get a list of Storm Packages loaded in the Cortex.', 'type': {'type': 'function', '_funcname': '_libPkgList', 'returns': {'type': 'list', 'desc': 'A list of Storm Package definitions.', }}}, {'name': 'deps', 'desc': 'Verify the dependencies for a Storm Package.', 'type': {'type': 'function', '_funcname': '_libPkgDeps', 'args': ( {'name': 'pkgdef', 'type': 'dict', 'desc': 'A Storm Package definition.', }, ), 'returns': {'type': 'dict', 'desc': 'A dictionary listing dependencies and if they are met.', }}}, ) _storm_lib_path = ('pkg',)
[docs] def getObjLocals(self): return { 'add': self._libPkgAdd, 'get': self._libPkgGet, 'has': self._libPkgHas, 'del': self._libPkgDel, 'list': self._libPkgList, 'deps': self._libPkgDeps, }
async def _libPkgAdd(self, pkgdef, verify=False): self.runt.confirm(('pkg', 'add'), None) pkgdef = await toprim(pkgdef) verify = await tobool(verify) await self.runt.snap.core.addStormPkg(pkgdef, verify=verify) @stormfunc(readonly=True) async def _libPkgGet(self, name): name = await tostr(name) pkgdef = await self.runt.snap.core.getStormPkg(name) if pkgdef is None: return None return Dict(pkgdef) @stormfunc(readonly=True) async def _libPkgHas(self, name): name = await tostr(name) pkgdef = await self.runt.snap.core.getStormPkg(name) if pkgdef is None: return False return True async def _libPkgDel(self, name): self.runt.confirm(('pkg', 'del'), None) await self.runt.snap.core.delStormPkg(name) @stormfunc(readonly=True) async def _libPkgList(self): pkgs = await self.runt.snap.core.getStormPkgs() return list(sorted(pkgs, key=lambda x: x.get('name'))) @stormfunc(readonly=True) async def _libPkgDeps(self, pkgdef): pkgdef = await toprim(pkgdef) return await self.runt.snap.core.verifyStormPkgDeps(pkgdef)
[docs] @registry.registerLib class LibDmon(Lib): ''' A Storm Library for interacting with StormDmons. ''' _storm_locals = ( {'name': 'add', 'desc': ''' Add a Storm Dmon to the Cortex. Examples: Add a dmon that executes a query:: $lib.dmon.add(${ myquery }, name='example dmon') ''', 'type': {'type': 'function', '_funcname': '_libDmonAdd', 'args': ( {'name': 'text', 'type': ['str', 'storm:query'], 'desc': 'The Storm query to execute in the Dmon loop.'}, {'name': 'name', 'type': 'str', 'desc': 'The name of the Dmon.', 'default': 'noname'}, {'name': 'ddef', 'type': 'dict', 'desc': 'Additional daemon definition fields. ', 'default': None}, ), 'returns': {'type': 'str', 'desc': 'The iden of the newly created Storm Dmon.'}}}, {'name': 'get', 'desc': 'Get a Storm Dmon definition by iden.', 'type': {'type': 'function', '_funcname': '_libDmonGet', 'args': ( {'name': 'iden', 'type': 'str', 'desc': 'The iden of the Storm Dmon to get.'}, ), 'returns': {'type': 'dict', 'desc': 'A Storm Dmon definition dict.', }}}, {'name': 'del', 'desc': 'Delete a Storm Dmon by iden.', 'type': {'type': 'function', '_funcname': '_libDmonDel', 'args': ( {'name': 'iden', 'type': 'str', 'desc': 'The iden of the Storm Dmon to delete.'}, ), 'returns': {'type': 'null', }}}, {'name': 'log', 'desc': 'Get the messages from a Storm Dmon.', 'type': {'type': 'function', '_funcname': '_libDmonLog', 'args': ( {'name': 'iden', 'type': 'str', 'desc': 'The iden of the Storm Dmon to get logs for.'}, ), 'returns': {'type': 'list', 'desc': 'A list of messages from the StormDmon.'}}}, {'name': 'list', 'desc': 'Get a list of Storm Dmons.', 'type': { 'type': 'function', '_funcname': '_libDmonList', 'returns': {'type': 'list', 'desc': 'A list of Storm Dmon definitions.'}}}, {'name': 'bump', 'desc': 'Restart the Dmon.', 'type': {'type': 'function', '_funcname': '_libDmonBump', 'args': ( {'name': 'iden', 'type': 'str', 'desc': 'The GUID of the dmon to restart.'}, ), 'returns': {'type': 'boolean', 'desc': 'True if the Dmon is restarted; False if the iden does not exist.'}}}, {'name': 'stop', 'desc': 'Stop a Storm Dmon.', 'type': {'type': 'function', '_funcname': '_libDmonStop', 'args': ( {'name': 'iden', 'type': 'str', 'desc': 'The GUID of the Dmon to stop.'}, ), 'returns': {'type': 'boolean', 'desc': '$lib.true unless the dmon does not exist or was already stopped.'}}}, {'name': 'start', 'desc': 'Start a storm dmon.', 'type': {'type': 'function', '_funcname': '_libDmonStart', 'args': ( {'name': 'iden', 'type': 'str', 'desc': 'The GUID of the dmon to start.'}, ), 'returns': {'type': 'boolean', 'desc': '$lib.true unless the dmon does not exist or was already started.'}}}, ) _storm_lib_path = ('dmon',)
[docs] def getObjLocals(self): return { 'add': self._libDmonAdd, 'get': self._libDmonGet, 'del': self._libDmonDel, 'log': self._libDmonLog, 'list': self._libDmonList, 'bump': self._libDmonBump, 'stop': self._libDmonStop, 'start': self._libDmonStart, }
async def _libDmonDel(self, iden): dmon = await self.runt.snap.core.getStormDmon(iden) if dmon is None: mesg = f'No storm dmon with iden: {iden}' raise s_exc.NoSuchIden(mesg=mesg) if dmon.get('user') != self.runt.user.iden: self.runt.confirm(('dmon', 'del', iden)) await self.runt.snap.core.delStormDmon(iden) @stormfunc(readonly=True) async def _libDmonGet(self, iden): return await self.runt.snap.core.getStormDmon(iden) @stormfunc(readonly=True) async def _libDmonList(self): return await self.runt.snap.core.getStormDmons() @stormfunc(readonly=True) async def _libDmonLog(self, iden): self.runt.confirm(('dmon', 'log')) return await self.runt.snap.core.getStormDmonLog(iden) async def _libDmonAdd(self, text, name='noname', ddef=None): varz = {} # closure style capture of runtime and query vars if isinstance(text, Query): varz.update(await toprim(text.varz)) varz.update(await toprim(self.runt.vars)) varz = s_msgpack.getvars(varz) text = await tostr(text) ddef = await toprim(ddef) viewiden = self.runt.snap.view.iden self.runt.confirm(('dmon', 'add'), gateiden=viewiden) opts = {'vars': varz, 'view': viewiden} if ddef is None: ddef = {} ddef['name'] = name ddef['user'] = self.runt.user.iden ddef['storm'] = text ddef['stormopts'] = opts ddef.setdefault('enabled', True) return await self.runt.snap.core.addStormDmon(ddef) async def _libDmonBump(self, iden): iden = await tostr(iden) ddef = await self.runt.snap.core.getStormDmon(iden) if ddef is None: return False viewiden = ddef['stormopts']['view'] self.runt.confirm(('dmon', 'add'), gateiden=viewiden) await self.runt.snap.core.bumpStormDmon(iden) return True async def _libDmonStop(self, iden): iden = await tostr(iden) ddef = await self.runt.snap.core.getStormDmon(iden) if ddef is None: return False viewiden = ddef['stormopts']['view'] self.runt.confirm(('dmon', 'add'), gateiden=viewiden) return await self.runt.snap.core.disableStormDmon(iden) async def _libDmonStart(self, iden): iden = await tostr(iden) ddef = await self.runt.snap.core.getStormDmon(iden) if ddef is None: return False viewiden = ddef['stormopts']['view'] self.runt.confirm(('dmon', 'add'), gateiden=viewiden) return await self.runt.snap.core.enableStormDmon(iden)
[docs] @registry.registerLib class LibService(Lib): ''' A Storm Library for interacting with Storm Services. ''' _storm_locals = ( {'name': 'add', 'desc': 'Add a Storm Service to the Cortex.', 'type': {'type': 'function', '_funcname': '_libSvcAdd', 'args': ( {'name': 'name', 'type': 'str', 'desc': 'Name of the Storm Service to add.', }, {'name': 'url', 'type': 'str', 'desc': 'The Telepath URL to the Storm Service.', }, ), 'returns': {'type': 'dict', 'desc': 'The Storm Service definition.', }}}, {'name': 'del', 'desc': 'Remove a Storm Service from the Cortex.', 'type': {'type': 'function', '_funcname': '_libSvcDel', 'args': ( {'name': 'iden', 'type': 'str', 'desc': 'The iden of the service to remove.', }, ), 'returns': {'type': 'null', }}}, {'name': 'get', 'desc': 'Get a Storm Service definition.', 'type': {'type': 'function', '_funcname': '_libSvcGet', 'args': ( {'name': 'name', 'type': 'str', 'desc': 'The local name, local iden, or remote name, ' 'of the service to get the definition for.', }, ), 'returns': {'type': 'dict', 'desc': 'A Storm Service definition.', }}}, {'name': 'has', 'desc': 'Check if a Storm Service is available in the Cortex.', 'type': {'type': 'function', '_funcname': '_libSvcHas', 'args': ( {'name': 'name', 'type': 'str', 'desc': 'The local name, local iden, or remote name, ' 'of the service to check for the existence of.', }, ), 'returns': {'type': 'boolean', 'desc': 'True if the service exists in the Cortex, False if it does not.', }}}, {'name': 'list', 'desc': ''' List the Storm Service definitions for the Cortex. Notes: The definition dictionaries have an additional ``ready`` key added to them to indicate if the Cortex is currently connected to the Storm Service or not. ''', 'type': {'type': 'function', '_funcname': '_libSvcList', 'returns': {'type': 'list', 'desc': 'A list of Storm Service definitions.', }}}, {'name': 'wait', 'desc': ''' Wait for a given service to be ready. Notes: If a timeout value is not specified, this will block a Storm query until the service is available. ''', 'type': {'type': 'function', '_funcname': '_libSvcWait', 'args': ( {'name': 'name', 'type': 'str', 'desc': 'The name, or iden, of the service to wait for.', }, {'name': 'timeout', 'type': 'int', 'desc': 'Number of seconds to wait for the service.', 'default': None, } ), 'returns': {'type': 'boolean', 'desc': 'Returns true if the service is available, false on a ' 'timeout waiting for the service to be ready.', }}}, ) _storm_lib_perms = ( {'perm': ('service', 'add'), 'gate': 'cortex', 'desc': 'Controls the ability to add a Storm Service to the Cortex.'}, {'perm': ('service', 'del'), 'gate': 'cortex', 'desc': 'Controls the ability to delete a Storm Service from the Cortex'}, {'perm': ('service', 'get'), 'gate': 'cortex', 'desc': 'Controls the ability to get the Service object for any Storm Service.'}, {'perm': ('service', 'get', '<iden>'), 'gate': 'cortex', 'desc': 'Controls the ability to get the Service object for a Storm Service by iden.'}, {'perm': ('service', 'list'), 'gate': 'cortex', 'desc': 'Controls the ability to list all available Storm Services and their service definitions.'}, ) _storm_lib_path = ('service',)
[docs] def getObjLocals(self): return { 'add': self._libSvcAdd, 'del': self._libSvcDel, 'get': self._libSvcGet, 'has': self._libSvcHas, 'list': self._libSvcList, 'wait': self._libSvcWait, }
async def _checkSvcGetPerm(self, ssvc): ''' Helper to handle service.get.* permissions ''' try: self.runt.confirm(('service', 'get', ssvc.iden)) except s_exc.AuthDeny as e: try: self.runt.confirm(('service', 'get', ssvc.name)) except s_exc.AuthDeny: raise e from None else: # TODO: Remove support for this permission in 3.0.0 mesg = 'Use of service.get.<servicename> permissions are deprecated.' await self.runt.warnonce(mesg, svcname=ssvc.name, svciden=ssvc.iden) async def _libSvcAdd(self, name, url): self.runt.confirm(('service', 'add')) sdef = { 'name': name, 'url': url, } return await self.runt.snap.core.addStormSvc(sdef) async def _libSvcDel(self, iden): self.runt.confirm(('service', 'del')) return await self.runt.snap.core.delStormSvc(iden) async def _libSvcGet(self, name): ssvc = self.runt.snap.core.getStormSvc(name) if ssvc is None: mesg = f'No service with name/iden: {name}' raise s_exc.NoSuchName(mesg=mesg) await self._checkSvcGetPerm(ssvc) return Service(self.runt, ssvc) @stormfunc(readonly=True) async def _libSvcHas(self, name): ssvc = self.runt.snap.core.getStormSvc(name) if ssvc is None: return False return True @stormfunc(readonly=True) async def _libSvcList(self): self.runt.confirm(('service', 'list')) retn = [] for ssvc in self.runt.snap.core.getStormSvcs(): sdef = dict(ssvc.sdef) sdef['ready'] = ssvc.ready.is_set() sdef['svcname'] = ssvc.svcname sdef['svcvers'] = ssvc.svcvers retn.append(sdef) return retn @stormfunc(readonly=True) async def _libSvcWait(self, name, timeout=None): name = await tostr(name) timeout = await toint(timeout, noneok=True) ssvc = self.runt.snap.core.getStormSvc(name) if ssvc is None: mesg = f'No service with name/iden: {name}' raise s_exc.NoSuchName(mesg=mesg, name=name) await self._checkSvcGetPerm(ssvc) # Short circuit asyncio.wait_for logic by checking the ready event # value. If we call wait_for with a timeout=0 we'll almost always # raise a TimeoutError unless the future previously had the option # to complete. if timeout == 0: return ssvc.ready.is_set() fut = ssvc.ready.wait() try: await asyncio.wait_for(fut, timeout=timeout) except asyncio.TimeoutError: return False else: return True
[docs] @registry.registerLib class LibTags(Lib): ''' Storm utility functions for tags. ''' _storm_lib_path = ('tags',) _storm_locals = ( {'name': 'prefix', 'desc': ''' Normalize and prefix a list of syn:tag:part values so they can be applied. Examples: Add tag prefixes and then use them to tag nodes:: $tags = $lib.tags.prefix($result.tags, vtx.visi) { for $tag in $tags { [ +#$tag ] } } ''', 'type': {'type': 'function', '_funcname': 'prefix', 'args': ( {'name': 'names', 'type': 'list', 'desc': 'A list of syn:tag:part values to normalize and prefix.'}, {'name': 'prefix', 'type': 'str', 'desc': 'The string prefix to add to the syn:tag:part values.'}, {'name': 'ispart', 'type': 'boolean', 'default': False, 'desc': 'Whether the names have already been normalized. Normalization will be skipped if set to true.'}, ), 'returns': {'type': 'list', 'desc': 'A list of normalized and prefixed syn:tag values.', }}}, )
[docs] def getObjLocals(self): return { 'prefix': self.prefix, }
[docs] @stormfunc(readonly=True) async def prefix(self, names, prefix, ispart=False): prefix = await tostr(prefix) ispart = await tobool(ispart) tagpart = self.runt.snap.core.model.type('syn:tag:part') retn = [] async for part in toiter(names): if not ispart: try: partnorm = tagpart.norm(part)[0] retn.append(f'{prefix}.{partnorm}') except s_exc.BadTypeValu: pass else: retn.append(f'{prefix}.{part}') return retn
[docs] @registry.registerLib class LibBase(Lib): ''' The Base Storm Library. This mainly contains utility functionality. ''' _storm_lib_path = () _storm_locals = ( {'name': 'len', 'desc': ''' Get the length of a item. This could represent the size of a string, or the number of keys in a dictionary, or the number of elements in an array. It may also be used to iterate an emitter or yield function and count the total.''', 'type': {'type': 'function', '_funcname': '_len', 'args': ( {'name': 'item', 'desc': 'The item to get the length of.', 'type': 'prim', }, ), 'returns': {'type': 'int', 'desc': 'The length of the item.', }}}, {'name': 'min', 'desc': 'Get the minimum value in a list of arguments.', 'type': {'type': 'function', '_funcname': '_min', 'args': ( {'name': '*args', 'type': 'any', 'desc': 'List of arguments to evaluate.', }, ), 'returns': {'type': 'int', 'desc': 'The smallest argument.', }}}, {'name': 'max', 'desc': 'Get the maximum value in a list of arguments.', 'type': {'type': 'function', '_funcname': '_max', 'args': ( {'name': '*args', 'type': 'any', 'desc': 'List of arguments to evaluate.', }, ), 'returns': {'type': 'int', 'desc': 'The largest argument.', }}}, {'name': 'set', 'desc': 'Get a Storm Set object.', 'type': {'type': 'function', '_funcname': '_set', 'args': ( {'name': '*vals', 'type': 'any', 'desc': 'Initial values to place in the set.', }, ), 'returns': {'type': 'set', 'desc': 'The new set.', }}}, {'name': 'exit', 'desc': 'Cause a Storm Runtime to stop running.', 'type': {'type': 'function', '_funcname': '_exit', 'args': ( {'name': 'mesg', 'type': 'str', 'desc': 'Optional string to warn.', 'default': None, }, {'name': '**kwargs', 'type': 'any', 'desc': 'Keyword arguments to substitute into the mesg.', }, ), 'returns': {'type': 'null', }}}, {'name': 'guid', 'desc': 'Get a random guid, or generate a guid from the arguments.', 'type': {'type': 'function', '_funcname': '_guid', 'args': ( {'name': '*args', 'type': 'prim', 'desc': 'Arguments which are hashed to create a guid.', }, {'name': 'valu', 'type': 'prim', 'default': '$lib.undef', 'desc': 'Create a guid from a single value (no positional arguments can be specified).', }, ), 'returns': {'type': 'str', 'desc': 'A guid.', }}}, {'name': 'fire', 'desc': ''' Fire an event onto the runtime. Notes: This fires events as ``storm:fire`` event types. The name of the event is placed into a ``type`` key, and any additional keyword arguments are added to a dictionary under the ``data`` key. Examples: Fire an event called ``demo`` with some data:: cli> storm $foo='bar' $lib.fire('demo', foo=$foo, knight='ni') ... ('storm:fire', {'type': 'demo', 'data': {'foo': 'bar', 'knight': 'ni'}}) ... ''', 'type': {'type': 'function', '_funcname': '_fire', 'args': ( {'name': 'name', 'type': 'str', 'desc': 'The name of the event to fire.', }, {'name': '**info', 'type': 'any', 'desc': 'Additional keyword arguments containing data to add to the event.', }, ), 'returns': {'type': 'null', }}}, {'name': 'list', 'desc': 'Get a Storm List object. This is deprecated, use ([]) to declare a list instead.', 'deprecated': {'eolvers': 'v3.0.0'}, 'type': {'type': 'function', '_funcname': '_list', 'args': ( {'name': '*vals', 'type': 'any', 'desc': 'Initial values to place in the list.', }, ), 'returns': {'type': 'list', 'desc': 'A new list object.', }}}, {'name': 'raise', 'desc': 'Raise an exception in the storm runtime.', 'type': {'type': 'function', '_funcname': '_raise', 'args': ( {'name': 'name', 'type': 'str', 'desc': 'The name of the error condition to raise.', }, {'name': 'mesg', 'type': 'str', 'desc': 'A friendly description of the specific error.', }, {'name': '**info', 'type': 'any', 'desc': 'Additional metadata to include in the exception.', }, ), 'returns': {'type': 'null', 'desc': 'This function does not return.', }}}, {'name': 'null', 'desc': ''' This constant represents a value of None that can be used in Storm. Examples: Create a dictionary object with a key whose value is null, and call ``$lib.fire()`` with it:: cli> storm $d=({"key": $lib.null}) $lib.fire('demo', d=$d) ('storm:fire', {'type': 'demo', 'data': {'d': {'key': None}}}) ''', 'type': 'null', }, {'name': 'undef', 'desc': ''' This constant can be used to unset variables and derefs. Examples: Unset the variable $foo:: $foo = $lib.undef Remove a dictionary key bar:: $foo.bar = $lib.undef Remove a list index of 0:: $foo.0 = $lib.undef ''', 'type': 'undef', }, {'name': 'true', 'desc': ''' This constant represents a value of True that can be used in Storm. Examples: Conditionally print a statement based on the constant value:: cli> storm if $lib.true { $lib.print('Is True') } else { $lib.print('Is False') } Is True ''', 'type': 'boolean', }, {'name': 'false', 'desc': ''' This constant represents a value of False that can be used in Storm. Examples: Conditionally print a statement based on the constant value:: cli> storm if $lib.false { $lib.print('Is True') } else { $lib.print('Is False') } Is False''', 'type': 'boolean', }, {'name': 'text', 'desc': 'Get a Storm Text object. This is deprecated; please use a list to append strings to, and then use ``$lib.str.join()`` to join them on demand.', 'deprecated': {'eolvers': '3.0.0'}, 'type': {'type': 'function', '_funcname': '_text', 'args': ( {'name': '*args', 'type': 'str', 'desc': 'An initial set of values to place in the Text. ' 'These values are joined together with an empty string.', }, ), 'returns': {'type': 'text', 'desc': 'The new Text object.', }}}, {'name': 'cast', 'desc': 'Normalize a value as a Synapse Data Model Type.', 'type': {'type': 'function', '_funcname': '_cast', 'args': ( {'name': 'name', 'type': 'str', 'desc': 'The name of the model type to normalize the value as.', }, {'name': 'valu', 'type': 'any', 'desc': 'The value to normalize.', }, ), 'returns': {'type': 'prim', 'desc': 'The normalized value.', }}}, {'name': 'warn', 'desc': ''' Print a warning message to the runtime. Notes: Arbitrary objects can be warned as well. They will have their Python __repr()__ printed. ''', 'type': {'type': 'function', '_funcname': '_warn', 'args': ( {'name': 'mesg', 'type': 'str', 'desc': 'String to warn.', }, {'name': '**kwargs', 'type': 'any', 'desc': 'Keyword arguments to substitute into the mesg.', }, ), 'returns': {'type': 'null', }}}, {'name': 'print', 'desc': ''' Print a message to the runtime. Examples: Print a simple string:: cli> storm $lib.print("Hello world!") Hello world! Format and print string based on variables:: cli> storm $d=({"key1": (1), "key2": "two"}) for ($key, $value) in $d { $lib.print('{k} => {v}', k=$key, v=$value) } key1 => 1 key2 => two Use values off of a node to format and print string:: cli> storm inet:ipv4:asn $lib.print("node: {ndef}, asn: {asn}", ndef=$node.ndef(), asn=:asn) | spin node: ('inet:ipv4', 16909060), asn: 1138 Notes: Arbitrary objects can be printed as well. They will have their Python __repr()__ printed. ''', 'type': {'type': 'function', '_funcname': '_print', 'args': ( {'name': 'mesg', 'type': 'str', 'desc': 'String to print.', }, {'name': '**kwargs', 'type': 'any', 'desc': 'Keyword arguments to substitute into the mesg.', }, ), 'returns': {'type': 'null', }}}, {'name': 'range', 'desc': ''' Generate a range of integers. Examples: Generate a sequence of integers based on the size of an array:: cli> storm $a=(foo,bar,(2)) for $i in $lib.range($lib.len($a)) {$lib.fire('test', indx=$i, valu=$a.$i)} Executing query at 2021/03/22 19:25:48.835 ('storm:fire', {'type': 'test', 'data': {'index': 0, 'valu': 'foo'}}) ('storm:fire', {'type': 'test', 'data': {'index': 1, 'valu': 'bar'}}) ('storm:fire', {'type': 'test', 'data': {'index': 2, 'valu': 2}}) Notes: The range behavior is the same as the Python3 ``range()`` builtin Sequence type. ''', 'type': {'type': 'function', '_funcname': '_range', 'args': ( {'name': 'stop', 'type': 'int', 'desc': 'The value to stop at.', }, {'name': 'start', 'type': 'int', 'desc': 'The value to start at.', 'default': None, }, {'name': 'step', 'type': 'int', 'desc': 'The range step size.', 'default': None, }, ), 'returns': {'name': 'Yields', 'type': 'int', 'desc': 'The sequence of integers.'}}}, {'name': 'pprint', 'desc': 'The pprint API should not be considered a stable interface.', 'type': {'type': 'function', '_funcname': '_pprint', 'args': ( {'name': 'item', 'type': 'any', 'desc': 'Item to pprint', }, {'name': 'prefix', 'type': 'str', 'desc': 'Line prefix.', 'default': '', }, {'name': 'clamp', 'type': 'int', 'desc': 'Line clamping length.', 'default': None, }, ), 'returns': {'type': 'null', }}}, {'name': 'sorted', 'desc': 'Yield sorted values.', 'type': {'type': 'function', '_funcname': '_sorted', 'args': ( {'name': 'valu', 'type': 'any', 'desc': 'An iterable object to sort.', }, {'name': 'reverse', 'type': 'boolean', 'desc': 'Reverse the sort order.', 'default': False}, ), 'returns': {'name': 'Yields', 'type': 'any', 'desc': 'Yields the sorted output.', }}}, {'name': 'import', 'desc': 'Import a Storm module.', 'type': {'type': 'function', '_funcname': '_libBaseImport', 'args': ( {'name': 'name', 'type': 'str', 'desc': 'Name of the module to import.', }, {'name': 'debug', 'type': 'boolean', 'default': False, 'desc': 'Enable debugging in the module.'}, {'name': 'reqvers', 'type': 'str', 'default': None, 'desc': 'Version requirement for the imported module.', }, ), 'returns': {'type': 'lib', 'desc': 'A ``lib`` instance representing the imported package.', }}}, {'name': 'trycast', 'desc': ''' Attempt to normalize a value and return status and the normalized value. Examples: Do something if the value is a valid IPV4:: ($ok, $ipv4) = $lib.trycast(inet:ipv4, 1.2.3.4) if $ok { $dostuff($ipv4) } ''', 'type': {'type': 'function', '_funcname': 'trycast', 'args': ( {'name': 'name', 'type': 'str', 'desc': 'The name of the model type to normalize the value as.', }, {'name': 'valu', 'type': 'any', 'desc': 'The value to normalize.', }, ), 'returns': {'type': 'list', 'desc': 'A list of (<bool>, <prim>) for status and normalized value.', }}}, {'name': 'repr', 'desc': ''' Attempt to convert a system mode value to a display mode string. Examples: Print the Synapse user name for an iden:: $lib.print($lib.repr(syn:user, $iden)) ''', 'type': {'type': 'function', '_funcname': '_repr', 'args': ( {'name': 'name', 'type': 'str', 'desc': 'The name of the model type.'}, {'name': 'valu', 'type': 'any', 'desc': 'The value to convert.'}, ), 'returns': {'type': 'str', 'desc': 'A display mode representation of the value.'}}}, {'name': 'debug', 'desc': ''' True if the current runtime has debugging enabled. Note: The debug state is inherited by sub-runtimes at instantiation time. Any changes to a runtime's debug state do not percolate automatically. Examples: Check if the runtime is in debug and print a message:: if $lib.debug { $lib.print('Doing stuff!') } Update the current runtime to enable debugging:: $lib.debug = $lib.true''', 'type': { 'type': ['gtor', 'stor'], '_storfunc': '_setRuntDebug', '_gtorfunc': '_getRuntDebug', 'returns': {'type': 'boolean'}}}, {'name': 'copy', 'desc': ''' Create and return a deep copy of the given storm object. Note: This is currently limited to msgpack compatible primitives. Examples: Make a copy of a list or dict:: $copy = $lib.copy($item) ''', 'type': {'type': 'function', '_funcname': '_copy', 'args': ( {'name': 'item', 'type': 'prim', 'desc': 'The item to make a copy of.', }, ), 'returns': {'type': 'prim', 'desc': 'A deep copy of the primitive object.', }}}, ) def __init__(self, runt, name=()): Lib.__init__(self, runt, name=name) self.stors['debug'] = self._setRuntDebug self.gtors['debug'] = self._getRuntDebug async def _getRuntDebug(self): return self.runt.debug @stormfunc(readonly=True) async def _setRuntDebug(self, debug): self.runt.debug = await tobool(debug)
[docs] def getObjLocals(self): return { 'len': self._len, 'min': self._min, 'max': self._max, 'set': self._set, 'copy': self._copy, 'exit': self._exit, 'guid': self._guid, 'fire': self._fire, 'list': self._list, 'null': None, 'undef': undef, 'true': True, 'false': False, 'text': self._text, 'cast': self._cast, 'repr': self._repr, 'warn': self._warn, 'print': self._print, 'raise': self._raise, 'range': self._range, 'pprint': self._pprint, 'sorted': self._sorted, 'import': self._libBaseImport, 'trycast': self.trycast, }
@stormfunc(readonly=True) async def _libBaseImport(self, name, debug=False, reqvers=None): name = await tostr(name) debug = await tobool(debug) reqvers = await tostr(reqvers, noneok=True) mdef = await self.runt.snap.core.getStormMod(name, reqvers=reqvers) if mdef is None: mesg = f'No storm module named {name} matching version requirement {reqvers}' raise s_exc.NoSuchName(mesg=mesg, name=name, reqvers=reqvers) text = mdef.get('storm') modconf = mdef.get('modconf') query = await self.runt.getStormQuery(text) asroot = False rootperms = mdef.get('asroot:perms') if rootperms is not None: for perm in rootperms: if self.runt.allowed(perm): asroot = True break if not asroot: permtext = ' or '.join(('.'.join(p) for p in rootperms)) mesg = f'Module ({name}) requires permission: {permtext}' raise s_exc.AuthDeny(mesg=mesg, user=self.runt.user.iden, username=self.runt.user.name) else: perm = ('storm', 'asroot', 'mod') + tuple(name.split('.')) asroot = self.runt.allowed(perm) if mdef.get('asroot', False) and not asroot: mesg = f'Module ({name}) elevates privileges. You need perm: storm.asroot.mod.{name}' raise s_exc.AuthDeny(mesg=mesg, user=self.runt.user.iden, username=self.runt.user.name) modr = await self.runt.getModRuntime(query, opts={'vars': {'modconf': modconf}}) modr.asroot = asroot if debug: modr.debug = debug self.runt.onfini(modr) async for item in modr.execute(): await asyncio.sleep(0) # pragma: no cover modlib = Lib(modr) modlib.locls.update(modr.vars) modlib.locls['__module__'] = mdef modlib.name = (name,) return modlib @stormfunc(readonly=True) async def _copy(self, item): # short circuit a few python types if item is None: return None if isinstance(item, (int, str, bool)): return item try: valu = fromprim(item) except s_exc.NoSuchType: mesg = 'Type does not have a Storm primitive and cannot be copied.' raise s_exc.BadArg(mesg=mesg) from None try: return await valu._storm_copy() except s_exc.NoSuchType: mesg = 'Nested type does not support being copied!' raise s_exc.BadArg(mesg=mesg) from None def _reqTypeByName(self, name): typeitem = self.runt.snap.core.model.type(name) if typeitem is not None: return typeitem # If a type cannot be found for the form, see if name is a property # that has a type we can use propitem = self.runt.snap.core.model.prop(name) if propitem is not None: return propitem.type mesg = f'No type or prop found for name {name}.' raise s_exc.NoSuchType(mesg=mesg) @stormfunc(readonly=True) async def _cast(self, name, valu): name = await toprim(name) valu = await toprim(valu) typeitem = self._reqTypeByName(name) # TODO an eventual mapping between model types and storm prims norm, info = typeitem.norm(valu) return fromprim(norm, basetypes=False)
[docs] @stormfunc(readonly=True) async def trycast(self, name, valu): name = await toprim(name) valu = await toprim(valu) typeitem = self._reqTypeByName(name) try: norm, info = typeitem.norm(valu) return (True, fromprim(norm, basetypes=False)) except s_exc.BadTypeValu: return (False, None)
@stormfunc(readonly=True) async def _repr(self, name, valu): name = await toprim(name) valu = await toprim(valu) return self._reqTypeByName(name).repr(valu) @stormfunc(readonly=True) async def _exit(self, mesg=None, **kwargs): if mesg: mesg = await self._get_mesg(mesg, **kwargs) await self.runt.warn(mesg, log=False) raise s_stormctrl.StormExit(mesg=mesg) raise s_stormctrl.StormExit() @stormfunc(readonly=True) async def _sorted(self, valu, reverse=False): valu = await toprim(valu) if isinstance(valu, dict): valu = list(valu.items()) for item in sorted(valu, reverse=reverse): yield item @stormfunc(readonly=True) async def _set(self, *vals): return Set(vals) @stormfunc(readonly=True) async def _list(self, *vals): s_common.deprecated('$lib.list()', curv='2.194.0') await self.runt.snap.warnonce('$lib.list() is deprecated. Use ([]) instead.') return List(list(vals)) @stormfunc(readonly=True) async def _text(self, *args): s_common.deprecated('$lib.text()', curv='2.194.0') runt = s_scope.get('runt') if runt: await runt.snap.warnonce('$lib.text() is deprecated. Please use a list to append strings to, and then use ``$lib.str.join()`` to join them on demand.') valu = ''.join(args) return Text(valu) @stormfunc(readonly=True) async def _guid(self, *args, valu=undef): if args: if valu is not undef: raise s_exc.BadArg(mesg='Valu cannot be specified if positional arguments are provided') args = await toprim(args) return s_common.guid(args) if valu is not undef: valu = await toprim(valu) return s_common.guid(valu) return s_common.guid() @stormfunc(readonly=True) async def _len(self, item): if isinstance(item, (types.GeneratorType, types.AsyncGeneratorType)): size = 0 async for _ in s_coro.agen(item): size += 1 await asyncio.sleep(0) return size try: return len(item) except TypeError: name = f'{item.__class__.__module__}.{item.__class__.__name__}' raise s_exc.StormRuntimeError(mesg=f'Object {name} does not have a length.', name=name) from None except Exception as e: # pragma: no cover name = f'{item.__class__.__module__}.{item.__class__.__name__}' raise s_exc.StormRuntimeError(mesg=f'Unknown error during len(): {repr(e)}', name=name) @stormfunc(readonly=True) async def _min(self, *args): args = await toprim(args) # allow passing in a list of ints vals = [] for arg in args: if isinstance(arg, (list, tuple)): vals.extend(arg) continue vals.append(arg) if len(vals) < 1: mesg = '$lib.min() must have at least one argument or a list containing at least one value.' raise s_exc.StormRuntimeError(mesg=mesg) ints = [await toint(x) for x in vals] return min(ints) @stormfunc(readonly=True) async def _max(self, *args): args = await toprim(args) # allow passing in a list of ints vals = [] for arg in args: if isinstance(arg, (list, tuple)): vals.extend(arg) continue vals.append(arg) if len(vals) < 1: mesg = '$lib.max() must have at least one argument or a list containing at least one value.' raise s_exc.StormRuntimeError(mesg=mesg) ints = [await toint(x) for x in vals] return max(ints) @staticmethod async def _get_mesg(mesg, **kwargs): if not isinstance(mesg, str): mesg = await torepr(mesg) elif kwargs: mesg = await kwarg_format(mesg, **kwargs) return mesg @stormfunc(readonly=True) async def _print(self, mesg, **kwargs): mesg = await self._get_mesg(mesg, **kwargs) await self.runt.printf(mesg) @stormfunc(readonly=True) async def _raise(self, name, mesg, **info): name = await tostr(name) mesg = await tostr(mesg) info = await toprim(info) s_common.reqjsonsafe(info) ctor = getattr(s_exc, name, None) if ctor is not None: raise ctor(mesg=mesg, **info) info['mesg'] = mesg info['errname'] = name raise s_exc.StormRaise(**info) @stormfunc(readonly=True) async def _range(self, stop, start=None, step=None): stop = await toint(stop) start = await toint(start, True) step = await toint(step, True) if start is not None: if step is not None: genr = range(start, stop, step) else: genr = range(start, stop) else: genr = range(stop) for valu in genr: yield valu await asyncio.sleep(0) @stormfunc(readonly=True) async def _pprint(self, item, prefix='', clamp=None): if clamp is not None: clamp = await toint(clamp) if clamp < 3: mesg = 'Invalid clamp length.' raise s_exc.StormRuntimeError(mesg=mesg, clamp=clamp) try: item = await toprim(item) except s_exc.NoSuchType: pass lines = pprint.pformat(item).splitlines() for line in lines: fline = f'{prefix}{line}' if clamp and len(fline) > clamp: await self.runt.printf(f'{fline[:clamp-3]}...') else: await self.runt.printf(fline) @stormfunc(readonly=True) async def _warn(self, mesg, **kwargs): mesg = await self._get_mesg(mesg, **kwargs) await self.runt.warn(mesg, log=False) @stormfunc(readonly=True) async def _fire(self, name, **info): info = await toprim(info) s_common.reqjsonsafe(info) await self.runt.snap.fire('storm:fire', type=name, data=info)
[docs] @registry.registerLib class LibDict(Lib): ''' A Storm Library for interacting with dictionaries. ''' _storm_locals = ( {'name': 'has', 'desc': 'Check a dictionary has a specific key.', 'type': {'type': 'function', '_funcname': '_has', 'args': ( {'name': 'valu', 'type': 'dict', 'desc': 'The dictionary being checked.'}, {'name': 'key', 'type': 'any', 'desc': 'The key to check.'}, ), 'returns': {'type': 'boolean', 'desc': 'True if the key is present, false if the key is not present.'}}}, {'name': 'keys', 'desc': 'Retrieve a list of keys in the specified dictionary.', 'type': {'type': 'function', '_funcname': '_keys', 'args': ( {'name': 'valu', 'type': 'dict', 'desc': 'The dictionary to operate on.'}, ), 'returns': {'type': 'list', 'desc': 'List of keys in the specified dictionary.', }}}, {'name': 'pop', 'desc': 'Remove specified key and return the corresponding value.', 'type': {'type': 'function', '_funcname': '_pop', 'args': ( {'name': 'valu', 'type': 'dict', 'desc': 'The dictionary to operate on.'}, {'name': 'key', 'type': 'any', 'desc': 'The key to pop.'}, {'name': 'default', 'type': 'any', 'default': '$lib.undef', 'desc': 'Optional default value to return if the key does not exist in the dictionary.'}, ), 'returns': {'type': 'any', 'desc': 'The popped value.', }}}, {'name': 'update', 'desc': 'Update the specified dictionary with keys/values from another dictionary.', 'type': {'type': 'function', '_funcname': '_update', 'args': ( {'name': 'valu', 'type': 'dict', 'desc': 'The target dictionary (update to).'}, {'name': 'other', 'type': 'dict', 'desc': 'The source dictionary (update from).'}, ), 'returns': {'type': 'null'}}}, {'name': 'values', 'desc': 'Retrieve a list of values in the specified dictionary.', 'type': {'type': 'function', '_funcname': '_values', 'args': ( {'name': 'valu', 'type': 'dict', 'desc': 'The dictionary to operate on.'}, ), 'returns': {'type': 'list', 'desc': 'List of values in the specified dictionary.', }}}, ) _storm_lib_path = ('dict',)
[docs] def getObjLocals(self): return { 'has': self._has, 'keys': self._keys, 'pop': self._pop, 'update': self._update, 'values': self._values, }
async def _check_type(self, valu, name='valu'): if isinstance(valu, (dict, Dict)): return typ = getattr(valu, '_storm_typename', None) if typ is None: prim = await toprim(valu) typ = type(prim).__name__ mesg = f'{name} argument must be a dict, not {typ}.' raise s_exc.BadArg(mesg=mesg) @stormfunc(readonly=True) async def _has(self, valu, key): await self._check_type(valu) key = await toprim(key) valu = await toprim(valu) return key in valu @stormfunc(readonly=True) async def _keys(self, valu): await self._check_type(valu) valu = await toprim(valu) return list(valu.keys()) @stormfunc(readonly=True) async def _pop(self, valu, key, default=undef): await self._check_type(valu) key = await toprim(key) real = await toprim(valu) if key not in real: if default == undef: mesg = f'Key {key} does not exist in dictionary.' raise s_exc.BadArg(mesg=mesg) return await toprim(default) # Make sure we have a storm Dict valu = fromprim(valu) ret = await valu.deref(key) await valu.setitem(key, undef) return ret @stormfunc(readonly=True) async def _update(self, valu, other): await self._check_type(valu) await self._check_type(other, name='other') valu = fromprim(valu) other = await toprim(other) for k, v in other.items(): await valu.setitem(k, v) @stormfunc(readonly=True) async def _values(self, valu): await self._check_type(valu) valu = await toprim(valu) return list(valu.values()) async def __call__(self, **kwargs): s_common.deprecated('$lib.dict()', curv='2.161.0') await self.runt.snap.warnonce('$lib.dict() is deprecated. Use ({}) instead.') return Dict(kwargs)
[docs] @registry.registerLib class LibPs(Lib): ''' A Storm Library for interacting with running tasks on the Cortex. ''' _storm_locals = ( # type: ignore {'name': 'kill', 'desc': 'Stop a running task on the Cortex.', 'type': {'type': 'function', '_funcname': '_kill', 'args': ( {'name': 'prefix', 'type': 'str', 'desc': 'The prefix of the task to stop. ' 'Tasks will only be stopped if there is a single prefix match.'}, ), 'returns': {'type': 'boolean', 'desc': 'True if the task was cancelled, False otherwise.', }}}, {'name': 'list', 'desc': 'List tasks the current user can access.', 'type': {'type': 'function', '_funcname': '_list', 'returns': {'type': 'list', 'desc': 'A list of task definitions.', }}}, ) _storm_lib_path = ('ps',)
[docs] def getObjLocals(self): return { 'kill': self._kill, 'list': self._list, }
async def _kill(self, prefix): idens = [] todo = s_common.todo('ps', self.runt.user) tasks = await self.dyncall('cell', todo) for task in tasks: iden = task.get('iden') if iden.startswith(prefix): idens.append(iden) if len(idens) == 0: mesg = 'Provided iden does not match any processes.' raise s_exc.StormRuntimeError(mesg=mesg, iden=prefix) if len(idens) > 1: mesg = 'Provided iden matches more than one process.' raise s_exc.StormRuntimeError(mesg=mesg, iden=prefix) todo = s_common.todo('kill', self.runt.user, idens[0]) return await self.dyncall('cell', todo) @stormfunc(readonly=True) async def _list(self): todo = s_common.todo('ps', self.runt.user) return await self.dyncall('cell', todo)
[docs] @registry.registerLib class LibStr(Lib): ''' A Storm Library for interacting with strings. ''' _storm_locals = ( {'name': 'join', 'desc': ''' Join items into a string using a separator. Examples: Join together a list of strings with a dot separator:: cli> storm $foo=$lib.str.join('.', ('rep', 'vtx', 'tag')) $lib.print($foo) rep.vtx.tag''', 'type': {'type': 'function', '_funcname': 'join', 'args': ( {'name': 'sepr', 'type': 'str', 'desc': 'The separator used to join strings with.', }, {'name': 'items', 'type': 'list', 'desc': 'A list of items to join together.', }, ), 'returns': {'type': 'str', 'desc': 'The joined string.', }}}, {'name': 'concat', 'desc': 'Concatenate a set of strings together.', 'type': {'type': 'function', '_funcname': 'concat', 'args': ( {'name': '*args', 'type': 'any', 'desc': 'Items to join together.', }, ), 'returns': {'type': 'str', 'desc': 'The joined string.', }}}, {'name': 'format', 'desc': ''' Format a text string. Examples: Format a string with a fixed argument and a variable:: cli> storm $list=(1,2,3,4) $str=$lib.str.format('Hello {name}, your list is {list}!', name='Reader', list=$list) $lib.print($str) Hello Reader, your list is ['1', '2', '3', '4']!''', 'type': {'type': 'function', '_funcname': 'format', 'args': ( {'name': 'text', 'type': 'str', 'desc': 'The base text string.', }, {'name': '**kwargs', 'type': 'any', 'desc': 'Keyword values which are substituted into the string.', }, ), 'returns': {'type': 'str', 'desc': 'The new string.', }}}, ) _storm_lib_path = ('str',)
[docs] def getObjLocals(self): return { 'join': self.join, 'concat': self.concat, 'format': self.format, }
[docs] @stormfunc(readonly=True) async def concat(self, *args): strs = [await tostr(a) for a in args] return ''.join(strs)
[docs] @stormfunc(readonly=True) async def format(self, text, **kwargs): text = await kwarg_format(text, **kwargs) return text
[docs] @stormfunc(readonly=True) async def join(self, sepr, items): strs = [await tostr(item) async for item in toiter(items)] return sepr.join(strs)
[docs] @registry.registerLib class LibAxon(Lib): ''' A Storm library for interacting with the Cortex's Axon. For APIs that accept an ssl_opts argument, the dictionary may contain the following values:: ({ 'verify': <bool> - Perform SSL/TLS verification. Is overridden by the ssl argument. 'client_cert': <str> - PEM encoded full chain certificate for use in mTLS. 'client_key': <str> - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert. }) For APIs that accept a proxy argument, the following values are supported:: $lib.null: Deprecated - Use the proxy defined by the http:proxy configuration option if set. $lib.true: Use the proxy defined by the http:proxy configuration option if set. $lib.false: Do not use the proxy defined by the http:proxy configuration option if set. <str>: A proxy URL string. ''' _storm_locals = ( {'name': 'wget', 'desc': """ A method to download an HTTP(S) resource into the Cortex's Axon. Notes: The response body will be stored regardless of the status code. See the ``Axon.wget()`` API documentation to see the complete structure of the response dictionary. Example: Get the Vertex Project website:: $headers = ({}) $headers."User-Agent" = Foo/Bar $resp = $lib.axon.wget("http://vertex.link", method=GET, headers=$headers) if $resp.ok { $lib.print("Downloaded: {size} bytes", size=$resp.size) } """, 'type': {'type': 'function', '_funcname': 'wget', 'args': ( {'name': 'url', 'type': 'str', 'desc': 'The URL to download'}, {'name': 'headers', 'type': 'dict', 'desc': 'An optional dictionary of HTTP headers to send.', 'default': None}, {'name': 'params', 'type': 'dict', 'desc': 'An optional dictionary of URL parameters to add.', 'default': None}, {'name': 'method', 'type': 'str', 'desc': 'The HTTP method to use.', 'default': 'GET'}, {'name': 'json', 'type': 'dict', 'desc': 'A JSON object to send as the body.', 'default': None}, {'name': 'body', 'type': 'bytes', 'desc': 'Bytes to send as the body.', 'default': None}, {'name': 'ssl', 'type': 'boolean', 'desc': 'Set to False to disable SSL/TLS certificate verification.', 'default': True}, {'name': 'timeout', 'type': 'int', 'desc': 'Timeout for the download operation.', 'default': None}, {'name': 'proxy', 'type': ['bool', 'str'], 'desc': 'Configure proxy usage. See $lib.axon help for additional details.', 'default': True}, {'name': 'ssl_opts', 'type': 'dict', 'desc': 'Optional SSL/TLS options. See $lib.axon help for additional details.', 'default': None}, ), 'returns': {'type': 'dict', 'desc': 'A status dictionary of metadata.'}}}, {'name': 'wput', 'desc': """ A method to upload a blob from the axon to an HTTP(S) endpoint. """, 'type': {'type': 'function', '_funcname': 'wput', 'args': ( {'name': 'sha256', 'type': 'str', 'desc': 'The sha256 of the file blob to upload.'}, {'name': 'url', 'type': 'str', 'desc': 'The URL to upload the file to.'}, {'name': 'headers', 'type': 'dict', 'desc': 'An optional dictionary of HTTP headers to send.', 'default': None}, {'name': 'params', 'type': 'dict', 'desc': 'An optional dictionary of URL parameters to add.', 'default': None}, {'name': 'method', 'type': 'str', 'desc': 'The HTTP method to use.', 'default': 'PUT'}, {'name': 'ssl', 'type': 'boolean', 'desc': 'Set to False to disable SSL/TLS certificate verification.', 'default': True}, {'name': 'timeout', 'type': 'int', 'desc': 'Timeout for the download operation.', 'default': None}, {'name': 'proxy', 'type': ['bool', 'str'], 'desc': 'Configure proxy usage. See $lib.axon help for additional details.', 'default': True}, {'name': 'ssl_opts', 'type': 'dict', 'desc': 'Optional SSL/TLS options. See $lib.axon help for additional details.', 'default': None}, ), 'returns': {'type': 'dict', 'desc': 'A status dictionary of metadata.'}}}, {'name': 'urlfile', 'desc': ''' Retrieve the target URL using the wget() function and construct an inet:urlfile node from the response. Notes: This accepts the same arguments as ``$lib.axon.wget()``. ''', 'type': {'type': 'function', '_funcname': 'urlfile', 'args': ( {'name': '*args', 'type': 'any', 'desc': 'Args from ``$lib.axon.wget()``.'}, {'name': '**kwargs', 'type': 'any', 'desc': 'Args from ``$lib.axon.wget()``.'}, ), 'returns': {'type': ['node', 'null'], 'desc': 'The ``inet:urlfile`` node on success, ``null`` on error.'}}}, {'name': 'del', 'desc': ''' Remove the bytes from the Cortex's Axon by sha256. Example: Delete files from the axon based on a tag:: file:bytes#foo +:sha256 $lib.axon.del(:sha256) ''', 'type': {'type': 'function', '_funcname': 'del_', 'args': ( {'name': 'sha256', 'type': 'hash:sha256', 'desc': 'The sha256 of the bytes to remove from the Axon.'}, ), 'returns': {'type': 'boolean', 'desc': 'True if the bytes were found and removed.'}}}, {'name': 'dels', 'desc': ''' Remove multiple byte blobs from the Cortex's Axon by a list of sha256 hashes. Example: Delete a list of files (by hash) from the Axon:: $list = ($hash0, $hash1, $hash2) $lib.axon.dels($list) ''', 'type': {'type': 'function', '_funcname': 'dels', 'args': ( {'name': 'sha256s', 'type': 'list', 'desc': 'A list of sha256 hashes to remove from the Axon.'}, ), 'returns': {'type': 'list', 'desc': 'A list of boolean values that are True if the bytes were found.'}}}, {'name': 'list', 'desc': ''' List (offset, sha256, size) tuples for files in the Axon in added order. Example: List files:: for ($offs, $sha256, $size) in $lib.axon.list() { $lib.print($sha256) } Start list from offset 10:: for ($offs, $sha256, $size) in $lib.axon.list(10) { $lib.print($sha256) } ''', 'type': {'type': 'function', '_funcname': 'list', 'args': ( {'name': 'offs', 'type': 'int', 'desc': 'The offset to start from.', 'default': 0}, {'name': 'wait', 'type': 'boolean', 'default': False, 'desc': 'Wait for new results and yield them in realtime.'}, {'name': 'timeout', 'type': 'int', 'default': None, 'desc': 'The maximum time to wait for a new result before returning.'}, ), 'returns': {'name': 'yields', 'type': 'list', 'desc': 'Tuple of (offset, sha256, size) in added order.'}}}, {'name': 'readlines', 'desc': ''' Yields lines of text from a plain-text file stored in the Axon. Examples: // Get the lines for a given file. for $line in $lib.axon.readlines($sha256) { $dostuff($line) } ''', 'type': {'type': 'function', '_funcname': 'readlines', 'args': ( {'name': 'sha256', 'type': 'str', 'desc': 'The SHA256 hash of the file.'}, {'name': 'errors', 'type': 'str', 'default': 'ignore', 'desc': 'Specify how encoding errors should handled.'}, ), 'returns': {'name': 'yields', 'type': 'str', 'desc': 'A line of text from the file.'}}}, {'name': 'jsonlines', 'desc': ''' Yields JSON objects from a JSON-lines file stored in the Axon. Example: Get the JSON objects from a given JSONL file:: for $item in $lib.axon.jsonlines($sha256) { $dostuff($item) } ''', 'type': {'type': 'function', '_funcname': 'jsonlines', 'args': ( {'name': 'sha256', 'type': 'str', 'desc': 'The SHA256 hash of the file.'}, {'name': 'errors', 'type': 'str', 'default': 'ignore', 'desc': 'Specify how encoding errors should handled.'}, ), 'returns': {'name': 'yields', 'type': 'any', 'desc': 'A JSON object parsed from a line of text.'}}}, {'name': 'csvrows', 'desc': ''' Yields CSV rows from a CSV file stored in the Axon. Notes: The dialect and fmtparams expose the Python csv.reader() parameters. Example: Get the rows from a given csv file:: for $row in $lib.axon.csvrows($sha256) { $dostuff($row) } Get the rows from a given tab separated file:: for $row in $lib.axon.csvrows($sha256, delimiter="\\t") { $dostuff($row) } ''', 'type': {'type': 'function', '_funcname': 'csvrows', 'args': ( {'name': 'sha256', 'type': 'str', 'desc': 'The SHA256 hash of the file.'}, {'name': 'dialect', 'type': 'str', 'desc': 'The default CSV dialect to use.', 'default': 'excel'}, {'name': 'errors', 'type': 'str', 'default': 'ignore', 'desc': 'Specify how encoding errors should handled.'}, {'name': '**fmtparams', 'type': 'any', 'desc': 'Format arguments.'}, ), 'returns': {'name': 'yields', 'type': 'list', 'desc': 'A list of strings from the CSV file.'}}}, {'name': 'metrics', 'desc': ''' Get runtime metrics of the Axon. Example: Print the total number of files stored in the Axon:: $data = $lib.axon.metrics() $lib.print("The Axon has {n} files", n=$data."file:count") ''', 'type': {'type': 'function', '_funcname': 'metrics', 'returns': {'type': 'dict', 'desc': 'A dictionary containing runtime data about the Axon.'}}}, {'name': 'put', 'desc': ''' Save the given bytes variable to the Axon the Cortex is configured to use. Examples: Save a base64 encoded buffer to the Axon:: cli> storm $s='dGVzdA==' $buf=$lib.base64.decode($s) ($size, $sha256)=$lib.axon.put($buf) $lib.print('size={size} sha256={sha256}', size=$size, sha256=$sha256) size=4 sha256=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08''', 'type': {'type': 'function', '_funcname': 'put', 'args': ( {'name': 'byts', 'type': 'bytes', 'desc': 'The bytes to save.', }, ), 'returns': {'type': 'list', 'desc': 'A tuple of the file size and sha256 value.', }}}, {'name': 'has', 'desc': ''' Check if the Axon the Cortex is configured to use has a given sha256 value. Examples: Check if the Axon has a given file:: # This example assumes the Axon does have the bytes cli> storm if $lib.axon.has(9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08) { $lib.print("Has bytes") } else { $lib.print("Does not have bytes") } Has bytes ''', 'type': {'type': 'function', '_funcname': 'has', 'args': ( {'name': 'sha256', 'type': 'str', 'desc': 'The sha256 value to check.', }, ), 'returns': {'type': 'boolean', 'desc': 'True if the Axon has the file, false if it does not.', }}}, {'name': 'size', 'desc': ''' Return the size of the bytes stored in the Axon for the given sha256. Examples: Get the size for a file given a variable named ``$sha256``:: $size = $lib.axon.size($sha256) ''', 'type': {'type': 'function', '_funcname': 'size', 'args': ( {'name': 'sha256', 'type': 'str', 'desc': 'The sha256 value to check.', }, ), 'returns': {'type': ['int', 'null'], 'desc': 'The size of the file or ``null`` if the file is not found.', }}}, {'name': 'hashset', 'desc': ''' Return additional hashes of the bytes stored in the Axon for the given sha256. Examples: Get the md5 hash for a file given a variable named ``$sha256``:: $hashset = $lib.axon.hashset($sha256) $md5 = $hashset.md5 ''', 'type': {'type': 'function', '_funcname': 'hashset', 'args': ( {'name': 'sha256', 'type': 'str', 'desc': 'The sha256 value to calculate hashes for.', }, ), 'returns': {'type': 'dict', 'desc': 'A dictionary of additional hashes.', }}}, {'name': 'upload', 'desc': ''' Upload a stream of bytes to the Axon as a file. Examples: Upload bytes from a generator:: ($size, $sha256) = $lib.axon.upload($getBytesChunks()) ''', 'type': {'type': 'function', '_funcname': 'upload', 'args': ( {'name': 'genr', 'type': 'generator', 'desc': 'A generator which yields bytes.', }, ), 'returns': {'type': 'list', 'desc': 'A tuple of the file size and sha256 value.', }}}, ) _storm_lib_path = ('axon',) _storm_lib_perms = ( {'perm': ('storm', 'lib', 'axon', 'del'), 'gate': 'cortex', 'desc': 'Controls the ability to remove a file from the Axon.'}, {'perm': ('storm', 'lib', 'axon', 'get'), 'gate': 'cortex', 'desc': 'Controls the ability to retrieve a file from the Axon.'}, {'perm': ('storm', 'lib', 'axon', 'has'), 'gate': 'cortex', 'desc': 'Controls the ability to check if the Axon contains a file.'}, {'perm': ('storm', 'lib', 'axon', 'wget'), 'gate': 'cortex', 'desc': 'Controls the ability to retrieve a file from URL and store it in the Axon.'}, {'perm': ('storm', 'lib', 'axon', 'wput'), 'gate': 'cortex', 'desc': 'Controls the ability to push a file from the Axon to a URL.'}, )
[docs] def getObjLocals(self): return { 'wget': self.wget, 'wput': self.wput, 'urlfile': self.urlfile, 'del': self.del_, 'dels': self.dels, 'list': self.list, 'readlines': self.readlines, 'jsonlines': self.jsonlines, 'csvrows': self.csvrows, 'metrics': self.metrics, 'put': self.put, 'has': self.has, 'size': self.size, 'upload': self.upload, 'hashset': self.hashset, }
[docs] @stormfunc(readonly=True) async def readlines(self, sha256, errors='ignore'): if not self.runt.allowed(('axon', 'get')): self.runt.confirm(('storm', 'lib', 'axon', 'get')) await self.runt.snap.core.getAxon() sha256 = await tostr(sha256) async for line in self.runt.snap.core.axon.readlines(sha256, errors=errors): yield line
[docs] @stormfunc(readonly=True) async def jsonlines(self, sha256, errors='ignore'): if not self.runt.allowed(('axon', 'get')): self.runt.confirm(('storm', 'lib', 'axon', 'get')) await self.runt.snap.core.getAxon() sha256 = await tostr(sha256) async for line in self.runt.snap.core.axon.jsonlines(sha256): yield line
[docs] async def dels(self, sha256s): if not self.runt.allowed(('axon', 'del')): self.runt.confirm(('storm', 'lib', 'axon', 'del')) sha256s = await toprim(sha256s) if not isinstance(sha256s, (list, tuple)): raise s_exc.BadArg() hashes = [s_common.uhex(s) for s in sha256s] await self.runt.snap.core.getAxon() axon = self.runt.snap.core.axon return await axon.dels(hashes)
[docs] async def del_(self, sha256): if not self.runt.allowed(('axon', 'del')): self.runt.confirm(('storm', 'lib', 'axon', 'del')) sha256 = await tostr(sha256) sha256b = s_common.uhex(sha256) await self.runt.snap.core.getAxon() axon = self.runt.snap.core.axon return await axon.del_(sha256b)
[docs] async def wget(self, url, headers=None, params=None, method='GET', json=None, body=None, ssl=True, timeout=None, proxy=True, ssl_opts=None): if not self.runt.allowed(('axon', 'wget')): self.runt.confirm(('storm', 'lib', 'axon', 'wget')) url = await tostr(url) method = await tostr(method) ssl = await tobool(ssl) body = await toprim(body) json = await toprim(json) params = await toprim(params) headers = await toprim(headers) timeout = await toprim(timeout) proxy = await toprim(proxy) ssl_opts = await toprim(ssl_opts) params = strifyHttpArg(params, multi=True) headers = strifyHttpArg(headers) await self.runt.snap.core.getAxon() kwargs = {} ok, proxy = await resolveAxonProxyArg(proxy) if ok: kwargs['proxy'] = proxy if ssl_opts is not None: axonvers = self.runt.snap.core.axoninfo['synapse']['version'] mesg = f'The ssl_opts argument requires an Axon Synapse version {AXON_MINVERS_SSLOPTS}, ' \ f'but the Axon is running {axonvers}' s_version.reqVersion(axonvers, AXON_MINVERS_SSLOPTS, mesg=mesg) kwargs['ssl_opts'] = ssl_opts axon = self.runt.snap.core.axon resp = await axon.wget(url, headers=headers, params=params, method=method, ssl=ssl, body=body, json=json, timeout=timeout, **kwargs) resp['original_url'] = url return resp
[docs] async def wput(self, sha256, url, headers=None, params=None, method='PUT', ssl=True, timeout=None, proxy=True, ssl_opts=None): if not self.runt.allowed(('axon', 'wput')): self.runt.confirm(('storm', 'lib', 'axon', 'wput')) url = await tostr(url) sha256 = await tostr(sha256) method = await tostr(method) proxy = await toprim(proxy) ssl = await tobool(ssl) params = await toprim(params) headers = await toprim(headers) timeout = await toprim(timeout) ssl_opts = await toprim(ssl_opts) params = strifyHttpArg(params, multi=True) headers = strifyHttpArg(headers) await self.runt.snap.core.getAxon() kwargs = {} ok, proxy = await resolveAxonProxyArg(proxy) if ok: kwargs['proxy'] = proxy if ssl_opts is not None: axonvers = self.runt.snap.core.axoninfo['synapse']['version'] mesg = f'The ssl_opts argument requires an Axon Synapse version {AXON_MINVERS_SSLOPTS}, ' \ f'but the Axon is running {axonvers}' s_version.reqVersion(axonvers, AXON_MINVERS_SSLOPTS, mesg=mesg) kwargs['ssl_opts'] = ssl_opts axon = self.runt.snap.core.axon sha256byts = s_common.uhex(sha256) return await axon.wput(sha256byts, url, headers=headers, params=params, method=method, ssl=ssl, timeout=timeout, **kwargs)
[docs] async def urlfile(self, *args, **kwargs): gateiden = self.runt.snap.wlyr.iden self.runt.confirm(('node', 'add', 'file:bytes'), gateiden=gateiden) self.runt.confirm(('node', 'add', 'inet:urlfile'), gateiden=gateiden) resp = await self.wget(*args, **kwargs) code = resp.get('code') if code != 200: mesg = f'$lib.axon.urlfile(): HTTP code {code}: {resp.get("reason")}' await self.runt.warn(mesg, log=False) return now = self.runt.model.type('time').norm('now')[0] original_url = resp.get('original_url') hashes = resp.get('hashes') sha256 = hashes.get('sha256') props = { 'size': resp.get('size'), 'md5': hashes.get('md5'), 'sha1': hashes.get('sha1'), 'sha256': sha256, '.seen': now, } filenode = await self.runt.snap.addNode('file:bytes', sha256, props=props) if not filenode.get('name'): info = s_urlhelp.chopurl(original_url) base = info.get('path').strip('/').split('/')[-1] if base: await filenode.set('name', base) props = {'.seen': now} urlfile = await self.runt.snap.addNode('inet:urlfile', (original_url, sha256), props=props) history = resp.get('history') if history is not None: redirs = [] src = original_url # We skip the first entry in history, since that URL is the original URL # having been redirected. The second+ history item represents the # requested URL. We then capture the last part of the chain in our list. # The recorded URLs after the original_url are all the resolved URLS, # since Location headers may be partial paths and this avoids needing to # do url introspection that has already been done by the Axon. for info in history[1:]: url = info.get('url') redirs.append((src, url)) src = url redirs.append((src, resp.get('url'))) for valu in redirs: props = {'.seen': now} await self.runt.snap.addNode('inet:urlredir', valu, props=props) return urlfile
[docs] @stormfunc(readonly=True) async def list(self, offs=0, wait=False, timeout=None): offs = await toint(offs) wait = await tobool(wait) timeout = await toint(timeout, noneok=True) if not self.runt.allowed(('axon', 'has')): self.runt.confirm(('storm', 'lib', 'axon', 'has')) await self.runt.snap.core.getAxon() axon = self.runt.snap.core.axon async for item in axon.hashes(offs, wait=wait, timeout=timeout): yield (item[0], s_common.ehex(item[1][0]), item[1][1])
[docs] @stormfunc(readonly=True) async def csvrows(self, sha256, dialect='excel', errors='ignore', **fmtparams): if not self.runt.allowed(('axon', 'get')): self.runt.confirm(('storm', 'lib', 'axon', 'get')) await self.runt.snap.core.getAxon() sha256 = await tostr(sha256) dialect = await tostr(dialect) fmtparams = await toprim(fmtparams) async for item in self.runt.snap.core.axon.csvrows(s_common.uhex(sha256), dialect, errors=errors, **fmtparams): yield item await asyncio.sleep(0)
[docs] @stormfunc(readonly=True) async def metrics(self): if not self.runt.allowed(('axon', 'has')): self.runt.confirm(('storm', 'lib', 'axon', 'has')) return await self.runt.snap.core.axon.metrics()
[docs] async def upload(self, genr): self.runt.confirm(('axon', 'upload')) await self.runt.snap.core.getAxon() async with await self.runt.snap.core.axon.upload() as upload: async for byts in s_coro.agen(genr): await upload.write(byts) size, sha256 = await upload.save() return size, s_common.ehex(sha256)
[docs] @stormfunc(readonly=True) async def has(self, sha256): sha256 = await tostr(sha256, noneok=True) if sha256 is None: return None self.runt.confirm(('axon', 'has')) await self.runt.snap.core.getAxon() return await self.runt.snap.core.axon.has(s_common.uhex(sha256))
[docs] @stormfunc(readonly=True) async def size(self, sha256): sha256 = await tostr(sha256) self.runt.confirm(('axon', 'has')) await self.runt.snap.core.getAxon() return await self.runt.snap.core.axon.size(s_common.uhex(sha256))
[docs] async def put(self, byts): if not isinstance(byts, bytes): mesg = '$lib.axon.put() requires a bytes argument' raise s_exc.BadArg(mesg=mesg) self.runt.confirm(('axon', 'upload')) await self.runt.snap.core.getAxon() size, sha256 = await self.runt.snap.core.axon.put(byts) return (size, s_common.ehex(sha256))
[docs] @stormfunc(readonly=True) async def hashset(self, sha256): sha256 = await tostr(sha256) self.runt.confirm(('axon', 'has')) await self.runt.snap.core.getAxon() return await self.runt.snap.core.axon.hashset(s_common.uhex(sha256))
[docs] @registry.registerLib class LibBytes(Lib): ''' A Storm Library for interacting with bytes storage. This Library is deprecated; use ``$lib.axon.*`` instead. ''' _storm_locals = ( {'name': 'put', 'desc': ''' Save the given bytes variable to the Axon the Cortex is configured to use. Examples: Save a base64 encoded buffer to the Axon:: cli> storm $s='dGVzdA==' $buf=$lib.base64.decode($s) ($size, $sha256)=$lib.bytes.put($buf) $lib.print('size={size} sha256={sha256}', size=$size, sha256=$sha256) size=4 sha256=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08''', 'type': {'type': 'function', '_funcname': '_libBytesPut', 'args': ( {'name': 'byts', 'type': 'bytes', 'desc': 'The bytes to save.', }, ), 'returns': {'type': 'list', 'desc': 'A tuple of the file size and sha256 value.', }}}, {'name': 'has', 'desc': ''' Check if the Axon the Cortex is configured to use has a given sha256 value. Examples: Check if the Axon has a given file:: # This example assumes the Axon does have the bytes cli> storm if $lib.bytes.has(9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08) { $lib.print("Has bytes") } else { $lib.print("Does not have bytes") } Has bytes ''', 'type': {'type': 'function', '_funcname': '_libBytesHas', 'args': ( {'name': 'sha256', 'type': 'str', 'desc': 'The sha256 value to check.', }, ), 'returns': {'type': 'boolean', 'desc': 'True if the Axon has the file, false if it does not.', }}}, {'name': 'size', 'desc': ''' Return the size of the bytes stored in the Axon for the given sha256. Examples: Get the size for a file given a variable named ``$sha256``:: $size = $lib.bytes.size($sha256) ''', 'type': {'type': 'function', '_funcname': '_libBytesSize', 'args': ( {'name': 'sha256', 'type': 'str', 'desc': 'The sha256 value to check.', }, ), 'returns': {'type': ['int', 'null'], 'desc': 'The size of the file or ``null`` if the file is not found.', }}}, {'name': 'hashset', 'desc': ''' Return additional hashes of the bytes stored in the Axon for the given sha256. Examples: Get the md5 hash for a file given a variable named ``$sha256``:: $hashset = $lib.bytes.hashset($sha256) $md5 = $hashset.md5 ''', 'type': {'type': 'function', '_funcname': '_libBytesHashset', 'args': ( {'name': 'sha256', 'type': 'str', 'desc': 'The sha256 value to calculate hashes for.', }, ), 'returns': {'type': 'dict', 'desc': 'A dictionary of additional hashes.', }}}, {'name': 'upload', 'desc': ''' Upload a stream of bytes to the Axon as a file. Examples: Upload bytes from a generator:: ($size, $sha256) = $lib.bytes.upload($getBytesChunks()) ''', 'type': {'type': 'function', '_funcname': '_libBytesUpload', 'args': ( {'name': 'genr', 'type': 'generator', 'desc': 'A generator which yields bytes.', }, ), 'returns': {'type': 'list', 'desc': 'A tuple of the file size and sha256 value.', }}}, ) _storm_lib_path = ('bytes',) _storm_lib_deprecation = {'eolvers': 'v3.0.0', 'mesg': 'Use the corresponding ``$lib.axon`` function.'}
[docs] def getObjLocals(self): return { 'put': self._libBytesPut, 'has': self._libBytesHas, 'size': self._libBytesSize, 'upload': self._libBytesUpload, 'hashset': self._libBytesHashset, }
async def _libBytesUpload(self, genr): self.runt.confirm(('axon', 'upload'), default=True) await self.runt.snap.core.getAxon() async with await self.runt.snap.core.axon.upload() as upload: async for byts in s_coro.agen(genr): await upload.write(byts) size, sha256 = await upload.save() return size, s_common.ehex(sha256) @stormfunc(readonly=True) async def _libBytesHas(self, sha256): sha256 = await tostr(sha256, noneok=True) if sha256 is None: return None self.runt.confirm(('axon', 'has'), default=True) await self.runt.snap.core.getAxon() todo = s_common.todo('has', s_common.uhex(sha256)) ret = await self.dyncall('axon', todo) return ret @stormfunc(readonly=True) async def _libBytesSize(self, sha256): sha256 = await tostr(sha256) self.runt.confirm(('axon', 'has'), default=True) await self.runt.snap.core.getAxon() todo = s_common.todo('size', s_common.uhex(sha256)) ret = await self.dyncall('axon', todo) return ret async def _libBytesPut(self, byts): if not isinstance(byts, bytes): mesg = '$lib.bytes.put() requires a bytes argument' raise s_exc.BadArg(mesg=mesg) self.runt.confirm(('axon', 'upload'), default=True) await self.runt.snap.core.getAxon() todo = s_common.todo('put', byts) size, sha2 = await self.dyncall('axon', todo) return (size, s_common.ehex(sha2)) @stormfunc(readonly=True) async def _libBytesHashset(self, sha256): sha256 = await tostr(sha256) self.runt.confirm(('axon', 'has'), default=True) await self.runt.snap.core.getAxon() todo = s_common.todo('hashset', s_common.uhex(sha256)) ret = await self.dyncall('axon', todo) return ret
[docs] @registry.registerLib class LibLift(Lib): ''' A Storm Library for interacting with lift helpers. ''' _storm_locals = ( {'name': 'byNodeData', 'desc': 'Lift nodes which have a given nodedata name set on them.', 'type': {'type': 'function', '_funcname': '_byNodeData', 'args': ( {'name': 'name', 'desc': 'The name to of the nodedata key to lift by.', 'type': 'str', }, ), 'returns': {'name': 'Yields', 'type': 'node', 'desc': 'Yields nodes to the pipeline. ' 'This must be used in conjunction with the ``yield`` keyword.', }}}, ) _storm_lib_path = ('lift',)
[docs] def getObjLocals(self): return { 'byNodeData': self._byNodeData, }
@stormfunc(readonly=True) async def _byNodeData(self, name): async for node in self.runt.snap.nodesByDataName(name): yield node
[docs] @registry.registerLib class LibTime(Lib): ''' A Storm Library for interacting with timestamps. ''' _storm_locals = ( {'name': 'now', 'desc': 'Get the current epoch time in milliseconds.', 'type': { 'type': 'function', '_funcname': '_now', 'returns': {'desc': 'Epoch time in milliseconds.', 'type': 'int', }}}, {'name': 'fromunix', 'desc': ''' Normalize a timestamp from a unix epoch time in seconds to milliseconds. Examples: Convert a timestamp from seconds to millis and format it:: cli> storm $seconds=1594684800 $millis=$lib.time.fromunix($seconds) $str=$lib.time.format($millis, '%A %d, %B %Y') $lib.print($str) Tuesday 14, July 2020''', 'type': {'type': 'function', '_funcname': '_fromunix', 'args': ( {'name': 'secs', 'type': 'int', 'desc': 'Unix epoch time in seconds.', }, ), 'returns': {'type': 'int', 'desc': 'The normalized time in milliseconds.', }}}, {'name': 'parse', 'desc': ''' Parse a timestamp string using ``datetime.strptime()`` into an epoch timestamp. Examples: Parse a string as for its month/day/year value into a timestamp:: cli> storm $s='06/01/2020' $ts=$lib.time.parse($s, '%m/%d/%Y') $lib.print($ts) 1590969600000''', 'type': {'type': 'function', '_funcname': '_parse', 'args': ( {'name': 'valu', 'type': 'str', 'desc': 'The timestamp string to parse.', }, {'name': 'format', 'type': 'str', 'desc': 'The format string to use for parsing.', }, {'name': 'errok', 'type': 'boolean', 'default': False, 'desc': 'If set, parsing errors will return ``$lib.null`` instead of raising an exception.'} ), 'returns': {'type': 'int', 'desc': 'The epoch timestamp for the string.', }}}, {'name': 'format', 'desc': ''' Format a Synapse timestamp into a string value using ``datetime.strftime()``. Examples: Format a timestamp into a string:: cli> storm $now=$lib.time.now() $str=$lib.time.format($now, '%A %d, %B %Y') $lib.print($str) Tuesday 14, July 2020''', 'type': {'type': 'function', '_funcname': '_format', 'args': ( {'name': 'valu', 'type': 'int', 'desc': 'A timestamp in epoch milliseconds.', }, {'name': 'format', 'type': 'str', 'desc': 'The strftime format string.', }, ), 'returns': {'type': 'str', 'desc': 'The formatted time string.', }}}, {'name': 'sleep', 'desc': ''' Pause the processing of data in the storm query. Notes: This has the effect of clearing the Snap's cache, so any node lifts performed after the ``$lib.time.sleep(...)`` executes will be lifted directly from storage. ''', 'type': {'type': 'function', '_funcname': '_sleep', 'args': ( {'name': 'valu', 'type': 'int', 'desc': 'The number of seconds to pause for.', }, ), 'returns': {'type': 'null', }}}, {'name': 'ticker', 'desc': ''' Periodically pause the processing of data in the storm query. Notes: This has the effect of clearing the Snap's cache, so any node lifts performed after each tick will be lifted directly from storage. ''', 'type': {'type': 'function', '_funcname': '_ticker', 'args': ( {'name': 'tick', 'desc': 'The amount of time to wait between each tick, in seconds.', 'type': 'int', }, {'name': 'count', 'default': None, 'type': 'int', 'desc': 'The number of times to pause the query before exiting the loop. ' 'This defaults to None and will yield forever if not set.', } ), 'returns': {'name': 'Yields', 'type': 'int', 'desc': 'This yields the current tick count after each time it wakes up.', }}}, {'name': 'year', 'desc': ''' Returns the year part of a time value. ''', 'type': {'type': 'function', '_funcname': 'year', 'args': ( {'name': 'tick', 'desc': 'A time value.', 'type': 'time', }, ), 'returns': {'type': 'int', 'desc': 'The year part of the time expression.', }}}, {'name': 'month', 'desc': ''' Returns the month part of a time value. ''', 'type': {'type': 'function', '_funcname': 'month', 'args': ( {'name': 'tick', 'desc': 'A time value.', 'type': 'time', }, ), 'returns': {'type': 'int', 'desc': 'The month part of the time expression.', }}}, {'name': 'day', 'desc': ''' Returns the day part of a time value. ''', 'type': {'type': 'function', '_funcname': 'day', 'args': ( {'name': 'tick', 'desc': 'A time value.', 'type': 'time', }, ), 'returns': {'type': 'int', 'desc': 'The day part of the time expression.', }}}, {'name': 'hour', 'desc': ''' Returns the hour part of a time value. ''', 'type': {'type': 'function', '_funcname': 'hour', 'args': ( {'name': 'tick', 'desc': 'A time value.', 'type': 'time', }, ), 'returns': {'type': 'int', 'desc': 'The hour part of the time expression.', }}}, {'name': 'minute', 'desc': ''' Returns the minute part of a time value. ''', 'type': {'type': 'function', '_funcname': 'minute', 'args': ( {'name': 'tick', 'desc': 'A time value.', 'type': 'time', }, ), 'returns': {'type': 'int', 'desc': 'The minute part of the time expression.', }}}, {'name': 'second', 'desc': ''' Returns the second part of a time value. ''', 'type': {'type': 'function', '_funcname': 'second', 'args': ( {'name': 'tick', 'desc': 'A time value.', 'type': 'time', }, ), 'returns': {'type': 'int', 'desc': 'The second part of the time expression.', }}}, {'name': 'dayofweek', 'desc': ''' Returns the index (beginning with monday as 0) of the day within the week. ''', 'type': {'type': 'function', '_funcname': 'dayofweek', 'args': ( {'name': 'tick', 'desc': 'A time value.', 'type': 'time', }, ), 'returns': {'type': 'int', 'desc': 'The index of the day within week.', }}}, {'name': 'dayofyear', 'desc': ''' Returns the index (beginning with 0) of the day within the year. ''', 'type': {'type': 'function', '_funcname': 'dayofyear', 'args': ( {'name': 'tick', 'desc': 'A time value.', 'type': 'time', }, ), 'returns': {'type': 'int', 'desc': 'The index of the day within year.', }}}, {'name': 'dayofmonth', 'desc': ''' Returns the index (beginning with 0) of the day within the month. ''', 'type': {'type': 'function', '_funcname': 'dayofmonth', 'args': ( {'name': 'tick', 'desc': 'A time value.', 'type': 'time', }, ), 'returns': {'type': 'int', 'desc': 'The index of the day within month.', }}}, {'name': 'monthofyear', 'desc': ''' Returns the index (beginning with 0) of the month within the year. ''', 'type': {'type': 'function', '_funcname': 'monthofyear', 'args': ( {'name': 'tick', 'desc': 'A time value.', 'type': 'time', }, ), 'returns': {'type': 'int', 'desc': 'The index of the month within year.', }}}, {'name': 'toUTC', 'desc': ''' Adjust an epoch milliseconds timestamp to UTC from the given timezone. ''', 'type': {'type': 'function', '_funcname': 'toUTC', 'args': ( {'name': 'tick', 'desc': 'A time value.', 'type': 'time'}, {'name': 'timezone', 'desc': 'A timezone name. See python pytz docs for options.', 'type': 'str'}, ), 'returns': {'type': 'list', 'desc': 'An ($ok, $valu) tuple.', }}}, ) _storm_lib_path = ('time',)
[docs] def getObjLocals(self): return { 'now': self._now, 'toUTC': self.toUTC, 'fromunix': self._fromunix, 'parse': self._parse, 'format': self._format, 'sleep': self._sleep, 'ticker': self._ticker, 'day': self.day, 'hour': self.hour, 'year': self.year, 'month': self.month, 'minute': self.minute, 'second': self.second, 'dayofweek': self.dayofweek, 'dayofyear': self.dayofyear, 'dayofmonth': self.dayofmonth, 'monthofyear': self.monthofyear, }
[docs] @stormfunc(readonly=True) async def toUTC(self, tick, timezone): tick = await toprim(tick) timezone = await tostr(timezone) timetype = self.runt.snap.core.model.type('time') norm, info = timetype.norm(tick) try: return (True, s_time.toUTC(norm, timezone)) except s_exc.BadArg as e: return (False, s_common.excinfo(e))
@stormfunc(readonly=True) def _now(self): return s_common.now()
[docs] @stormfunc(readonly=True) async def day(self, tick): tick = await toprim(tick) timetype = self.runt.snap.core.model.type('time') norm, info = timetype.norm(tick) return s_time.day(norm)
[docs] @stormfunc(readonly=True) async def hour(self, tick): tick = await toprim(tick) timetype = self.runt.snap.core.model.type('time') norm, info = timetype.norm(tick) return s_time.hour(norm)
[docs] @stormfunc(readonly=True) async def year(self, tick): tick = await toprim(tick) timetype = self.runt.snap.core.model.type('time') norm, info = timetype.norm(tick) return s_time.year(norm)
[docs] @stormfunc(readonly=True) async def month(self, tick): tick = await toprim(tick) timetype = self.runt.snap.core.model.type('time') norm, info = timetype.norm(tick) return s_time.month(norm)
[docs] @stormfunc(readonly=True) async def minute(self, tick): tick = await toprim(tick) timetype = self.runt.snap.core.model.type('time') norm, info = timetype.norm(tick) return s_time.minute(norm)
[docs] @stormfunc(readonly=True) async def second(self, tick): tick = await toprim(tick) timetype = self.runt.snap.core.model.type('time') norm, info = timetype.norm(tick) return s_time.second(norm)
[docs] @stormfunc(readonly=True) async def dayofweek(self, tick): tick = await toprim(tick) timetype = self.runt.snap.core.model.type('time') norm, info = timetype.norm(tick) return s_time.dayofweek(norm)
[docs] @stormfunc(readonly=True) async def dayofyear(self, tick): tick = await toprim(tick) timetype = self.runt.snap.core.model.type('time') norm, info = timetype.norm(tick) return s_time.dayofyear(norm)
[docs] @stormfunc(readonly=True) async def dayofmonth(self, tick): tick = await toprim(tick) timetype = self.runt.snap.core.model.type('time') norm, info = timetype.norm(tick) return s_time.dayofmonth(norm)
[docs] @stormfunc(readonly=True) async def monthofyear(self, tick): tick = await toprim(tick) timetype = self.runt.snap.core.model.type('time') norm, info = timetype.norm(tick) return s_time.month(norm) - 1
@stormfunc(readonly=True) async def _format(self, valu, format): timetype = self.runt.snap.core.model.type('time') # Give a times string a shot at being normed prior to formatting. try: norm, _ = timetype.norm(valu) except s_exc.BadTypeValu as e: mesg = f'Failed to norm a time value prior to formatting - {str(e)}' raise s_exc.StormRuntimeError(mesg=mesg, valu=valu, format=format) from None if norm == timetype.futsize: mesg = 'Cannot format a timestamp for ongoing/future time.' raise s_exc.StormRuntimeError(mesg=mesg, valu=valu, format=format) try: dt = datetime.datetime(1970, 1, 1) + datetime.timedelta(milliseconds=norm) ret = dt.strftime(format) except Exception as e: mesg = f'Error during time format - {str(e)}' raise s_exc.StormRuntimeError(mesg=mesg, valu=valu, format=format) from None return ret @stormfunc(readonly=True) async def _parse(self, valu, format, errok=False): valu = await tostr(valu) errok = await tobool(errok) try: dt = datetime.datetime.strptime(valu, format) except ValueError as e: if errok: return None mesg = f'Error during time parsing - {str(e)}' raise s_exc.StormRuntimeError(mesg=mesg, valu=valu, format=format) from None if dt.tzinfo is not None: # Convert the aware dt to UTC, then strip off the tzinfo dt = dt.astimezone(datetime.timezone.utc).replace(tzinfo=None) return int((dt - s_time.EPOCH).total_seconds() * 1000) @stormfunc(readonly=True) async def _sleep(self, valu): await self.runt.snap.waitfini(timeout=float(valu)) await self.runt.snap.clearCache() async def _ticker(self, tick, count=None): if count is not None: count = await toint(count) tick = float(tick) offs = 0 while True: await self.runt.snap.waitfini(timeout=tick) await self.runt.snap.clearCache() yield offs offs += 1 if count is not None and offs == count: break async def _fromunix(self, secs): secs = float(secs) return int(secs * 1000)
[docs] @registry.registerLib class LibRegx(Lib): ''' A Storm library for searching/matching with regular expressions. ''' _storm_locals = ( {'name': 'search', 'desc': ''' Search the given text for the pattern and return the matching groups. Note: In order to get the matching groups, patterns must use parentheses to indicate the start and stop of the regex to return portions of. If groups are not used, a successful match will return a empty list and a unsuccessful match will return ``$lib.null``. Example: Extract the matching groups from a piece of text:: $m = $lib.regex.search("^([0-9])+.([0-9])+.([0-9])+$", $text) if $m { ($maj, $min, $pat) = $m }''', 'type': {'type': 'function', '_funcname': 'search', 'args': ( {'name': 'pattern', 'type': 'str', 'desc': 'The regular expression pattern.', }, {'name': 'text', 'type': 'str', 'desc': 'The text to match.', }, {'name': 'flags', 'type': 'int', 'desc': 'Regex flags to control the match behavior.', 'default': 0}, ), 'returns': {'type': 'list', 'desc': 'A list of strings for the matching groups in the pattern.', }}}, {'name': 'findall', 'desc': ''' Search the given text for the patterns and return a list of matching strings. Note: If multiple matching groups are specified, the return value is a list of lists of strings. Example: Extract the matching strings from a piece of text:: for $x in $lib.regex.findall("G[0-9]{4}", "G0006 and G0001") { $dostuff($x) } ''', 'type': {'type': 'function', '_funcname': 'findall', 'args': ( {'name': 'pattern', 'type': 'str', 'desc': 'The regular expression pattern.', }, {'name': 'text', 'type': 'str', 'desc': 'The text to match.', }, {'name': 'flags', 'type': 'int', 'desc': 'Regex flags to control the match behavior.', 'default': 0}, ), 'returns': {'type': 'list', 'desc': 'A list of lists of strings for the matching groups in the pattern.', }}}, {'name': 'matches', 'desc': ''' Check if text matches a pattern. Returns $lib.true if the text matches the pattern, otherwise $lib.false. Notes: This API requires the pattern to match at the start of the string. Example: Check if the variable matches a expression:: if $lib.regex.matches("^[0-9]+.[0-9]+.[0-9]+$", $text) { $lib.print("It's semver! ...probably") } ''', 'type': {'type': 'function', '_funcname': 'matches', 'args': ( {'name': 'pattern', 'type': 'str', 'desc': 'The regular expression pattern.', }, {'name': 'text', 'type': 'str', 'desc': 'The text to match.', }, {'name': 'flags', 'type': 'int', 'desc': 'Regex flags to control the match behavior.', 'default': 0, }, ), 'returns': {'type': 'boolean', 'desc': 'True if there is a match, False otherwise.', }}}, {'name': 'replace', 'desc': ''' Replace any substrings that match the given regular expression with the specified replacement. Example: Replace a portion of a string with a new part based on a regex:: $norm = $lib.regex.replace("\\sAND\\s", " & ", "Ham and eggs!", $lib.regex.flags.i) ''', 'type': {'type': 'function', '_funcname': 'replace', 'args': ( {'name': 'pattern', 'type': 'str', 'desc': 'The regular expression pattern.', }, {'name': 'replace', 'type': 'str', 'desc': 'The text to replace matching sub strings.', }, {'name': 'text', 'type': 'str', 'desc': 'The input text to search/replace.', }, {'name': 'flags', 'type': 'int', 'desc': 'Regex flags to control the match behavior.', 'default': 0, }, ), 'returns': {'type': 'str', 'desc': 'The new string with matches replaced.', }}}, {'name': 'escape', 'desc': ''' Escape arbitrary strings for use in a regular expression pattern. Example: Escape node values for use in a regex pattern:: for $match in $lib.regex.findall($lib.regex.escape($node.repr()), $mydocument) { // do something with $match } Escape node values for use in regular expression filters:: it:dev:str~=$lib.regex.escape($node.repr()) ''', 'type': {'type': 'function', '_funcname': 'escape', 'args': ( {'name': 'text', 'type': 'str', 'desc': 'The text to escape.', }, ), 'returns': {'type': 'str', 'desc': 'Input string with special characters escaped.', }}}, {'name': 'flags.i', 'desc': 'Regex flag to indicate that case insensitive matches are allowed.', 'type': 'int', }, {'name': 'flags.m', 'desc': 'Regex flag to indicate that multiline matches are allowed.', 'type': 'int', }, ) _storm_lib_path = ('regex',) def __init__(self, runt, name=()): Lib.__init__(self, runt, name=name) self.compiled = {}
[docs] def getObjLocals(self): return { 'search': self.search, 'matches': self.matches, 'findall': self.findall, 'replace': self.replace, 'escape': self.escape, 'flags': {'i': regex.IGNORECASE, 'm': regex.MULTILINE, }, }
async def _getRegx(self, pattern, flags): lkey = (pattern, flags) regx = self.compiled.get(lkey) if regx is None: regx = self.compiled[lkey] = regex.compile(pattern, flags=flags) return regx
[docs] @stormfunc(readonly=True) async def replace(self, pattern, replace, text, flags=0): text = await tostr(text) flags = await toint(flags) pattern = await tostr(pattern) replace = await tostr(replace) regx = await self._getRegx(pattern, flags) return regx.sub(replace, text)
[docs] @stormfunc(readonly=True) async def matches(self, pattern, text, flags=0): text = await tostr(text) flags = await toint(flags) pattern = await tostr(pattern) regx = await self._getRegx(pattern, flags) return regx.match(text) is not None
[docs] @stormfunc(readonly=True) async def search(self, pattern, text, flags=0): text = await tostr(text) flags = await toint(flags) pattern = await tostr(pattern) regx = await self._getRegx(pattern, flags) m = regx.search(text) if m is None: return None return m.groups()
[docs] @stormfunc(readonly=True) async def findall(self, pattern, text, flags=0): text = await tostr(text) flags = await toint(flags) pattern = await tostr(pattern) regx = await self._getRegx(pattern, flags) return regx.findall(text)
[docs] @stormfunc(readonly=True) async def escape(self, text): text = await tostr(text) return regex.escape(text)
[docs] @registry.registerLib class LibCsv(Lib): ''' A Storm Library for interacting with csvtool. ''' _storm_locals = ( {'name': 'emit', 'desc': 'Emit a ``csv:row`` event to the Storm runtime for the given args.', 'type': {'type': 'function', '_funcname': '_libCsvEmit', 'args': ( {'name': '*args', 'type': 'any', 'desc': 'Items which are emitted as a ``csv:row`` event.', }, {'name': 'table', 'type': 'str', 'default': None, 'desc': 'The name of the table to emit data too. Optional.', }, ), 'returns': {'type': 'null', }}}, ) _storm_lib_path = ('csv',)
[docs] def getObjLocals(self): return { 'emit': self._libCsvEmit, }
@stormfunc(readonly=True) async def _libCsvEmit(self, *args, table=None): row = [await toprim(a) for a in args] await self.runt.snap.fire('csv:row', row=row, table=table)
[docs] @registry.registerLib class LibExport(Lib): ''' A Storm Library for exporting data. ''' _storm_lib_path = ('export',) _storm_locals = ( {'name': 'toaxon', 'desc': ''' Run a query as an export (fully resolving relationships between nodes in the output set) and save the resulting stream of packed nodes to the axon. ''', 'type': {'type': 'function', '_funcname': 'toaxon', 'args': ( {'name': 'query', 'type': 'str', 'desc': 'A query to run as an export.', }, {'name': 'opts', 'type': 'dict', 'desc': 'Storm runtime query option params.', 'default': None, }, ), 'returns': {'type': 'list', 'desc': 'Returns a tuple of (size, sha256).', }}}, )
[docs] def getObjLocals(self): return { 'toaxon': self.toaxon, }
[docs] async def toaxon(self, query, opts=None): query = await tostr(query) opts = await toprim(opts) if opts is None: opts = {} if not isinstance(opts, dict): mesg = '$lib.export.toaxon() opts argument must be a dictionary.' raise s_exc.BadArg(mesg=mesg) opts['user'] = self.runt.snap.user.iden opts.setdefault('view', self.runt.snap.view.iden) return await self.runt.snap.core.exportStormToAxon(query, opts=opts)
[docs] @registry.registerLib class LibFeed(Lib): ''' A Storm Library for interacting with Cortex feed functions. ''' _storm_locals = ( {'name': 'genr', 'desc': ''' Yield nodes being added to the graph by adding data with a given ingest type. Notes: This is using the Runtimes's Snap to call addFeedNodes(). This only yields nodes if the feed function yields nodes. If the generator is not entirely consumed there is no guarantee that all of the nodes which should be made by the feed function will be made. ''', 'type': {'type': 'function', '_funcname': '_libGenr', 'args': ( {'name': 'name', 'type': 'str', 'desc': 'Name of the ingest function to send data too.', }, {'name': 'data', 'type': 'prim', 'desc': 'Data to send to the ingest function.', }, ), 'returns': {'name': 'Yields', 'type': 'node', 'desc': 'Yields Nodes as they are created by the ingest function.', }}}, {'name': 'list', 'desc': 'Get a list of feed functions.', 'type': {'type': 'function', '_funcname': '_libList', 'returns': {'type': 'list', 'desc': 'A list of feed functions.', }}}, {'name': 'ingest', 'desc': ''' Add nodes to the graph with a given ingest type. Notes: This is using the Runtimes's Snap to call addFeedData(), after setting the snap.strict mode to False. This will cause node creation and property setting to produce warning messages, instead of causing the Storm Runtime to be torn down.''', 'type': {'type': 'function', '_funcname': '_libIngest', 'args': ( {'name': 'name', 'type': 'str', 'desc': 'Name of the ingest function to send data too.', }, {'name': 'data', 'type': 'prim', 'desc': 'Data to send to the ingest function.', }, ), 'returns': {'type': 'null', }}}, ) _storm_lib_path = ('feed',)
[docs] def getObjLocals(self): return { 'genr': self._libGenr, 'list': self._libList, 'ingest': self._libIngest, 'fromAxon': self._fromAxon, }
async def _fromAxon(self, sha256): ''' Use the feed API to load a syn.nodes formatted export from the axon. Args: sha256 (str): The sha256 of the file saved in the axon. Returns: int: The number of nodes loaded. ''' sha256 = await tostr(sha256) opts = { 'user': self.runt.snap.user.iden, 'view': self.runt.snap.view.iden, } return await self.runt.snap.core.feedFromAxon(sha256, opts=opts) async def _libGenr(self, name, data): name = await tostr(name) data = await toprim(data) self.runt.layerConfirm(('feed:data', *name.split('.'))) # small work around for the feed API consistency if name == 'syn.nodes': async for node in self.runt.snap.addNodes(data): yield node return await self.runt.snap.addFeedData(name, data) @stormfunc(readonly=True) async def _libList(self): todo = ('getFeedFuncs', (), {}) return await self.runt.dyncall('cortex', todo) async def _libIngest(self, name, data): name = await tostr(name) data = await toprim(data) self.runt.layerConfirm(('feed:data', *name.split('.'))) # TODO this should be a reentrent safe with block strict = self.runt.snap.strict self.runt.snap.strict = False await self.runt.snap.addFeedData(name, data) self.runt.snap.strict = strict
[docs] @registry.registerLib class LibPipe(Lib): ''' A Storm library for interacting with non-persistent queues. ''' _storm_locals = ( {'name': 'gen', 'desc': ''' Generate and return a Storm Pipe. Notes: The filler query is run in parallel with $pipe. This requires the permission ``storm.pipe.gen`` to use. Examples: Fill a pipe with a query and consume it with another:: $pipe = $lib.pipe.gen(${ $pipe.puts((1, 2, 3)) }) for $items in $pipe.slices(size=2) { $dostuff($items) } ''', 'type': {'type': 'function', '_funcname': '_methPipeGen', 'args': ( {'name': 'filler', 'type': ['str', 'storm:query'], 'desc': 'A Storm query to fill the Pipe.', }, {'name': 'size', 'type': 'int', 'default': 10000, 'desc': 'Maximum size of the pipe.', }, ), 'returns': {'type': 'pipe', 'desc': 'The pipe containing query results.', }}}, ) _storm_lib_path = ('pipe',)
[docs] def getObjLocals(self): return { 'gen': self._methPipeGen, }
@stormfunc(readonly=True) async def _methPipeGen(self, filler, size=10000): size = await toint(size) text = await tostr(filler) if size < 1 or size > 10000: mesg = '$lib.pipe.gen() size must be 1-10000' raise s_exc.BadArg(mesg=mesg) pipe = Pipe(self.runt, size) opts = {'vars': {'pipe': pipe}} query = await self.runt.getStormQuery(text) async def coro(): try: async with self.runt.getSubRuntime(query, opts=opts) as runt: async for item in runt.execute(): await asyncio.sleep(0) except asyncio.CancelledError: # pragma: no cover raise except Exception as e: await self.runt.warn(f'pipe filler error: {e}', log=False) await pipe.close() self.runt.snap.schedCoro(coro()) return pipe
[docs] @registry.registerType class Pipe(StormType): ''' A Storm Pipe provides fast ephemeral queues. ''' _storm_locals = ( {'name': 'put', 'desc': 'Add a single item to the Pipe.', 'type': {'type': 'function', '_funcname': '_methPipePut', 'args': ( {'name': 'item', 'type': 'any', 'desc': 'An object to add to the Pipe.', }, ), 'returns': {'type': 'null', }}}, {'name': 'puts', 'desc': 'Add a list of items to the Pipe.', 'type': {'type': 'function', '_funcname': '_methPipePuts', 'args': ( {'name': 'items', 'type': 'list', 'desc': 'A list of items to add.', }, ), 'returns': {'type': 'null', }}}, {'name': 'slice', 'desc': 'Return a list of up to size items from the Pipe.', 'type': {'type': 'function', '_funcname': '_methPipeSlice', 'args': ( {'name': 'size', 'type': 'int', 'default': 1000, 'desc': 'The max number of items to return.', }, ), 'returns': {'type': 'list', 'desc': 'A list of at least 1 item from the Pipe.', }}}, {'name': 'slices', 'desc': ''' Yield lists of up to size items from the Pipe. Notes: The loop will exit when the Pipe is closed and empty. Examples: Operation on slices from a pipe one at a time:: for $slice in $pipe.slices(1000) { for $item in $slice { $dostuff($item) } } Operate on slices from a pipe in bulk:: for $slice in $pipe.slices(1000) { $dostuff_batch($slice) }''', 'type': {'type': 'function', '_funcname': '_methPipeSlices', 'args': ( {'name': 'size', 'type': 'int', 'default': 1000, 'desc': 'The max number of items to yield per slice.', }, ), 'returns': {'name': 'Yields', 'type': 'any', 'desc': 'Yields objects from the Pipe.', }}}, {'name': 'size', 'desc': 'Retrieve the number of items in the Pipe.', 'type': {'type': 'function', '_funcname': '_methPipeSize', 'returns': {'type': 'int', 'desc': 'The number of items in the Pipe.', }}}, ) _storm_typename = 'pipe' def __init__(self, runt, size): StormType.__init__(self) self.runt = runt self.locls.update(self.getObjLocals()) self.queue = s_queue.Queue(maxsize=size)
[docs] def getObjLocals(self): return { 'put': self._methPipePut, 'puts': self._methPipePuts, 'slice': self._methPipeSlice, 'slices': self._methPipeSlices, 'size': self._methPipeSize, }
@stormfunc(readonly=True) async def _methPipePuts(self, items): items = await toprim(items) return await self.queue.puts(items) @stormfunc(readonly=True) async def _methPipePut(self, item): item = await toprim(item) return await self.queue.put(item)
[docs] async def close(self): ''' Close the pipe for writing. This will cause the slice()/slices() API to return once drained. ''' await self.queue.close()
@stormfunc(readonly=True) async def _methPipeSize(self): return await self.queue.size() @stormfunc(readonly=True) async def _methPipeSlice(self, size=1000): size = await toint(size) if size < 1 or size > 10000: mesg = '$pipe.slice() size must be 1-10000' raise s_exc.BadArg(mesg=mesg) items = await self.queue.slice(size=size) if items is None: return None return List(items) @stormfunc(readonly=True) async def _methPipeSlices(self, size=1000): size = await toint(size) if size < 1 or size > 10000: mesg = '$pipe.slice() size must be 1-10000' raise s_exc.BadArg(mesg=mesg) async for items in self.queue.slices(size=size): yield List(items)
[docs] @registry.registerLib class LibQueue(Lib): ''' A Storm Library for interacting with persistent Queues in the Cortex. ''' _storm_locals = ( {'name': 'add', 'desc': 'Add a Queue to the Cortex with a given name.', 'type': {'type': 'function', '_funcname': '_methQueueAdd', 'args': ( {'name': 'name', 'type': 'str', 'desc': 'The name of the queue to add.', }, ), 'returns': {'type': 'queue', }}}, {'name': 'gen', 'desc': 'Add or get a Storm Queue in a single operation.', 'type': {'type': 'function', '_funcname': '_methQueueGen', 'args': ( {'name': 'name', 'type': 'str', 'desc': 'The name of the Queue to add or get.', }, ), 'returns': {'type': 'queue', }}}, {'name': 'del', 'desc': 'Delete a given named Queue.', 'type': {'type': 'function', '_funcname': '_methQueueDel', 'args': ( {'name': 'name', 'type': 'str', 'desc': 'The name of the queue to delete.', }, ), 'returns': {'type': 'null', }}}, {'name': 'get', 'desc': 'Get an existing Storm Queue object.', 'type': {'type': 'function', '_funcname': '_methQueueGet', 'args': ( {'name': 'name', 'type': 'str', 'desc': 'The name of the Queue to get.', }, ), 'returns': {'type': 'queue', 'desc': 'A ``queue`` object.', }}}, {'name': 'list', 'desc': 'Get a list of the Queues in the Cortex.', 'type': {'type': 'function', '_funcname': '_methQueueList', 'returns': {'type': 'list', 'desc': 'A list of queue definitions the current user is allowed to interact with.', }}}, ) _storm_lib_path = ('queue',)
[docs] def getObjLocals(self): return { 'add': self._methQueueAdd, 'gen': self._methQueueGen, 'del': self._methQueueDel, 'get': self._methQueueGet, 'list': self._methQueueList, }
async def _methQueueAdd(self, name): info = { 'time': s_common.now(), 'creator': self.runt.snap.user.iden, } todo = s_common.todo('addCoreQueue', name, info) gatekeys = ((self.runt.user.iden, ('queue', 'add'), None),) info = await self.dyncall('cortex', todo, gatekeys=gatekeys) return Queue(self.runt, name, info) @stormfunc(readonly=True) async def _methQueueGet(self, name): todo = s_common.todo('getCoreQueue', name) gatekeys = ((self.runt.user.iden, ('queue', 'get'), f'queue:{name}'),) info = await self.dyncall('cortex', todo, gatekeys=gatekeys) return Queue(self.runt, name, info) async def _methQueueGen(self, name): try: return await self._methQueueGet(name) except s_exc.NoSuchName: return await self._methQueueAdd(name) async def _methQueueDel(self, name): todo = s_common.todo('delCoreQueue', name) gatekeys = ((self.runt.user.iden, ('queue', 'del',), f'queue:{name}'), ) await self.dyncall('cortex', todo, gatekeys=gatekeys) @stormfunc(readonly=True) async def _methQueueList(self): retn = [] todo = s_common.todo('listCoreQueues') qlist = await self.dyncall('cortex', todo) for queue in qlist: if not allowed(('queue', 'get'), f"queue:{queue['name']}"): continue retn.append(queue) return retn
[docs] @registry.registerType class Queue(StormType): ''' A StormLib API instance of a named channel in the Cortex multiqueue. ''' _storm_locals = ( {'name': 'name', 'desc': 'The name of the Queue.', 'type': 'str', }, {'name': 'get', 'desc': 'Get a particular item from the Queue.', 'type': {'type': 'function', '_funcname': '_methQueueGet', 'args': ( {'name': 'offs', 'type': 'int', 'desc': 'The offset to retrieve an item from.', 'default': 0, }, {'name': 'cull', 'type': 'boolean', 'default': True, 'desc': 'Culls items up to, but not including, the specified offset.', }, {'name': 'wait', 'type': 'boolean', 'default': True, 'desc': 'Wait for the offset to be available before returning the item.', }, ), 'returns': {'type': 'list', 'desc': 'A tuple of the offset and the item from the queue. If wait is false and ' 'the offset is not present, null is returned.', }}}, {'name': 'pop', 'desc': 'Pop a item from the Queue at a specific offset.', 'type': {'type': 'function', '_funcname': '_methQueuePop', 'args': ( {'name': 'offs', 'type': 'int', 'default': None, 'desc': 'Offset to pop the item from. If not specified, the first item in the queue will be' ' popped.', }, {'name': 'wait', 'type': 'boolean', 'default': False, 'desc': 'Wait for an item to be available to pop.'}, ), 'returns': {'type': 'list', 'desc': 'The offset and item popped from the queue. If there is no item at the ' 'offset or the queue is empty and wait is false, it returns null.', }}}, {'name': 'put', 'desc': 'Put an item into the queue.', 'type': {'type': 'function', '_funcname': '_methQueuePut', 'args': ( {'name': 'item', 'type': 'prim', 'desc': 'The item being put into the queue.', }, ), 'returns': {'type': 'int', 'desc': 'The queue offset of the item.'}}}, {'name': 'puts', 'desc': 'Put multiple items into the Queue.', 'type': {'type': 'function', '_funcname': '_methQueuePuts', 'args': ( {'name': 'items', 'type': 'list', 'desc': 'The items to put into the Queue.', }, ), 'returns': {'type': 'int', 'desc': 'The queue offset of the first item.'}}}, {'name': 'gets', 'desc': 'Get multiple items from the Queue as a iterator.', 'type': {'type': 'function', '_funcname': '_methQueueGets', 'args': ( {'name': 'offs', 'type': 'int', 'desc': 'The offset to retrieve an items from.', 'default': 0, }, {'name': 'wait', 'type': 'boolean', 'default': True, 'desc': 'Wait for the offset to be available before returning the item.', }, {'name': 'cull', 'type': 'boolean', 'default': False, 'desc': 'Culls items up to, but not including, the specified offset.', }, {'name': 'size', 'type': 'int', 'desc': 'The maximum number of items to yield', 'default': None, }, ), 'returns': {'name': 'Yields', 'type': 'list', 'desc': 'Yields tuples of the offset and item.', }}}, {'name': 'cull', 'desc': 'Remove items from the queue up to, and including, the offset.', 'type': {'type': 'function', '_funcname': '_methQueueCull', 'args': ( {'name': 'offs', 'type': 'int', 'desc': 'The offset which to cull records from the queue.', }, ), 'returns': {'type': 'null', }}}, {'name': 'size', 'desc': 'Get the number of items in the Queue.', 'type': {'type': 'function', '_funcname': '_methQueueSize', 'returns': {'type': 'int', 'desc': 'The number of items in the Queue.', }}}, ) _storm_typename = 'queue' _ismutable = False def __init__(self, runt, name, info): StormType.__init__(self) self.runt = runt self.name = name self.info = info self.gateiden = f'queue:{name}' self.locls.update(self.getObjLocals()) self.locls['name'] = self.name def __hash__(self): return hash((self._storm_typename, self.name)) def __eq__(self, othr): if not isinstance(othr, type(self)): return False return self.name == othr.name
[docs] def getObjLocals(self): return { 'get': self._methQueueGet, 'pop': self._methQueuePop, 'put': self._methQueuePut, 'puts': self._methQueuePuts, 'gets': self._methQueueGets, 'cull': self._methQueueCull, 'size': self._methQueueSize, }
async def _methQueueCull(self, offs): offs = await toint(offs) gatekeys = self._getGateKeys('get') await self.runt.reqGateKeys(gatekeys) await self.runt.snap.core.coreQueueCull(self.name, offs) @stormfunc(readonly=True) async def _methQueueSize(self): gatekeys = self._getGateKeys('get') await self.runt.reqGateKeys(gatekeys) return await self.runt.snap.core.coreQueueSize(self.name) async def _methQueueGets(self, offs=0, wait=True, cull=False, size=None): wait = await toint(wait) offs = await toint(offs) size = await toint(size, noneok=True) gatekeys = self._getGateKeys('get') await self.runt.reqGateKeys(gatekeys) async for item in self.runt.snap.core.coreQueueGets(self.name, offs, cull=cull, wait=wait, size=size): yield item async def _methQueuePuts(self, items): items = await toprim(items) gatekeys = self._getGateKeys('put') await self.runt.reqGateKeys(gatekeys) return await self.runt.snap.core.coreQueuePuts(self.name, items) async def _methQueueGet(self, offs=0, cull=True, wait=True): offs = await toint(offs) wait = await toint(wait) gatekeys = self._getGateKeys('get') await self.runt.reqGateKeys(gatekeys) return await self.runt.snap.core.coreQueueGet(self.name, offs, cull=cull, wait=wait) async def _methQueuePop(self, offs=None, wait=False): offs = await toint(offs, noneok=True) wait = await tobool(wait) gatekeys = self._getGateKeys('get') await self.runt.reqGateKeys(gatekeys) # emulate the old behavior on no argument core = self.runt.snap.core if offs is None: async for item in core.coreQueueGets(self.name, 0, wait=wait): return await core.coreQueuePop(self.name, item[0]) return return await core.coreQueuePop(self.name, offs) async def _methQueuePut(self, item): return await self._methQueuePuts((item,)) def _getGateKeys(self, perm): return ((self.runt.user.iden, ('queue', perm), self.gateiden),)
[docs] async def stormrepr(self): return f'{self._storm_typename}: {self.name}'
[docs] @registry.registerLib class LibTelepath(Lib): ''' A Storm Library for making Telepath connections to remote services. ''' _storm_locals = ( {'name': 'open', 'desc': 'Open and return a Telepath RPC proxy.', 'type': {'type': 'function', '_funcname': '_methTeleOpen', 'args': ( {'name': 'url', 'type': 'str', 'desc': 'The Telepath URL to connect to.', }, ), 'returns': {'type': 'telepath:proxy', 'desc': 'A object representing a Telepath Proxy.', }}}, ) _storm_lib_path = ('telepath',) _storm_lib_perms = ( {'perm': ('storm', 'lib', 'telepath', 'open'), 'gate': 'cortex', 'desc': 'Controls the ability to open an arbitrary telepath URL. USE WITH CAUTION.'}, {'perm': ('storm', 'lib', 'telepath', 'open', '<scheme>'), 'gate': 'cortex', 'desc': 'Controls the ability to open a telepath URL with a specific URI scheme. USE WITH CAUTION.'}, )
[docs] def getObjLocals(self): return { 'open': self._methTeleOpen, }
async def _methTeleOpen(self, url): url = await tostr(url) scheme = url.split('://')[0] if not self.runt.allowed(('lib', 'telepath', 'open', scheme)): self.runt.confirm(('storm', 'lib', 'telepath', 'open', scheme)) try: return Proxy(self.runt, await self.runt.getTeleProxy(url)) except s_exc.SynErr: raise except Exception as e: mesg = f'Failed to connect to Telepath service: "{s_urlhelp.sanitizeUrl(url)}" error: {str(e)}' raise s_exc.StormRuntimeError(mesg=mesg) from e
[docs] @registry.registerType class Proxy(StormType): ''' Implements the Storm API for a Telepath proxy. These can be created via ``$lib.telepath.open()``. Storm Service objects are also Telepath proxy objects. Methods called off of these objects are executed like regular Telepath RMI calls. An example of calling a method which returns data:: $prox = $lib.telepath.open($url) $result = $prox.doWork($data) return ( $result ) An example of calling a method which is a generator:: $prox = $lib.telepath.open($url) for $item in $prox.genrStuff($data) { $doStuff($item) } ''' _storm_typename = 'telepath:proxy' def __init__(self, runt, proxy, path=None): StormType.__init__(self, path=path) self.runt = runt self.proxy = proxy
[docs] async def deref(self, name): name = await tostr(name) if name[0] == '_': mesg = f'No proxy method named {name}' raise s_exc.NoSuchName(mesg=mesg, name=name) meth = getattr(self.proxy, name, None) if isinstance(meth, s_telepath.GenrMethod): return ProxyGenrMethod(meth) if isinstance(meth, s_telepath.Method): return ProxyMethod(self.runt, meth)
[docs] async def stormrepr(self): return f'{self._storm_typename}: {self.proxy}'
[docs] @registry.registerType class ProxyMethod(StormType): ''' Implements the call methods for the telepath:proxy. An example of calling a method which returns data:: $prox = $lib.telepath.open($url) $result = $prox.doWork($data) $doStuff($result) ''' _storm_typename = 'telepath:proxy:method' def __init__(self, runt, meth, path=None): StormType.__init__(self, path=path) self.runt = runt self.meth = meth async def __call__(self, *args, **kwargs): args = await toprim(args) kwargs = await toprim(kwargs) # TODO: storm types fromprim() ret = await self.meth(*args, **kwargs) if isinstance(ret, s_telepath.Share): self.runt.snap.onfini(ret) return Proxy(self.runt, ret) return ret
[docs] async def stormrepr(self): return f'{self._storm_typename}: {self.meth}'
[docs] @registry.registerType class ProxyGenrMethod(StormType): ''' Implements the generator methods for the telepath:proxy. An example of calling a method which is a generator:: $prox = $lib.telepath.open($url) for $item in $prox.genrStuff($data) { $doStuff($item) } ''' _storm_typename = 'telepath:proxy:genrmethod' def __init__(self, meth, path=None): StormType.__init__(self, path=path) self.meth = meth async def __call__(self, *args, **kwargs): args = await toprim(args) kwargs = await toprim(kwargs) async for prim in self.meth(*args, **kwargs): # TODO: storm types fromprim() yield prim
[docs] async def stormrepr(self): return f'{self._storm_typename}: {self.meth}'
# @registry.registerType
[docs] class Service(Proxy): def __init__(self, runt, ssvc): Proxy.__init__(self, runt, ssvc.proxy) self.name = ssvc.name
[docs] async def deref(self, name): name = await tostr(name) try: await self.proxy.waitready() return await Proxy.deref(self, name) except asyncio.TimeoutError: mesg = f'Timeout waiting for storm service {self.name}.{name}' raise s_exc.StormRuntimeError(mesg=mesg, name=name, service=self.name) from None except AttributeError as e: # pragma: no cover # possible client race condition seen in the real world mesg = f'Error dereferencing storm service - {self.name}.{name} - {str(e)}' raise s_exc.StormRuntimeError(mesg=mesg, name=name, service=self.name) from None
[docs] @registry.registerLib class LibBase64(Lib): ''' A Storm Library for encoding and decoding base64 data. ''' _storm_locals = ( {'name': 'encode', 'desc': 'Encode a bytes object to a base64 encoded string.', 'type': {'type': 'function', '_funcname': '_encode', 'args': ( {'name': 'valu', 'type': 'bytes', 'desc': 'The object to encode.', }, {'name': 'urlsafe', 'type': 'boolean', 'default': True, 'desc': 'Perform the encoding in a urlsafe manner if true.', }, ), 'returns': {'type': 'str', 'desc': 'A base64 encoded string.', }}}, {'name': 'decode', 'desc': 'Decode a base64 string into a bytes object.', 'type': {'type': 'function', '_funcname': '_decode', 'args': ( {'name': 'valu', 'type': 'str', 'desc': 'The string to decode.', }, {'name': 'urlsafe', 'type': 'boolean', 'default': True, 'desc': 'Perform the decoding in a urlsafe manner if true.', }, ), 'returns': {'type': 'bytes', 'desc': 'A bytes object for the decoded data.', }}}, ) _storm_lib_path = ('base64',)
[docs] def getObjLocals(self): return { 'encode': self._encode, 'decode': self._decode }
@stormfunc(readonly=True) async def _encode(self, valu, urlsafe=True): try: if urlsafe: return base64.urlsafe_b64encode(valu).decode('ascii') return base64.b64encode(valu).decode('ascii') except TypeError as e: mesg = f'Error during base64 encoding - {str(e)}: {s_common.trimText(repr(valu))}' raise s_exc.StormRuntimeError(mesg=mesg, urlsafe=urlsafe) from None @stormfunc(readonly=True) async def _decode(self, valu, urlsafe=True): try: if urlsafe: return base64.urlsafe_b64decode(valu) return base64.b64decode(valu) except (binascii.Error, TypeError) as e: mesg = f'Error during base64 decoding - {str(e)}: {s_common.trimText(repr(valu))}' raise s_exc.StormRuntimeError(mesg=mesg, urlsafe=urlsafe) from None
[docs] @functools.total_ordering class Prim(StormType): ''' The base type for all Storm primitive values. ''' def __init__(self, valu, path=None): StormType.__init__(self, path=path) self.valu = valu def __int__(self): mesg = 'Storm type {__class__.__name__.lower()} cannot be cast to an int' raise s_exc.BadCast(mesg) def __len__(self): name = f'{self.__class__.__module__}.{self.__class__.__name__}' raise s_exc.StormRuntimeError(mesg=f'Object {name} does not have a length.', name=name) def __eq__(self, othr): if not isinstance(othr, type(self)): return False return self.valu == othr.valu def __lt__(self, other): if not isinstance(other, type(self)): mesg = f"'<' not supported between instance of {self.__class__.__name__} and {other.__class__.__name__}" raise TypeError(mesg) return self.valu < other.valu
[docs] def value(self): return self.valu
[docs] async def iter(self): # pragma: no cover for x in (): yield x name = f'{self.__class__.__module__}.{self.__class__.__name__}' raise s_exc.StormRuntimeError(mesg=f'Object {name} is not iterable.', name=name)
[docs] async def nodes(self): # pragma: no cover for x in (): yield x
[docs] async def bool(self): return bool(await s_coro.ornot(self.value))
[docs] async def stormrepr(self): # pragma: no cover return f'{self._storm_typename}: {await s_coro.ornot(self.value)}'
[docs] @registry.registerType class Str(Prim): ''' Implements the Storm API for a String object. ''' _storm_locals = ( {'name': 'split', 'desc': ''' Split the string into multiple parts based on a separator. Example: Split a string on the colon character:: ($foo, $bar) = $baz.split(":")''', 'type': {'type': 'function', '_funcname': '_methStrSplit', 'args': ( {'name': 'text', 'type': 'str', 'desc': 'The text to split the string up with.', }, {'name': 'maxsplit', 'type': 'int', 'default': -1, 'desc': 'The max number of splits.', }, ), 'returns': {'type': 'list', 'desc': 'A list of parts representing the split string.', }}}, {'name': 'rsplit', 'desc': ''' Split the string into multiple parts, from the right, based on a separator. Example: Split a string on the colon character:: ($foo, $bar) = $baz.rsplit(":", maxsplit=1)''', 'type': {'type': 'function', '_funcname': '_methStrRsplit', 'args': ( {'name': 'text', 'type': 'str', 'desc': 'The text to split the string up with.', }, {'name': 'maxsplit', 'type': 'int', 'default': -1, 'desc': 'The max number of splits.', }, ), 'returns': {'type': 'list', 'desc': 'A list of parts representing the split string.', }}}, {'name': 'endswith', 'desc': 'Check if a string ends with text.', 'type': {'type': 'function', '_funcname': '_methStrEndswith', 'args': ( {'name': 'text', 'type': 'str', 'desc': 'The text to check.', }, ), 'returns': {'type': 'boolean', 'desc': 'True if the text ends with the string, false otherwise.', }}}, {'name': 'startswith', 'desc': 'Check if a string starts with text.', 'type': {'type': 'function', '_funcname': '_methStrStartswith', 'args': ( {'name': 'text', 'type': 'str', 'desc': 'The text to check.', }, ), 'returns': {'type': 'boolean', 'desc': 'True if the text starts with the string, false otherwise.', }}}, {'name': 'ljust', 'desc': 'Left justify the string.', 'type': {'type': 'function', '_funcname': '_methStrLjust', 'args': ( {'name': 'size', 'type': 'int', 'desc': 'The length of character to left justify.', }, {'name': 'fillchar', 'type': 'str', 'default': ' ', 'desc': 'The character to use for padding.', }, ), 'returns': {'type': 'str', 'desc': 'The left justified string.', }}}, {'name': 'rjust', 'desc': 'Right justify the string.', 'type': {'type': 'function', '_funcname': '_methStrRjust', 'args': ( {'name': 'size', 'type': 'int', 'desc': 'The length of character to right justify.', }, {'name': 'fillchar', 'type': 'str', 'default': ' ', 'desc': 'The character to use for padding.', }, ), 'returns': {'type': 'str', 'desc': 'The right justified string.', }}}, {'name': 'encode', 'desc': 'Encoding a string value to bytes.', 'type': {'type': 'function', '_funcname': '_methEncode', 'args': ( {'name': 'encoding', 'type': 'str', 'desc': 'Encoding to use. Defaults to utf8.', 'default': 'utf8', }, ), 'returns': {'type': 'bytes', 'desc': 'The encoded string.', }}}, {'name': 'replace', 'desc': ''' Replace occurrences of a string with a new string, optionally restricting the number of replacements. Example: Replace instances of the string "bar" with the string "baz":: $foo.replace('bar', 'baz')''', 'type': {'type': 'function', '_funcname': '_methStrReplace', 'args': ( {'name': 'oldv', 'type': 'str', 'desc': 'The value to replace.', }, {'name': 'newv', 'type': 'str', 'desc': 'The value to add into the string.', }, {'name': 'maxv', 'type': 'int', 'desc': 'The maximum number of occurrences to replace.', 'default': None, }, ), 'returns': {'type': 'str', 'desc': 'The new string with replaced instances.', }}}, {'name': 'strip', 'desc': ''' Remove leading and trailing characters from a string. Examples: Removing whitespace and specific characters:: $strippedFoo = $foo.strip() $strippedBar = $bar.strip(asdf)''', 'type': {'type': 'function', '_funcname': '_methStrStrip', 'args': ( {'name': 'chars', 'type': 'str', 'default': None, 'desc': 'A list of characters to remove. If not specified, whitespace is stripped.', }, ), 'returns': {'type': 'str', 'desc': 'The stripped string.', }}}, {'name': 'lstrip', 'desc': ''' Remove leading characters from a string. Examples: Removing whitespace and specific characters:: $strippedFoo = $foo.lstrip() $strippedBar = $bar.lstrip(w)''', 'type': {'type': 'function', '_funcname': '_methStrLstrip', 'args': ( {'name': 'chars', 'type': 'str', 'default': None, 'desc': 'A list of characters to remove. If not specified, whitespace is stripped.', }, ), 'returns': {'type': 'str', 'desc': 'The stripped string.', }}}, {'name': 'rstrip', 'desc': ''' Remove trailing characters from a string. Examples: Removing whitespace and specific characters:: $strippedFoo = $foo.rstrip() $strippedBar = $bar.rstrip(asdf) ''', 'type': {'type': 'function', '_funcname': '_methStrRstrip', 'args': ( {'name': 'chars', 'type': 'str', 'default': None, 'desc': 'A list of characters to remove. If not specified, whitespace is stripped.', }, ), 'returns': {'type': 'str', 'desc': 'The stripped string.', }}}, {'name': 'lower', 'desc': ''' Get a lowercased copy of the string. Examples: Printing a lowercased string:: $foo="Duck" $lib.print($foo.lower())''', 'type': {'type': 'function', '_funcname': '_methStrLower', 'returns': {'type': 'str', 'desc': 'The lowercased string.', }}}, {'name': 'upper', 'desc': ''' Get a uppercased copy of the string. Examples: Printing a uppercased string:: $foo="Duck" $lib.print($foo.upper())''', 'type': {'type': 'function', '_funcname': '_methStrUpper', 'returns': {'type': 'str', 'desc': 'The uppercased string.', }}}, {'name': 'title', 'desc': ''' Get a title cased copy of the string. Examples: Printing a title cased string:: $foo="Hello world." $lib.print($foo.title())''', 'type': {'type': 'function', '_funcname': '_methStrTitle', 'returns': {'type': 'str', 'desc': 'The title cased string.', }}}, {'name': 'slice', 'desc': ''' Get a substring slice of the string. Examples: Slice from index to 1 to 5:: $x="foobar" $y=$x.slice(1,5) // "ooba" Slice from index 3 to the end of the string:: $y=$x.slice(3) // "bar" ''', 'type': {'type': 'function', '_funcname': '_methStrSlice', 'args': ( {'name': 'start', 'type': 'int', 'desc': 'The starting character index.'}, {'name': 'end', 'type': 'int', 'default': None, 'desc': 'The ending character index. If not specified, slice to the end of the string'}, ), 'returns': {'type': 'str', 'desc': 'The slice substring.'}}}, {'name': 'reverse', 'desc': ''' Get a reversed copy of the string. Examples: Printing a reversed string:: $foo="foobar" $lib.print($foo.reverse())''', 'type': {'type': 'function', '_funcname': '_methStrReverse', 'returns': {'type': 'str', 'desc': 'The reversed string.', }}}, {'name': 'find', 'desc': ''' Find the offset of a given string within another. Examples: Find values in the string ``asdf``:: $x = asdf $x.find(d) // returns 2 $x.find(v) // returns null ''', 'type': {'type': 'function', '_funcname': '_methStrFind', 'args': ( {'name': 'valu', 'type': 'str', 'desc': 'The substring to find.'}, ), 'returns': {'type': 'int', 'desc': 'The first offset of substring or null.'}}}, {'name': 'size', 'desc': 'Return the length of the string.', 'type': {'type': 'function', '_funcname': '_methStrSize', 'returns': {'type': 'int', 'desc': 'The size of the string.', }}}, {'name': 'format', 'desc': ''' Format a text string from an existing string. Examples: Format a string with a fixed argument and a variable:: $template='Hello {name}, list is {list}!' $list=(1,2,3,4) $new=$template.format(name='Reader', list=$list) ''', 'type': {'type': 'function', '_funcname': '_methStrFormat', 'args': ( {'name': '**kwargs', 'type': 'any', 'desc': 'Keyword values which are substituted into the string.', }, ), 'returns': {'type': 'str', 'desc': 'The new string.', }}}, {'name': 'json', 'desc': 'Parse a JSON string and return the deserialized data.', 'type': {'type': 'function', '_funcname': '_methStrJson', 'args': (), 'returns': {'type': 'prim', 'desc': 'The JSON deserialized object.', }}}, ) _storm_typename = 'str' _ismutable = False def __init__(self, valu, path=None): Prim.__init__(self, valu, path=path) self.locls.update(self.getObjLocals())
[docs] def getObjLocals(self): return { 'find': self._methStrFind, 'size': self._methStrSize, 'split': self._methStrSplit, 'rsplit': self._methStrRsplit, 'endswith': self._methStrEndswith, 'startswith': self._methStrStartswith, 'ljust': self._methStrLjust, 'rjust': self._methStrRjust, 'encode': self._methEncode, 'replace': self._methStrReplace, 'strip': self._methStrStrip, 'lstrip': self._methStrLstrip, 'rstrip': self._methStrRstrip, 'lower': self._methStrLower, 'upper': self._methStrUpper, 'title': self._methStrTitle, 'slice': self._methStrSlice, 'reverse': self._methStrReverse, 'format': self._methStrFormat, 'json': self._methStrJson, }
def __int__(self): return int(self.value(), 0) def __str__(self): return self.value() def __len__(self): return len(self.valu) def __hash__(self): # As a note, this hash of the typename and the value means that s_stormtypes.Str('foo') != 'foo' return hash((self._storm_typename, self.valu)) def __eq__(self, othr): if isinstance(othr, (Str, str)): return str(self) == str(othr) return False @stormfunc(readonly=True) async def _methStrFind(self, valu): text = await tostr(valu) retn = self.valu.find(text) if retn == -1: retn = None return retn @stormfunc(readonly=True) async def _methStrFormat(self, **kwargs): text = await kwarg_format(self.valu, **kwargs) return text @stormfunc(readonly=True) async def _methStrSize(self): return len(self.valu) @stormfunc(readonly=True) async def _methEncode(self, encoding='utf8'): try: return self.valu.encode(encoding, 'surrogatepass') except UnicodeEncodeError as e: raise s_exc.StormRuntimeError(mesg=f'{e}: {s_common.trimText(repr(self.valu))}') from None @stormfunc(readonly=True) async def _methStrSplit(self, text, maxsplit=-1): maxsplit = await toint(maxsplit) return self.valu.split(text, maxsplit=maxsplit) @stormfunc(readonly=True) async def _methStrRsplit(self, text, maxsplit=-1): maxsplit = await toint(maxsplit) return self.valu.rsplit(text, maxsplit=maxsplit) @stormfunc(readonly=True) async def _methStrEndswith(self, text): return self.valu.endswith(text) @stormfunc(readonly=True) async def _methStrStartswith(self, text): return self.valu.startswith(text) @stormfunc(readonly=True) async def _methStrRjust(self, size, fillchar=' '): return self.valu.rjust(await toint(size), await tostr(fillchar)) @stormfunc(readonly=True) async def _methStrLjust(self, size, fillchar=' '): return self.valu.ljust(await toint(size), await tostr(fillchar)) @stormfunc(readonly=True) async def _methStrReplace(self, oldv, newv, maxv=None): if maxv is None: return self.valu.replace(oldv, newv) else: return self.valu.replace(oldv, newv, int(maxv)) @stormfunc(readonly=True) async def _methStrStrip(self, chars=None): return self.valu.strip(chars) @stormfunc(readonly=True) async def _methStrLstrip(self, chars=None): return self.valu.lstrip(chars) @stormfunc(readonly=True) async def _methStrRstrip(self, chars=None): return self.valu.rstrip(chars) @stormfunc(readonly=True) async def _methStrLower(self): return self.valu.lower() @stormfunc(readonly=True) async def _methStrUpper(self): return self.valu.upper() @stormfunc(readonly=True) async def _methStrTitle(self): return self.valu.title() @stormfunc(readonly=True) async def _methStrSlice(self, start, end=None): start = await toint(start) if end is None: return self.valu[start:] end = await toint(end) return self.valu[start:end] @stormfunc(readonly=True) async def _methStrReverse(self): return self.valu[::-1] @stormfunc(readonly=True) async def _methStrJson(self): try: return json.loads(self.valu, strict=True) except Exception as e: mesg = f'Text is not valid JSON: {self.valu}' raise s_exc.BadJsonText(mesg=mesg)
[docs] @registry.registerType class Bytes(Prim): ''' Implements the Storm API for a Bytes object. ''' _storm_locals = ( {'name': 'decode', 'desc': 'Decode bytes to a string.', 'type': {'type': 'function', '_funcname': '_methDecode', 'args': ( {'name': 'encoding', 'type': 'str', 'desc': 'The encoding to use.', 'default': 'utf8', }, {'name': 'errors', 'type': 'str', 'desc': 'The error handling scheme to use.', 'default': 'surrogatepass', }, ), 'returns': {'type': 'str', 'desc': 'The decoded string.', }}}, {'name': 'bunzip', 'desc': ''' Decompress the bytes using bzip2. Example: Decompress bytes with bzip2:: $foo = $mybytez.bunzip()''', 'type': {'type': 'function', '_funcname': '_methBunzip', 'returns': {'type': 'bytes', 'desc': 'Decompressed bytes.', }}}, {'name': 'gunzip', 'desc': ''' Decompress the bytes using gzip and return them. Example: Decompress bytes with bzip2:: $foo = $mybytez.gunzip()''', 'type': {'type': 'function', '_funcname': '_methGunzip', 'returns': {'type': 'bytes', 'desc': 'Decompressed bytes.', }}}, {'name': 'bzip', 'desc': ''' Compress the bytes using bzip2 and return them. Example: Compress bytes with bzip:: $foo = $mybytez.bzip()''', 'type': {'type': 'function', '_funcname': '_methBzip', 'returns': {'type': 'bytes', 'desc': 'The bzip2 compressed bytes.', }}}, {'name': 'gzip', 'desc': ''' Compress the bytes using gzip and return them. Example: Compress bytes with gzip:: $foo = $mybytez.gzip()''', 'type': {'type': 'function', '_funcname': '_methGzip', 'returns': {'type': 'bytes', 'desc': 'The gzip compressed bytes.', }}}, {'name': 'json', 'desc': ''' Load JSON data from bytes. Notes: The bytes must be UTF8, UTF16 or UTF32 encoded. Example: Load bytes to a object:: $foo = $mybytez.json()''', 'type': {'type': 'function', '_funcname': '_methJsonLoad', 'args': ( {'name': 'encoding', 'type': 'str', 'desc': 'Specify an encoding to use.', 'default': None, }, {'name': 'errors', 'type': 'str', 'desc': 'Specify an error handling scheme to use.', 'default': 'surrogatepass', }, ), 'returns': {'type': 'prim', 'desc': 'The deserialized object.', }}}, {'name': 'slice', 'desc': ''' Slice a subset of bytes from an existing bytes. Examples: Slice from index to 1 to 5:: $subbyts = $byts.slice(1,5) Slice from index 3 to the end of the bytes:: $subbyts = $byts.slice(3) ''', 'type': {'type': 'function', '_funcname': '_methSlice', 'args': ( {'name': 'start', 'type': 'int', 'desc': 'The starting byte index.'}, {'name': 'end', 'type': 'int', 'default': None, 'desc': 'The ending byte index. If not specified, slice to the end.'}, ), 'returns': {'type': 'bytes', 'desc': 'The slice of bytes.', }}}, {'name': 'unpack', 'desc': ''' Unpack structures from bytes using python struct.unpack syntax. Examples: Unpack 3 unsigned 16 bit integers in little endian format:: ($x, $y, $z) = $byts.unpack("<HHH") ''', 'type': {'type': 'function', '_funcname': '_methUnpack', 'args': ( {'name': 'fmt', 'type': 'str', 'desc': 'A python struck.pack format string.'}, {'name': 'offset', 'type': 'int', 'desc': 'An offset to begin unpacking from.', 'default': 0}, ), 'returns': {'type': 'list', 'desc': 'The unpacked primitive values.', }}}, ) _storm_typename = 'bytes' _ismutable = False def __init__(self, valu, path=None): Prim.__init__(self, valu, path=path) self.locls.update(self.getObjLocals())
[docs] def getObjLocals(self): return { 'decode': self._methDecode, 'bunzip': self._methBunzip, 'gunzip': self._methGunzip, 'bzip': self._methBzip, 'gzip': self._methGzip, 'json': self._methJsonLoad, 'slice': self._methSlice, 'unpack': self._methUnpack, }
def __len__(self): return len(self.valu) def __str__(self): return self.valu.decode() def __hash__(self): return hash((self._storm_typename, self.valu)) def __eq__(self, othr): if isinstance(othr, Bytes): return self.valu == othr.valu return False async def _storm_copy(self): item = await s_coro.ornot(self.value) return s_msgpack.deepcopy(item, use_list=True) @stormfunc(readonly=True) async def _methSlice(self, start, end=None): start = await toint(start) if end is None: return self.valu[start:] end = await toint(end) return self.valu[start:end] @stormfunc(readonly=True) async def _methUnpack(self, fmt, offset=0): fmt = await tostr(fmt) offset = await toint(offset) try: return struct.unpack_from(fmt, self.valu, offset=offset) except struct.error as e: raise s_exc.BadArg(mesg=f'unpack() error: {e}') @stormfunc(readonly=True) async def _methDecode(self, encoding='utf8', errors='surrogatepass'): encoding = await tostr(encoding) errors = await tostr(errors) try: return self.valu.decode(encoding, errors) except UnicodeDecodeError as e: raise s_exc.StormRuntimeError(mesg=f'{e}: {s_common.trimText(repr(self.valu))}') from None async def _methBunzip(self): return bz2.decompress(self.valu) @stormfunc(readonly=True) async def _methBzip(self): return bz2.compress(self.valu) async def _methGunzip(self): return gzip.decompress(self.valu) @stormfunc(readonly=True) async def _methGzip(self): return gzip.compress(self.valu) @stormfunc(readonly=True) async def _methJsonLoad(self, encoding=None, errors='surrogatepass'): try: valu = self.valu errors = await tostr(errors) if encoding is None: encoding = json.detect_encoding(valu) else: encoding = await tostr(encoding) return json.loads(valu.decode(encoding, errors)) except UnicodeDecodeError as e: raise s_exc.StormRuntimeError(mesg=f'{e}: {s_common.trimText(repr(valu))}') from None except json.JSONDecodeError as e: mesg = f'Unable to decode bytes as json: {e.args[0]}' raise s_exc.BadJsonText(mesg=mesg)
[docs] @registry.registerType class Dict(Prim): ''' Implements the Storm API for a Dictionary object. ''' _storm_typename = 'dict' _ismutable = True def __len__(self): return len(self.valu) async def _storm_copy(self): item = await s_coro.ornot(self.value) return s_msgpack.deepcopy(item, use_list=True)
[docs] async def iter(self): for item in tuple(self.valu.items()): yield item
[docs] @stormfunc(readonly=True) async def setitem(self, name, valu): if ismutable(name): raise s_exc.BadArg(mesg='Mutable values are not allowed as dictionary keys', name=await torepr(name)) name = await toprim(name) if valu is undef: self.valu.pop(name, None) return self.valu[name] = valu
[docs] async def deref(self, name): name = await toprim(name) return self.valu.get(name)
[docs] async def value(self, use_list=False): return {await toprim(k): await toprim(v, use_list=use_list) for (k, v) in self.valu.items()}
[docs] async def stormrepr(self): reprs = ["{}: {}".format(await torepr(k), await torepr(v)) for (k, v) in list(self.valu.items())] rval = ', '.join(reprs) return f'{{{rval}}}'
[docs] @registry.registerType class CmdOpts(Dict): ''' A dictionary like object that holds a reference to a command options namespace. ( This allows late-evaluation of command arguments rather than forcing capture ) ''' _storm_typename = 'cmdopts' _ismutable = False def __len__(self): valu = vars(self.valu.opts) return len(valu) def __hash__(self): valu = vars(self.valu.opts) return hash((self._storm_typename, tuple(valu.items())))
[docs] @stormfunc(readonly=True) async def setitem(self, name, valu): # due to self.valu.opts potentially being replaced # we disallow setitem() to prevent confusion name = await tostr(name) mesg = 'CmdOpts may not be modified by the runtime' raise s_exc.StormRuntimeError(mesg=mesg, name=name)
[docs] async def deref(self, name): name = await tostr(name) return getattr(self.valu.opts, name, None)
[docs] async def value(self, use_list=False): valu = vars(self.valu.opts) return {await toprim(k): await toprim(v, use_list=use_list) for (k, v) in valu.items()}
[docs] async def iter(self): valu = vars(self.valu.opts) for item in valu.items(): yield item
[docs] async def stormrepr(self): valu = vars(self.valu.opts) reprs = ["{}: {}".format(await torepr(k), await torepr(v)) for (k, v) in valu.items()] rval = ', '.join(reprs) return f'{self._storm_typename}: {{{rval}}}'
[docs] @registry.registerType class Set(Prim): ''' Implements the Storm API for a Set object. ''' _storm_locals = ( {'name': 'add', 'desc': 'Add a item to the set. Each argument is added to the set.', 'type': {'type': 'function', '_funcname': '_methSetAdd', 'args': ( {'name': '*items', 'type': 'any', 'desc': 'The items to add to the set.', }, ), 'returns': {'type': 'null', }}}, {'name': 'has', 'desc': 'Check if a item is a member of the set.', 'type': {'type': 'function', '_funcname': '_methSetHas', 'args': ( {'name': 'item', 'type': 'any', 'desc': 'The item to check the set for membership.', }, ), 'returns': {'type': 'boolean', 'desc': 'True if the item is in the set, false otherwise.', }}}, {'name': 'rem', 'desc': 'Remove an item from the set.', 'type': {'type': 'function', '_funcname': '_methSetRem', 'args': ( {'name': '*items', 'type': 'any', 'desc': 'Items to be removed from the set.', }, ), 'returns': {'type': 'null', }}}, {'name': 'adds', 'desc': 'Add the contents of a iterable items to the set.', 'type': {'type': 'function', '_funcname': '_methSetAdds', 'args': ( {'name': '*items', 'type': 'any', 'desc': 'Iterables items to add to the set.', }, ), 'returns': {'type': 'null', }}}, {'name': 'rems', 'desc': 'Remove the contents of a iterable object from the set.', 'type': {'type': 'function', '_funcname': '_methSetRems', 'args': ( {'name': '*items', 'type': 'any', 'desc': 'Iterables items to remove from the set.', }, ), 'returns': {'type': 'null', }}}, {'name': 'list', 'desc': 'Get a list of the current members of the set.', 'type': {'type': 'function', '_funcname': '_methSetList', 'returns': {'type': 'list', 'desc': 'A list containing the members of the set.', }}}, {'name': 'size', 'desc': 'Get the size of the set.', 'type': {'type': 'function', '_funcname': '_methSetSize', 'returns': {'type': 'int', 'desc': 'The size of the set.', }}}, ) _storm_typename = 'set' _ismutable = True def __init__(self, valu, path=None): valu = list(valu) for item in valu: if ismutable(item): mesg = f'{repr(item)} is mutable and cannot be used in a set.' raise s_exc.StormRuntimeError(mesg=mesg) Prim.__init__(self, set(valu), path=path) self.locls.update(self.getObjLocals())
[docs] def getObjLocals(self): return { 'add': self._methSetAdd, 'has': self._methSetHas, 'rem': self._methSetRem, 'adds': self._methSetAdds, 'rems': self._methSetRems, 'list': self._methSetList, 'size': self._methSetSize, }
[docs] async def iter(self): for item in self.valu: yield item
def __len__(self): return len(self.valu) async def _methSetSize(self): return len(self) @stormfunc(readonly=True) async def _methSetHas(self, item): return item in self.valu @stormfunc(readonly=True) async def _methSetAdd(self, *items): for i in items: if ismutable(i): mesg = f'{await torepr(i)} is mutable and cannot be used in a set.' raise s_exc.StormRuntimeError(mesg=mesg) self.valu.add(i) @stormfunc(readonly=True) async def _methSetAdds(self, *items): for item in items: async for i in toiter(item): if ismutable(i): mesg = f'{await torepr(i)} is mutable and cannot be used in a set.' raise s_exc.StormRuntimeError(mesg=mesg) self.valu.add(i) @stormfunc(readonly=True) async def _methSetRem(self, *items): [self.valu.discard(i) for i in items] @stormfunc(readonly=True) async def _methSetRems(self, *items): for item in items: [self.valu.discard(i) async for i in toiter(item)] @stormfunc(readonly=True) async def _methSetList(self): return list(self.valu)
[docs] async def stormrepr(self): reprs = [await torepr(k) for k in self.valu] rval = ', '.join(reprs) return f'{{{rval}}}'
[docs] @registry.registerType class List(Prim): ''' Implements the Storm API for a List instance. ''' _storm_locals = ( {'name': 'has', 'desc': 'Check if a value is in the list.', 'type': {'type': 'function', '_funcname': '_methListHas', 'args': ( {'name': 'valu', 'type': 'any', 'desc': 'The value to check.', }, ), 'returns': {'type': 'boolean', 'desc': 'True if the item is in the list, false otherwise.', }}}, {'name': 'pop', 'desc': 'Pop and return the entry at the specified index in the list. If no index is specified, pop the last entry.', 'type': {'type': 'function', '_funcname': '_methListPop', 'args': ( {'name': 'index', 'type': 'int', 'desc': 'Index of entry to pop.', 'default': -1}, ), 'returns': {'type': 'any', 'desc': 'The entry at the specified index in the list.', }}}, {'name': 'size', 'desc': 'Return the length of the list.', 'type': {'type': 'function', '_funcname': '_methListSize', 'returns': {'type': 'int', 'desc': 'The size of the list.', }}}, {'name': 'sort', 'desc': 'Sort the list in place.', 'type': {'type': 'function', '_funcname': '_methListSort', 'args': ( {'name': 'reverse', 'type': 'boolean', 'desc': 'Sort the list in reverse order.', 'default': False}, ), 'returns': {'type': 'null', }}}, {'name': 'index', 'desc': 'Return a single field from the list by index.', 'type': {'type': 'function', '_funcname': '_methListIndex', 'args': ( {'name': 'valu', 'type': 'int', 'desc': 'The list index value.', }, ), 'returns': {'type': 'any', 'desc': 'The item present in the list at the index position.', }}}, {'name': 'length', 'desc': 'Get the length of the list. This is deprecated; please use ``.size()`` instead.', 'deprecated': {'eolvers': 'v3.0.0'}, 'type': {'type': 'function', '_funcname': '_methListLength', 'returns': {'type': 'int', 'desc': 'The size of the list.', }}}, {'name': 'append', 'desc': 'Append a value to the list.', 'type': {'type': 'function', '_funcname': '_methListAppend', 'args': ( {'name': 'valu', 'type': 'any', 'desc': 'The item to append to the list.', }, ), 'returns': {'type': 'null', }}}, {'name': 'reverse', 'desc': 'Reverse the order of the list in place', 'type': {'type': 'function', '_funcname': '_methListReverse', 'returns': {'type': 'null', }}}, {'name': 'slice', 'desc': ''' Get a slice of the list. Examples: Slice from index to 1 to 5:: $x=(f, o, o, b, a, r) $y=$x.slice(1,5) // (o, o, b, a) Slice from index 3 to the end of the list:: $y=$x.slice(3) // (b, a, r) ''', 'type': {'type': 'function', '_funcname': '_methListSlice', 'args': ( {'name': 'start', 'type': 'int', 'desc': 'The starting index.'}, {'name': 'end', 'type': 'int', 'default': None, 'desc': 'The ending index. If not specified, slice to the end of the list.'}, ), 'returns': {'type': 'list', 'desc': 'The slice of the list.'}}}, {'name': 'extend', 'desc': ''' Extend a list using another iterable. Examples: Populate a list by extending it with to other lists:: $list = () $foo = (f, o, o) $bar = (b, a, r) $list.extend($foo) $list.extend($bar) // $list is now (f, o, o, b, a, r) ''', 'type': {'type': 'function', '_funcname': '_methListExtend', 'args': ( {'name': 'valu', 'type': 'list', 'desc': 'A list or other iterable.'}, ), 'returns': {'type': 'null'}}}, {'name': 'unique', 'desc': 'Get a copy of the list containing unique items.', 'type': {'type': 'function', '_funcname': '_methListUnique', 'returns': {'type': 'list'}}}, {'name': 'rem', 'desc': 'Remove a specific item from anywhere in the list.', 'type': {'type': 'function', '_funcname': '_methListRemove', 'args': ( {'name': 'item', 'type': 'any', 'desc': 'An item in the list.'}, {'name': 'all', 'type': 'boolean', 'default': False, 'desc': 'Remove all instances of item from the list.'}, ), 'returns': {'type': 'boolean', 'desc': 'Boolean indicating if the item was removed from the list.'}}}, ) _storm_typename = 'list' _ismutable = True def __init__(self, valu, path=None): Prim.__init__(self, valu, path=path) self.locls.update(self.getObjLocals())
[docs] def getObjLocals(self): return { 'has': self._methListHas, 'pop': self._methListPop, 'size': self._methListSize, 'sort': self._methListSort, 'index': self._methListIndex, 'length': self._methListLength, 'append': self._methListAppend, 'reverse': self._methListReverse, 'slice': self._methListSlice, 'extend': self._methListExtend, 'unique': self._methListUnique, 'rem': self._methListRemove, }
[docs] @stormfunc(readonly=True) async def setitem(self, name, valu): indx = await toint(name) if valu is undef: try: self.valu.pop(indx) except IndexError: pass return self.valu[indx] = valu
async def _storm_copy(self): item = await s_coro.ornot(self.value) return s_msgpack.deepcopy(item, use_list=True) async def _derefGet(self, name): return await self._methListIndex(name) def __len__(self): return len(self.valu) @stormfunc(readonly=True) async def _methListHas(self, valu): if valu in self.valu: return True prim = await toprim(valu) if prim == valu: return False return prim in self.valu @stormfunc(readonly=True) async def _methListPop(self, index=-1): index = await toint(index) try: return self.valu.pop(index) except IndexError as exc: mesg = str(exc) raise s_exc.StormRuntimeError(mesg=mesg) @stormfunc(readonly=True) async def _methListAppend(self, valu): ''' ''' self.valu.append(valu) @stormfunc(readonly=True) async def _methListIndex(self, valu): indx = await toint(valu) try: return self.valu[indx] except IndexError as e: raise s_exc.StormRuntimeError(mesg=str(e), valurepr=await self.stormrepr(), len=len(self.valu), indx=indx) from None @stormfunc(readonly=True) async def _methListReverse(self): self.valu.reverse() @stormfunc(readonly=True) async def _methListLength(self): s_common.deprecated('StormType List.length()') runt = s_scope.get('runt') if runt: await runt.snap.warnonce('StormType List.length() is deprecated. Use the size() method.') return len(self) @stormfunc(readonly=True) async def _methListSort(self, reverse=False): reverse = await tobool(reverse, noneok=True) try: self.valu.sort(reverse=reverse) except TypeError as e: raise s_exc.StormRuntimeError(mesg=f'Error sorting list: {str(e)}', valurepr=await self.stormrepr()) from None @stormfunc(readonly=True) async def _methListSize(self): return len(self) async def _methListSlice(self, start, end=None): start = await toint(start) if end is None: return self.valu[start:] end = await toint(end) return self.valu[start:end] async def _methListExtend(self, valu): async for item in toiter(valu): self.valu.append(item)
[docs] async def value(self, use_list=False): if use_list: return [await toprim(v, use_list=use_list) for v in self.valu] return tuple([await toprim(v, use_list=use_list) for v in self.valu])
[docs] async def iter(self): for item in self.valu: yield item
@stormfunc(readonly=True) async def _methListUnique(self): ret = [] checkret = [] for val in self.valu: try: _cval = await toprim(val) except s_exc.NoSuchType: _cval = val if _cval in checkret: continue checkret.append(_cval) ret.append(val) return ret async def _methListRemove(self, item, all=False): item = await toprim(item) all = await tobool(all) if item not in self.valu: return False while item in self.valu: self.valu.remove(item) if not all: break return True
[docs] async def stormrepr(self): reprs = [await torepr(k) for k in self.valu] rval = ', '.join(reprs) return f'[{rval}]'
[docs] @registry.registerType class Bool(Prim): ''' Implements the Storm API for a boolean instance. ''' _storm_typename = 'boolean' _ismutable = False def __str__(self): return str(self.value()).lower() def __int__(self): return int(self.value()) def __hash__(self): return hash((self._storm_typename, self.value()))
[docs] @registry.registerType class Number(Prim): ''' Implements the Storm API for a Number instance. Storm Numbers are high precision fixed point decimals corresponding to the the hugenum storage type. ''' _storm_locals = ( {'name': 'scaleb', 'desc': ''' Return the number multiplied by 10**other. Example: Multiply the value by 10**-18:: $baz.scaleb(-18)''', 'type': {'type': 'function', '_funcname': '_methScaleb', 'args': ( {'name': 'other', 'type': 'int', 'desc': 'The amount to adjust the exponent.', }, ), 'returns': {'type': 'number', 'desc': 'The exponent adjusted number.', }}}, {'name': 'toint', 'desc': ''' Return the number as an integer. By default, decimal places will be truncated. Optionally, rounding rules can be specified by providing the name of a Python decimal rounding mode to the 'rounding' argument. Example: Round the value stored in $baz up instead of truncating:: $baz.toint(rounding=ROUND_UP)''', 'type': {'type': 'function', '_funcname': '_methToInt', 'args': ( {'name': 'rounding', 'type': 'str', 'default': None, 'desc': 'An optional rounding mode to use.', }, ), 'returns': {'type': 'int', 'desc': 'The number as an integer.', }}}, {'name': 'tostr', 'desc': 'Return the number as a string.', 'type': {'type': 'function', '_funcname': '_methToStr', 'returns': {'type': 'str', 'desc': 'The number as a string.', }}}, {'name': 'tofloat', 'desc': 'Return the number as a float.', 'type': {'type': 'function', '_funcname': '_methToFloat', 'returns': {'type': 'float', 'desc': 'The number as a float.', }}}, ) _storm_typename = 'number' _ismutable = False def __init__(self, valu, path=None): try: valu = s_common.hugenum(valu) except (TypeError, decimal.DecimalException) as e: mesg = f'Failed to make number from {valu!r}' raise s_exc.BadCast(mesg=mesg) from e Prim.__init__(self, valu, path=path) self.locls.update(self.getObjLocals())
[docs] def getObjLocals(self): return { 'toint': self._methToInt, 'tostr': self._methToStr, 'tofloat': self._methToFloat, 'scaleb': self._methScaleb, }
@stormfunc(readonly=True) async def _methScaleb(self, other): newv = s_common.hugescaleb(self.value(), await toint(other)) return Number(newv) @stormfunc(readonly=True) async def _methToInt(self, rounding=None): if rounding is None: return int(self.valu) try: return int(self.valu.quantize(decimal.Decimal('1'), rounding=rounding)) except TypeError as e: raise s_exc.StormRuntimeError(mesg=f'Error rounding number: {str(e)}', valurepr=await self.stormrepr()) from None @stormfunc(readonly=True) async def _methToStr(self): return str(self.valu) @stormfunc(readonly=True) async def _methToFloat(self): return float(self.valu) def __str__(self): return str(self.value()) def __int__(self): return int(self.value()) def __float__(self): return float(self.value()) def __hash__(self): return hash((self._storm_typename, self.value())) def __eq__(self, othr): if isinstance(othr, float): othr = s_common.hugenum(othr) return self.value() == othr elif isinstance(othr, (int, decimal.Decimal)): return self.value() == othr elif isinstance(othr, Number): return self.value() == othr.value() return False def __lt__(self, othr): if isinstance(othr, float): othr = s_common.hugenum(othr) return self.value() < othr elif isinstance(othr, (int, decimal.Decimal)): return self.value() < othr elif isinstance(othr, Number): return self.value() < othr.value() mesg = f"comparison not supported between instance of {self.__class__.__name__} and {othr.__class__.__name__}" raise TypeError(mesg) def __add__(self, othr): if isinstance(othr, float): othr = s_common.hugenum(othr) return Number(s_common.hugeadd(self.value(), othr)) elif isinstance(othr, (int, decimal.Decimal)): return Number(s_common.hugeadd(self.value(), othr)) elif isinstance(othr, Number): return Number(s_common.hugeadd(self.value(), othr.value())) mesg = f"'+' not supported between instance of {self.__class__.__name__} and {othr.__class__.__name__}" raise TypeError(mesg) __radd__ = __add__ def __sub__(self, othr): if isinstance(othr, float): othr = s_common.hugenum(othr) return Number(s_common.hugesub(self.value(), othr)) elif isinstance(othr, (int, decimal.Decimal)): return Number(s_common.hugesub(self.value(), othr)) elif isinstance(othr, Number): return Number(s_common.hugesub(self.value(), othr.value())) mesg = f"'-' not supported between instance of {self.__class__.__name__} and {othr.__class__.__name__}" raise TypeError(mesg) def __rsub__(self, othr): othr = Number(othr) return othr.__sub__(self) def __mul__(self, othr): if isinstance(othr, float): othr = s_common.hugenum(othr) return Number(s_common.hugemul(self.value(), othr)) elif isinstance(othr, (int, decimal.Decimal)): return Number(s_common.hugemul(self.value(), othr)) elif isinstance(othr, Number): return Number(s_common.hugemul(self.value(), othr.value())) mesg = f"'*' not supported between instance of {self.__class__.__name__} and {othr.__class__.__name__}" raise TypeError(mesg) __rmul__ = __mul__ def __truediv__(self, othr): if isinstance(othr, float): othr = s_common.hugenum(othr) return Number(s_common.hugediv(self.value(), othr)) elif isinstance(othr, (int, decimal.Decimal)): return Number(s_common.hugediv(self.value(), othr)) elif isinstance(othr, Number): return Number(s_common.hugediv(self.value(), othr.value())) mesg = f"'/' not supported between instance of {self.__class__.__name__} and {othr.__class__.__name__}" raise TypeError(mesg) def __rtruediv__(self, othr): othr = Number(othr) return othr.__truediv__(self) def __pow__(self, othr): if isinstance(othr, float): othr = s_common.hugenum(othr) return Number(s_common.hugepow(self.value(), othr)) elif isinstance(othr, (int, decimal.Decimal)): return Number(s_common.hugepow(self.value(), othr)) elif isinstance(othr, Number): return Number(s_common.hugepow(self.value(), othr.value())) mesg = f"'**' not supported between instance of {self.__class__.__name__} and {othr.__class__.__name__}" raise TypeError(mesg) def __rpow__(self, othr): othr = Number(othr) return othr.__pow__(self) def __mod__(self, othr): if isinstance(othr, float): othr = s_common.hugenum(othr) return Number(s_common.hugemod(self.value(), othr)[1]) elif isinstance(othr, (int, decimal.Decimal)): return Number(s_common.hugemod(self.value(), othr)[1]) elif isinstance(othr, Number): return Number(s_common.hugemod(self.value(), othr.value())[1]) mesg = f"'%' not supported between instance of {self.__class__.__name__} and {othr.__class__.__name__}" raise TypeError(mesg) def __rmod__(self, othr): othr = Number(othr) return othr.__mod__(self)
[docs] async def stormrepr(self): return str(self.value())
[docs] @registry.registerLib class LibGlobals(Lib): ''' A Storm Library for interacting with global variables which are persistent across the Cortex. ''' _storm_locals = ( {'name': 'get', 'desc': 'Get a Cortex global variables.', 'type': {'type': 'function', '_funcname': '_methGet', 'args': ( {'name': 'name', 'type': 'str', 'desc': 'Name of the variable.', }, {'name': 'default', 'type': 'prim', 'default': None, 'desc': 'Default value to return if the variable is not set.', }, ), 'returns': {'type': 'prim', 'desc': 'The variable value.', }}}, {'name': 'pop', 'desc': 'Delete a variable value from the Cortex.', 'type': {'type': 'function', '_funcname': '_methPop', 'args': ( {'name': 'name', 'type': 'str', 'desc': 'Name of the variable.', }, {'name': 'default', 'type': 'prim', 'default': None, 'desc': 'Default value to return if the variable is not set.', }, ), 'returns': {'type': 'prim', 'desc': 'The variable value.', }}}, {'name': 'set', 'desc': 'Set a variable value in the Cortex.', 'type': {'type': 'function', '_funcname': '_methSet', 'args': ( {'name': 'name', 'type': 'str', 'desc': 'The name of the variable to set.', }, {'name': 'valu', 'type': 'prim', 'desc': 'The value to set.', }, ), 'returns': {'type': 'prim', 'desc': 'The variable value.', }}}, {'name': 'list', 'desc': 'Get a list of variable names and values.', 'type': {'type': 'function', '_funcname': '_methList', 'returns': {'type': 'list', 'desc': 'A list of tuples with variable names and values that the user can access.', }}}, ) _storm_lib_path = ('globals', ) _storm_lib_perms = ( {'perm': ('globals',), 'gate': 'cortex', 'desc': 'Used to control all operations for global variables.'}, {'perm': ('globals', 'get'), 'gate': 'cortex', 'desc': 'Used to control read access to all global variables.'}, {'perm': ('globals', 'get', '<name>'), 'gate': 'cortex', 'desc': 'Used to control read access to a specific global variable.'}, {'perm': ('globals', 'set'), 'gate': 'cortex', 'desc': 'Used to control edit access to all global variables.'}, {'perm': ('globals', 'set', '<name>'), 'gate': 'cortex', 'desc': 'Used to control edit access to a specific global variable.'}, {'perm': ('globals', 'pop'), 'gate': 'cortex', 'desc': 'Used to control delete access to all global variables.'}, {'perm': ('globals', 'pop', '<name>'), 'gate': 'cortex', 'desc': 'Used to control delete access to a specific global variable.'}, ) def __init__(self, runt, name): Lib.__init__(self, runt, name)
[docs] def getObjLocals(self): return { 'get': self._methGet, 'pop': self._methPop, 'set': self