Source code for core.observables.observable

from __future__ import unicode_literals

from datetime import datetime
import re
import operator
import logging
from mongoengine import *
from flask_mongoengine.wtf import model_form
from flask import url_for

from core.helpers import iterify
from core.database import Node, TagListField
from core.observables import ObservableTag, Tag
from core.entities import Entity
from core.errors import ObservableValidationError


[docs]class Observable(Node): """Base class for Observables in Yeti Observables describe elements that can be seen in investigations, incidents, reports, intelligence, etc. They are usually technical data about specific threats or actors. Attributes: value: The observable's technical value (the observed URL, hostname, IP address...) sources: An array of strings that define how the observable was inserted description: A free-text description of the observable context: A JSON object providing extra information as to why the observable was added. Context can be added trough the API or through Feeds tags: An array of :class:`core.observables.tag.ObservableTag` objects last_analyses: An array of JSON objects indicating the last analysis time for a particular analytics created: Creation date last_tagged: Date when a given observable was last tagged exclude_fields: Fields to be excluded from automatic form creation """ SEARCH_ALIASES = {} DISPLAY_FIELDS = [("value", "Value"), ("context", "Context"), ("tags", "Tags"), ("sources", "Sources"), ("created", "Created")] value = StringField( verbose_name="Value", required=True, sparse=True) sources = ListField(StringField(), verbose_name="Sources") description = StringField(verbose_name="Description") context = ListField(DictField(), verbose_name="Context") tags = ListField(EmbeddedDocumentField(ObservableTag), verbose_name="Tags") last_analyses = DictField(verbose_name="Last analyses") created = DateTimeField(default=datetime.utcnow) last_tagged = DateTimeField(default=None) exclude_fields = [ 'sources', 'context', 'last_analyses', 'created', 'attached_files', 'last_tagged' ] meta = { "allow_inheritance": True, "indexes": [ "tags", "last_analyses", "created", { "fields": ["#value"], "cls": False, } ], "index_background": True, "ordering": ["-created"], } ignore = [] search_regex = None
[docs] @classmethod def get_form(klass): """Gets the appropriate form for a given obseravble""" form = model_form(klass, exclude=klass.exclude_fields) form.tags = TagListField() return form
def __unicode__(self): return u"{} ({} context)".format(self.value, len(self.context))
[docs] @staticmethod def guess_type(string): """Tries to guess the type of observable given a ``string``. Args: string: The string that will be used to guess the observable type from. Returns: An observable Class. Raises: ObservableValidationError if no type could be guessed. """ from core.observables import Url, Ip, Email, Path, Hostname, Hash, Bitcoin, MacAddress if string and string.strip() != '': for t in [Url, Ip, Email, Path, Hostname, Hash, Bitcoin, MacAddress]: if t.check_type(string): return t raise ObservableValidationError( "{} was not recognized as a viable datatype".format(string))
@staticmethod def from_string(string): from core.observables import Url, Ip, Hostname, Email, Hash, MacAddress results = dict() for t in [Url, Ip, Email, Hostname, Hash, MacAddress]: results[t.__name__] = t.extract(string) return results
[docs] @classmethod def add_text(cls, text, tags=[], force_type=None): """Adds and returns an observable for a given string. Args: text: the text that will be used to add an Observable from. Returns: A saved Observable instance. """ if force_type: observable_type = Observable.subclass_from_name(force_type) else: observable_type = Observable.guess_type(text) o = observable_type.get_or_create(value=text) if tags: o.tag(tags) return o
@classmethod def check_type(cls, txt): match = re.match('^{}$'.format(cls.regex), txt, re.UNICODE) if match: return cls.is_valid(match) return False @classmethod def extract(cls, txt): results = {} if cls.search_regex: search_regex = re.compile(cls.search_regex, re.UNICODE) else: search_regex = re.compile(cls.regex, re.UNICODE) for match in re.finditer(search_regex, txt): try: valid = cls.is_valid(match) except UnicodeDecodeError: continue if not valid: continue try: observable = cls(value=match.group('search')) observable.normalize() except ObservableValidationError: continue except ValueError as e: logging.error('Value error: {} - on: {}'.format(e, observable.value)) continue if observable.value not in cls.ignore: # Replace with existing observable if there is one try: observable = cls.objects.get(value=observable.value) except cls.DoesNotExist: pass results[match.group('search')] = observable return results @classmethod def is_valid(cls, match): return True def normalize(self): pass
[docs] def clean(self): if self.check_type(self.value): self.normalize() else: raise ObservableValidationError( "'{}' is not a valid '{}'".format( self.value, self.__class__.__name__))
[docs] @staticmethod def change_all_tags(old_tags, new_tag): """Changes tags on all observables Args: old_tags: A string or array of strings representing tag names to change new_tag: The new tag name by which all ``old_tags`` should be replaced """ old_tags = iterify(old_tags) for o in Observable.objects(tags__name__in=old_tags): for old_tag in old_tags: o.change_tag(old_tag, new_tag)
[docs] def add_context(self, context, replace_source=None, dedup_list=[]): """Adds context to an Observable. "Context" is represented by a JSON object (or Python ``dict()``) that will be added to the Observable's ``context`` set. Context should provide information on why the Observable has been added to the database. Context can be any information, but it needs to have a ``source`` key that can point the analyst to the source of the context. Args: context: a JSON object representing the context to be added. replace_source: If defined, contexts having a ``source`` attribute set to ``replace_source`` will be deleted before insert dedup_list: takes a list of fields to ignore during dedup comparison. i.e. date/count type fields. Empty list will skip the partial dedup as dedup for the exact same context is already builtin. Returns: A fresh instance of the Observable as it exists in the database. """ assert 'source' in context context = { k: v for k, v in sorted(context.items(), key=operator.itemgetter(0)) } if replace_source: # This does not work : cannot traverse and set context atomically # self.modify({"context__source": c}, set__context__S=context) self.modify(pull__context__source=replace_source) if dedup_list: for c in self.context: remove = True for key in c: if key in dedup_list: continue if c[key] != context.get(key, ''): remove = False break if remove: self.modify(pull__context=c) self.modify(add_to_set__context=context) return self.reload()
[docs] def remove_context(self, context): """Removes Context from an observable. Args: context: a JSON object representing the context to be removed. Returns: A fresh instance of the Observable as it exists in the database. """ context = { k: v for k, v in sorted(context.items(), key=operator.itemgetter(0)) } self.modify(pull__context=context) return self.reload()
[docs] def add_source(self, source): """Adds a source to the observable instance Args: source: a string to add to the array of sources. """ return self.modify(add_to_set__sources=source)
[docs] def get_tags(self, fresh=True): """Returns an array of strings containing an observables' fresh tags names. Args: fresh: set to ``False`` to also include non-fresh tags in the result Returns: Array of strings containing an observables' fresh tags names. """ return [t.name for t in self.tags if (t.fresh or not fresh)]
def find_tags(self): # find related tags and count them new_tags = {} for tag in self.tags: tag = Tag.objects.get(name=tag.name) for produces in tag.produces: new_tags[produces] = new_tags.get(tag, 0) + 1 # remove already known tags localtags = [tag.name for tag in self.tags] for tag in new_tags.copy(): if tag in localtags: new_tags.pop(tag) return new_tags def has_tag(self, tag_to_search): for tag in self.tags: if tag.name == tag_to_search: return True else: return False def change_tag(self, old_tag, new_tag): if not self.modify({"tags__name": old_tag, "tags__name__ne": new_tag}, set__tags__S__name=new_tag): self.modify({"tags__name": old_tag}, pull__tags__name=old_tag) self.modify({ "tags__name": new_tag }, set__tags__S__last_seen=datetime.utcnow()) return self.reload() def untag(self, tags): for tag in iterify(tags): self.modify(pull__tags__name=tag)
[docs] def tag(self, new_tags, strict=False, expiration=None): """Tags an observable. An observable can be tagged to add more information as to what it represents. Args: new_tags: An array of strings to tag the observable with. strict: Set to ``True`` to replace all existing tags with the ``new_tags``. expiration: Timedelta field after which the Tag will not be considered fresh anymore. Returns: A fresh Observable instance as reloaded from the database. """ new_tags = iterify(new_tags) if strict: remove = set([t.name for t in self.tags]) - set(new_tags) for tag in remove: self.modify(pull__tags__name=tag) tagged = False for new_tag in new_tags: if new_tag.strip() != '': tagged = True new_tag = Tag(name=new_tag) new_tag.clean() try: # check if tag is a replacement tag = Tag.objects.get(replaces=new_tag.name) except DoesNotExist: tag = Tag.get_or_create(name=new_tag.name) if not expiration: expiration = tag.default_expiration extra_tags = tag.produces + [tag] # search for related entities and link them for e in Entity.objects(tags__in=[tag.name]): self.active_link_to(e, 'Tagged', 'tags', clean_old=False) for tag in extra_tags: if not self.modify( {"tags__name": tag.name}, set__tags__S__fresh=True, set__tags__S__last_seen=datetime.utcnow()): self.modify( push__tags=ObservableTag( name=tag.name, expiration=expiration)) tag.modify(inc__count=1) if tagged: self.update(set__last_tagged=datetime.utcnow()) return self.reload()
def get_last_tagged(self): if not self.last_tagged: last = datetime(1970, 1, 1) for tag in self.tags: if tag.last_seen > last: last = tag.last_seen self.update(set__last_tagged=last) return last else: return self.last_tagged def get_first_tagged(self): first_tagged = None for tag in self.tags: if not first_tagged or tag.first_seen < first_tagged: first_tagged = tag.first_seen return first_tagged def expire_tags(self): for tag in self.tags: if tag.expiration: if (tag.last_seen + tag.expiration) < datetime.utcnow() and tag.fresh: tag.fresh = False self.save() elif (tag.last_seen + tag.expiration) > datetime.utcnow() and not tag.fresh: tag.fresh = True self.save() return self def fresh_tags(self): return [tag for tag in self.tags if tag.fresh] def analysis_done(self, module_name): ts = datetime.utcnow() return self.modify(**{"set__last_analyses__{}".format(module_name): ts}) def info(self): i = { k: v for k, v in self._data.items() if k in [ "value", "context", "last_analyses", "created", "sources", "description" ] } i['tags'] = [t.info() for t in self.tags] if self.id: i['id'] = str(self.id) i['type'] = self.__class__.__name__ try: i['url'] = url_for( "api.Observable:post", id=str(self.id), _external=True) i['human_url'] = url_for( "frontend.ObservableView:get", id=str(self.id), _external=True) except RuntimeError: pass return i