diff --git a/workflows/Ews2Case.py b/workflows/Ews2Case.py index d4aced3..7e4f112 100644 --- a/workflows/Ews2Case.py +++ b/workflows/Ews2Case.py @@ -1,8 +1,18 @@ #!/usr/bin/env python3 # -*- coding: utf8 -*- +''' +Ews2Case with added functionality to find observables in the email body. +Code for searchObservables, isWhitelisted, and loadWhitelists taken +directly from https://github.com/xme/dockers/tree/master/imap2thehive +All credit to the original functions goes to Xme +''' + +from bs4 import BeautifulSoup import os, sys import logging +import re +import time current_dir = os.path.dirname(os.path.abspath(__file__)) app_dir = current_dir + '/..' sys.path.insert(0, current_dir) @@ -12,6 +22,26 @@ from objects.TheHiveConnector import TheHiveConnector from objects.TempAttachment import TempAttachment +from thehive4py.api import TheHiveApi +from thehive4py.models import Case, CaseTask, CaseObservable, CustomFieldHelper +from thehive4py.models import Alert, AlertArtifact + +#Location of whitelist file +WHITELIST = '/home/thehive/Synapse/Synapse/workflows/objects/thehive.whitelist' +#Example contents of thehive.whitelist +''' +\.domain\.com +.domain.com +domain.com +172\.16\.\d{1,3}\.\d{1,3} +192\.168\.\d{1,3}\.\d{1,3} +''' +#List to hold contents from whitelist file +whitelists = [] + +#TheHive Api key +API_KEY = '******' + def connectEws(): logger = logging.getLogger(__name__) logger.info('%s.connectEws starts', __name__) @@ -21,12 +51,13 @@ def connectEws(): try: cfg = getConf() - ewsConnector = EwsConnector(cfg) folder_name = cfg.get('EWS', 'folder_name') unread = ewsConnector.scan(folder_name) - theHiveConnector = TheHiveConnector(cfg) + theHiveConnector = TheHiveConnector(cfg) + + api = TheHiveApi('http://127.0.0.1:9000', API_KEY) for msg in unread: #type(msg) @@ -72,7 +103,12 @@ def connectEws(): commTask = theHiveConnector.craftCommTask() commTaskId = theHiveConnector.createTask(esCaseId, commTask) + fullBody = getEmailBody(msg) + + #Scan body message for observables, returns list of observables + observables = searchObservables(fullBody) + taskLog = theHiveConnector.craftTaskLog(fullBody) createdTaskLogId = theHiveConnector.addTaskLog(commTaskId, taskLog) @@ -113,7 +149,22 @@ def connectEws(): theHiveConnector.addFileObservable(esCaseId, tmpFilepath, comment) - + #Parse obserables + for o in observables: + if isWhitelisted(o['value']): + print("skipping %s" % o['value']) + else: + observable = CaseObservable( + dataType = o['type'], + data = o['value'], + tlp = 2, + ioc = False, + tags=['Synapse'], + message = 'Found in the email body' + ) + #send observables to case + response = api.create_case_observable(esCaseId, observable) + time.sleep(1) report['success'] = True return report @@ -147,17 +198,93 @@ def getEmailBody(email): 'Subject: ' + str(email.subject) + '\n\n') body = email.text_body + #email.text_body should get the body either it is + #html or raw text + #unfortunately it is only supported with Exchange 2013 + #so we need to get the body from another way + + if body is None: + #alternate way to get the body + soup = BeautifulSoup(email.body, 'html.parser') + try: + #html email + body = soup.body.text + except AttributeError: + #non html email + body = soup.text + + return ('```\n' + replyToInfo + str(body) + '\n```') + +def searchObservables(body): + + observables = [] + # Observable types + # Source: https://github.com/armbues/ioc_parser/blob/master/iocp/data/patterns.ini + observableTypes = [ + { 'type': 'filename', 'regex': r'\b([A-Za-z0-9-_\.]+\.(exe|dll|bat|sys|htm|html|js|jar|jpg|png|vb|scr|pif|chm|zip|rar|cab|pdf|doc|docx|ppt|pptx|xls|xlsx|swf|gif))\b' }, + { 'type': 'url', 'regex': r'\b([a-z]{3,}\:\/\/[a-z0-9.\-:/?=&;]{16,})\b' }, + { 'type': 'ip', 'regex': r'\b(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\b' }, + { 'type': 'fqdn', 'regex': r'\b(([a-z0-9\-]{2,}\[?\.\]?){2,}(abogado|ac|academy|accountants|active|actor|ad|adult|ae|aero|af|ag|agency|ai|airforce|al|allfinanz|alsace|am|amsterdam|an|android|ao|aq|aquarelle|ar|archi|army|arpa|as|asia|associates|at|attorney|au|auction|audio|autos|aw|ax|axa|az|ba|band|bank|bar|barclaycard|barclays|bargains|bayern|bb|bd|be|beer|berlin|best|bf|bg|bh|bi|bid|bike|bingo|bio|biz|bj|black|blackfriday|bloomberg|blue|bm|bmw|bn|bnpparibas|bo|boo|boutique|br|brussels|bs|bt|budapest|build|builders|business|buzz|bv|bw|by|bz|bzh|ca|cal|camera|camp|cancerresearch|canon|capetown|capital|caravan|cards|care|career|careers|cartier|casa|cash|cat|catering|cc|cd|center|ceo|cern|cf|cg|ch|channel|chat|cheap|christmas|chrome|church|ci|citic|city|ck|cl|claims|cleaning|click|clinic|clothing|club|cm|cn|co|coach|codes|coffee|college|cologne|com|community|company|computer|condos|construction|consulting|contractors|cooking|cool|coop|country|cr|credit|creditcard|cricket|crs|cruises|cu|cuisinella|cv|cw|cx|cy|cymru|cz|dabur|dad|dance|dating|day|dclk|de|deals|degree|delivery|democrat|dental|dentist|desi|design|dev|diamonds|diet|digital|direct|directory|discount|dj|dk|dm|dnp|do|docs|domains|doosan|durban|dvag|dz|eat|ec|edu|education|ee|eg|email|emerck|energy|engineer|engineering|enterprises|equipment|er|es|esq|estate|et|eu|eurovision|eus|events|everbank|exchange|expert|exposed|fail|farm|fashion|feedback|fi|finance|financial|firmdale|fish|fishing|fit|fitness|fj|fk|flights|florist|flowers|flsmidth|fly|fm|fo|foo|forsale|foundation|fr|frl|frogans|fund|furniture|futbol|ga|gal|gallery|garden|gb|gbiz|gd|ge|gent|gf|gg|ggee|gh|gi|gift|gifts|gives|gl|glass|gle|global|globo|gm|gmail|gmo|gmx|gn|goog|google|gop|gov|gp|gq|gr|graphics|gratis|green|gripe|gs|gt|gu|guide|guitars|guru|gw|gy|hamburg|hangout|haus|healthcare|help|here|hermes|hiphop|hiv|hk|hm|hn|holdings|holiday|homes|horse|host|hosting|house|how|hr|ht|hu|ibm|id|ie|ifm|il|im|immo|immobilien|in|industries|info|ing|ink|institute|insure|int|international|investments|io|iq|ir|irish|is|it|iwc|jcb|je|jetzt|jm|jo|jobs|joburg|jp|juegos|kaufen|kddi|ke|kg|kh|ki|kim|kitchen|kiwi|km|kn|koeln|kp|kr|krd|kred|kw|ky|kyoto|kz|la|lacaixa|land|lat|latrobe|lawyer|lb|lc|lds|lease|legal|lgbt|li|lidl|life|lighting|limited|limo|link|lk|loans|london|lotte|lotto|lr|ls|lt|ltda|lu|luxe|luxury|lv|ly|ma|madrid|maison|management|mango|market|marketing|marriott|mc|md|me|media|meet|melbourne|meme|memorial|menu|mg|mh|miami|mil|mini|mk|ml|mm|mn|mo|mobi|moda|moe|monash|money|mormon|mortgage|moscow|motorcycles|mov|mp|mq|mr|ms|mt|mu|museum|mv|mw|mx|my|mz|na|nagoya|name|navy|nc|ne|net|network|neustar|new|nexus|nf|ng|ngo|nhk|ni|ninja|nl|no|np|nr|nra|nrw|ntt|nu|nyc|nz|okinawa|om|one|ong|onl|ooo|org|organic|osaka|otsuka|ovh|pa|paris|partners|parts|party|pe|pf|pg|ph|pharmacy|photo|photography|photos|physio|pics|pictures|pink|pizza|pk|pl|place|plumbing|pm|pn|pohl|poker|porn|post|pr|praxi|press|pro|prod|productions|prof|properties|property|ps|pt|pub|pw|qa|qpon|quebec|re|realtor|recipes|red|rehab|reise|reisen|reit|ren|rentals|repair|report|republican|rest|restaurant|reviews|rich|rio|rip|ro|rocks|rodeo|rs|rsvp|ru|ruhr|rw|ryukyu|sa|saarland|sale|samsung|sarl|sb|sc|sca|scb|schmidt|schule|schwarz|science|scot|sd|se|services|sew|sexy|sg|sh|shiksha|shoes|shriram|si|singles|sj|sk|sky|sl|sm|sn|so|social|software|sohu|solar|solutions|soy|space|spiegel|sr|st|style|su|supplies|supply|support|surf|surgery|suzuki|sv|sx|sy|sydney|systems|sz|taipei|tatar|tattoo|tax|tc|td|technology|tel|temasek|tennis|tf|tg|th|tienda|tips|tires|tirol|tj|tk|tl|tm|tn|to|today|tokyo|tools|top|toshiba|town|toys|tp|tr|trade|training|travel|trust|tt|tui|tv|tw|tz|ua|ug|uk|university|uno|uol|us|uy|uz|va|vacations|vc|ve|vegas|ventures|versicherung|vet|vg|vi|viajes|video|villas|vision|vlaanderen|vn|vodka|vote|voting|voto|voyage|vu|wales|wang|watch|webcam|website|wed|wedding|wf|whoswho|wien|wiki|williamhill|wme|work|works|world|ws|wtc|wtf|xyz|yachts|yandex|ye|yoga|yokohama|youtube|yt|za|zm|zone|zuerich|zw))\b' }, + { 'type': 'domain', 'regex': r'\b(([a-z0-9\-]{2,}\[?\.\]?){1}(abogado|ac|academy|accountants|active|actor|ad|adult|ae|aero|af|ag|agency|ai|airforce|al|allfinanz|alsace|am|amsterdam|an|android|ao|aq|aquarelle|ar|archi|army|arpa|as|asia|associates|at|attorney|au|auction|audio|autos|aw|ax|axa|az|ba|band|bank|bar|barclaycard|barclays|bargains|bayern|bb|bd|be|beer|berlin|best|bf|bg|bh|bi|bid|bike|bingo|bio|biz|bj|black|blackfriday|bloomberg|blue|bm|bmw|bn|bnpparibas|bo|boo|boutique|br|brussels|bs|bt|budapest|build|builders|business|buzz|bv|bw|by|bz|bzh|ca|cal|camera|camp|cancerresearch|canon|capetown|capital|caravan|cards|care|career|careers|cartier|casa|cash|cat|catering|cc|cd|center|ceo|cern|cf|cg|ch|channel|chat|cheap|christmas|chrome|church|ci|citic|city|ck|cl|claims|cleaning|click|clinic|clothing|club|cm|cn|co|coach|codes|coffee|college|cologne|com|community|company|computer|condos|construction|consulting|contractors|cooking|cool|coop|country|cr|credit|creditcard|cricket|crs|cruises|cu|cuisinella|cv|cw|cx|cy|cymru|cz|dabur|dad|dance|dating|day|dclk|de|deals|degree|delivery|democrat|dental|dentist|desi|design|dev|diamonds|diet|digital|direct|directory|discount|dj|dk|dm|dnp|do|docs|domains|doosan|durban|dvag|dz|eat|ec|edu|education|ee|eg|email|emerck|energy|engineer|engineering|enterprises|equipment|er|es|esq|estate|et|eu|eurovision|eus|events|everbank|exchange|expert|exposed|fail|farm|fashion|feedback|fi|finance|financial|firmdale|fish|fishing|fit|fitness|fj|fk|flights|florist|flowers|flsmidth|fly|fm|fo|foo|forsale|foundation|fr|frl|frogans|fund|furniture|futbol|ga|gal|gallery|garden|gb|gbiz|gd|ge|gent|gf|gg|ggee|gh|gi|gift|gifts|gives|gl|glass|gle|global|globo|gm|gmail|gmo|gmx|gn|goog|google|gop|gov|gp|gq|gr|graphics|gratis|green|gripe|gs|gt|gu|guide|guitars|guru|gw|gy|hamburg|hangout|haus|healthcare|help|here|hermes|hiphop|hiv|hk|hm|hn|holdings|holiday|homes|horse|host|hosting|house|how|hr|ht|hu|ibm|id|ie|ifm|il|im|immo|immobilien|in|industries|info|ing|ink|institute|insure|int|international|investments|io|iq|ir|irish|is|it|iwc|jcb|je|jetzt|jm|jo|jobs|joburg|jp|juegos|kaufen|kddi|ke|kg|kh|ki|kim|kitchen|kiwi|km|kn|koeln|kp|kr|krd|kred|kw|ky|kyoto|kz|la|lacaixa|land|lat|latrobe|lawyer|lb|lc|lds|lease|legal|lgbt|li|lidl|life|lighting|limited|limo|link|lk|loans|london|lotte|lotto|lr|ls|lt|ltda|lu|luxe|luxury|lv|ly|ma|madrid|maison|management|mango|market|marketing|marriott|mc|md|me|media|meet|melbourne|meme|memorial|menu|mg|mh|miami|mil|mini|mk|ml|mm|mn|mo|mobi|moda|moe|monash|money|mormon|mortgage|moscow|motorcycles|mov|mp|mq|mr|ms|mt|mu|museum|mv|mw|mx|my|mz|na|nagoya|name|navy|nc|ne|net|network|neustar|new|nexus|nf|ng|ngo|nhk|ni|ninja|nl|no|np|nr|nra|nrw|ntt|nu|nyc|nz|okinawa|om|one|ong|onl|ooo|org|organic|osaka|otsuka|ovh|pa|paris|partners|parts|party|pe|pf|pg|ph|pharmacy|photo|photography|photos|physio|pics|pictures|pink|pizza|pk|pl|place|plumbing|pm|pn|pohl|poker|porn|post|pr|praxi|press|pro|prod|productions|prof|properties|property|ps|pt|pub|pw|qa|qpon|quebec|re|realtor|recipes|red|rehab|reise|reisen|reit|ren|rentals|repair|report|republican|rest|restaurant|reviews|rich|rio|rip|ro|rocks|rodeo|rs|rsvp|ru|ruhr|rw|ryukyu|sa|saarland|sale|samsung|sarl|sb|sc|sca|scb|schmidt|schule|schwarz|science|scot|sd|se|services|sew|sexy|sg|sh|shiksha|shoes|shriram|si|singles|sj|sk|sky|sl|sm|sn|so|social|software|sohu|solar|solutions|soy|space|spiegel|sr|st|style|su|supplies|supply|support|surf|surgery|suzuki|sv|sx|sy|sydney|systems|sz|taipei|tatar|tattoo|tax|tc|td|technology|tel|temasek|tennis|tf|tg|th|tienda|tips|tires|tirol|tj|tk|tl|tm|tn|to|today|tokyo|tools|top|toshiba|town|toys|tp|tr|trade|training|travel|trust|tt|tui|tv|tw|tz|ua|ug|uk|university|uno|uol|us|uy|uz|va|vacations|vc|ve|vegas|ventures|versicherung|vet|vg|vi|viajes|video|villas|vision|vlaanderen|vn|vodka|vote|voting|voto|voyage|vu|wales|wang|watch|webcam|website|wed|wedding|wf|whoswho|wien|wiki|williamhill|wme|work|works|world|ws|wtc|wtf|xyz|yachts|yandex|ye|yoga|yokohama|youtube|yt|za|zm|zone|zuerich|zw))\b' }, + { 'type': 'mail', 'regex': r'\b([a-z][_a-z0-9-.+]+@[a-z0-9-.]+\.[a-z]+)\b' }, + { 'type': 'hash', 'regex': r'\b([a-f0-9]{32}|[A-F0-9]{32})\b' }, + { 'type': 'hash', 'regex': r'\b([a-f0-9]{40}|[A-F0-9]{40})\b' }, + { 'type': 'hash', 'regex': r'\b([a-f0-9]{64}|[A-F0-9]{64})\b' } + ] + + for o in observableTypes: + for match in re.findall(o['regex'], body, re.MULTILINE|re.IGNORECASE): + # Bug: If match is a tuple (example for domain or fqdn), use the 1st element + + if type(match) is tuple: + match = match[0] + observables.append({ 'type': o['type'], 'value': match }) + + + return observables + +def loadWhitelists(filename): + ''' + Read regex from the provided file, validate them and populate the list + ''' + if not filename: + return [] + + try: + lines = [line.rstrip('\n') for line in open(filename)] + except IOError as e: + log.error('Cannot read %s: %s' % (filename, e.strerror)) + sys.exit(1) + + i = 1 + w = [] + for l in lines: + if len(l) > 0: + if l[0] == '#': + # Skip comments and empty lines + continue + try: + re.compile(l) + except re.error: + log.error('Line %d: Regular expression "%s" is invalid.' % (l, f)) + sys.exit(1) + i += 1 + w.append(l) + return w - #alternate way to get the body - #soup = BeautifulSoup(email.body, 'html.parser') - #try: - # #html email - # body = soup.body.text - #except AttributeError: - # #non html email - # body = soup.text +def isWhitelisted(string): + ''' + Check if the provided string matches one of the whitelist regexes + ''' + global whitelists + whitelists = loadWhitelists(WHITELIST) + found = False + for w in whitelists: + if re.search(w, string, re.IGNORECASE): + found = True + break + return found - return ('```\n' + replyToInfo + body + '\n```') if __name__ == '__main__': - connectEws() + connectEws()