Merge branch 'develop/3.0.0' into 'master'

New release - 3.0.0

Closes #22, #19, and #32

See merge request cert/malduck!48

docs/conf.py

@@ -22,7 +22,7 @@
 author = 'CERT Polska'
 
 # The full version, including alpha/beta/rc tags
-release = '2.1.1'
+release = '3.0.0'
 
 
 # -- General configuration ---------------------------------------------------

docs/procmem.rst

@@ -1,4 +1,4 @@
-Memory model objects with PE support (procmem)
+Memory model objects (procmem)
 ==============================================
 
 .. automodule:: malduck.procmem
@@ -37,3 +37,11 @@ CuckooProcessMemory (cuckoomem)
 
 .. autoclass:: malduck.procmem.cuckoomem.CuckooProcessMemory
     :members:
+
+IDAProcessMemory (idamem)
+---------------------------------
+
+.. autoclass:: malduck.idamem
+
+.. autoclass:: malduck.procmem.idamem.IDAProcessMemory
+    :members:

malduck/__init__.py

@@ -8,7 +8,7 @@
 from .hash.crc import crc32
 from .hash.sha import md5, sha1, sha224, sha384, sha256, sha512
 from .string.inet import ipv4
-from .string.ops import asciiz, utf16z, chunks, enhex, unhex, uleb128
+from .string.ops import asciiz, utf16z, chunks, chunks_iter, enhex, unhex, uleb128
 from .structure import Structure
 
 from .pe import pe2cuckoo
@@ -19,7 +19,7 @@
 )
 
 from .short import (
-    aes, blowfish, des3, rc4, pe, aplib, gzip, procmem, procmempe, procmemelf, cuckoomem, pad, unpad,
+    aes, blowfish, des3, rc4, pe, aplib, gzip, procmem, procmempe, procmemelf, cuckoomem, idamem, pad, unpad,
     insn, rsa, verify, base64, rabbit, serpent, lznt1, pkcs7, unpkcs7
 )
 

malduck/compression/aplib.py

@@ -1,8 +1,10 @@
 from .components.aplib import ap_depack
 from ..py2compat import binary_type
 
+import logging
 import struct
-import warnings
+
+log = logging.getLogger(__name__)
 
 
 class aPLib(object):
@@ -29,7 +31,7 @@ class aPLib(object):
     """
     def decompress(self, buf, length=None, headerless=False):
         if length is not None:
-            warnings.warn("Length argument is ignored by aPLib.decompress")
+            log.warning("Length argument is ignored by aPLib.decompress")
         try:
             # Trim header
             if not headerless and buf.startswith(b"AP32"):

malduck/crypto/aes.py

@@ -57,7 +57,7 @@ def export_key(self):
 
 class AES(object):
     r"""
-    AES decryption object
+    AES encryption/decryption object
 
     :param key: Encryption key
     :type key: bytes
@@ -82,7 +82,18 @@ def __init__(self, key, iv=None, mode="cbc"):
         self.aes = Cipher(
             algorithms.AES(key), self.modes[mode](iv),
             backend=default_backend()
-        ).decryptor()
+        )
+
+    def encrypt(self, data):
+        """
+        Encrypt provided data
+
+        :param data: Buffer with data
+        :type data: bytes
+        :return: Encrypted data
+        """
+        aes_enc = self.aes.encryptor()
+        return aes_enc.update(data) + aes_enc.finalize()        
 
     def decrypt(self, data):
         """
@@ -92,7 +103,8 @@ def decrypt(self, data):
         :type data: bytes
         :return: Decrypted data
         """
-        return self.aes.update(data) + self.aes.finalize()
+        aes_dec = self.aes.decryptor()
+        return aes_dec.update(data) + aes_dec.finalize()
 
     @staticmethod
     def import_key(data):

malduck/extractor/extract_manager.py

@@ -1,12 +1,44 @@
+import json
+import logging
 import os
-import warnings
 
 from .extractor import Extractor
 from .loaders import load_modules
 from ..yara import Yara
 
+log = logging.getLogger(__name__)
 
-def merge_configs(config, new_config):
+
+def is_config_better(base_config, new_config):
+    """
+    Checks whether new config looks more reliable than base.
+    Currently just checking the amount of non-empty keys.
+    """
+    base = [(k, v) for k, v in base_config.items() if v]
+    new = [(k, v) for k, v in new_config.items() if v]
+    return len(new) > len(base)
+
+
+def sanitize_config(config):
+    """
+    Sanitize static configuration by removing empty strings/collections
+
+    :param config: Configuration to sanitize
+    :return: Sanitized configuration
+    """
+    return {k: v for k, v in config.items() if v in [0, False] or v}
+
+
+def merge_configs(base_config, new_config):
+    """
+    Merge static configurations.
+    Used internally. Removes "family" key from the result, which is set explicitly by ExtractManager.push_config
+
+    :param base_config: Base configuration
+    :param new_config: Changes to apply
+    :return: Merged configuration
+    """
+    config = dict(base_config)
     for k, v in new_config.items():
         if k == "family":
             continue
@@ -23,6 +55,7 @@ def merge_configs(config, new_config):
                                "value of '{key}' with '{new_value}'".format(key=k,
                                                                             old_value=config[k],
                                                                             new_value=v))
+    return config
 
 
 class ExtractorModules(object):
@@ -54,7 +87,7 @@ def on_error(self, exc, module_name):
         :param module_name: Name of module which throwed exception
         :type module_name: str
         """
-        warnings.warn("{} not loaded: {}".format(module_name, exc))
+        log.warning("{} not loaded: {}".format(module_name, exc))
 
 
 class ExtractManager(object):
@@ -112,56 +145,88 @@ def on_extractor_error(self, exc, extractor, method_name):
         :type method_name: str
         """
         import traceback
-        warnings.warn("{}.{} throwed exception: {}".format(
+        log.warning("{}.{} throwed exception: {}".format(
                       extractor.__class__.__name__,
                       method_name,
                       traceback.format_exc()))
 
-    def push_file(self, filepath, base=0, pe=None, elf=None, image=None):
+    def push_file(self, filepath, base=0):
         """
         Pushes file for extraction. Config extractor entrypoint.
-        
+
         :param filepath: Path to extracted file
         :type filepath: str
         :param base: Memory dump base address
         :type base: int
-        :param pe: Determines whether file contains PE (default: detect automatically)
-        :type pe: bool or None ("detect")
-        :param elf: Determines whether file contains ELF (default: detect automatically)
-        :type elf: bool or None ("detect")
-        :param image: If pe is True, determines whether file contains PE image (default: detect automatically)
-        :type image: bool or None ("detect")
-        """
-        from ..procmem import ProcessMemory, ProcessMemoryPE, ProcessMemoryELF
+        :return: Family name if ripped successfully and provided better configuration than previous files.
+                 Returns None otherwise.
+        """
+        from ..procmem import ProcessMemory
+        log.debug("Started extraction of file {}:{:x}".format(filepath, base))
         with ProcessMemory.from_file(filepath, base=base) as p:
-            if pe is None and p.readp(0, 2) == b"MZ":
-                pe = True
-            if elf is None and p.readp(0, 4) == b"\x7fELF":
-                elf = True
-            if pe and elf:
-                raise RuntimeError("A binary can't be both ELF and PE file")
-            if pe:
-                p = ProcessMemoryPE.from_memory(p, image=image, detect_image=image is None)
-            elif elf:
-                if image is False:
-                    raise RuntimeError("ELF dumps are not supported yet")
-                p = ProcessMemoryELF.from_memory(p, image=True)
-            self.push_procmem(p)
-
-    def push_procmem(self, p):
+            return self.push_procmem(p, rip_binaries=True)
+
+    def push_config(self, family, config):
+        config["family"] = family
+        if family not in self.configs:
+            self.configs[family] = config
+        else:
+            base_config = self.configs[family]
+            if is_config_better(base_config, config):
+                log.debug("Config looks better")
+                self.configs[family] = config
+                return family
+            else:
+                log.debug("Config doesn't look better - ignoring.")
+
+    def push_procmem(self, p, rip_binaries=False):
         """
         Pushes ProcessMemory object for extraction
 
         :param p: ProcessMemory object
         :type p: :class:`malduck.procmem.ProcessMemory`
+        :param rip_binaries: Look for binaries (PE, ELF) in provided ProcessMemory and try to perform extraction using
+        specialized variants (ProcessMemoryPE, ProcessMemoryELF)
+        :type rip_binaries: bool (default: False)
+        :return: Family name if ripped successfully and provided better configuration than previous procmems.
+                 Returns None otherwise.
         """
-        extractor = ProcmemExtractManager(self)
-        extractor.push_procmem(p)
-        if extractor.config:
-            if extractor.family not in self.configs:
-                self.configs[extractor.family] = extractor.config
+        from ..procmem import ProcessMemoryPE, ProcessMemoryELF
+        from ..procmem.binmem import ProcessMemoryBinary
+
+        binaries = [p]
+        if rip_binaries:
+            binaries += list(ProcessMemoryPE.load_binaries_from_memory(p)) + \
+                        list(ProcessMemoryELF.load_binaries_from_memory(p))
+        matches = p.yarav(self.rules)
+
+        fmt_procmem = lambda p: "{}:{}:{:x}".format(p.__class__.__name__,
+                                                    "IMG" if getattr(p, "is_image", False) else "DMP", p.imgbase)
+
+        def extract_config(procmem):
+            log.debug("{} - ripping...".format(fmt_procmem(procmem)))
+            extractor = ProcmemExtractManager(self)
+            matches.remap(procmem.p2v)
+            extractor.push_procmem(procmem, _matches=matches)
+            if extractor.family:
+                log.debug("{} - found {}!".format(fmt_procmem(procmem), extractor.family))
+                return self.push_config(extractor.family, extractor.config)
             else:
-                merge_configs(self.configs[extractor.family], extractor.config)
+                log.debug("{} - No luck.".format(fmt_procmem(procmem)))
+
+        log.debug("Matched rules: {}".format(matches.keys()))
+
+        ripped_family = None
+
+        for binary in binaries:
+            found_family = extract_config(binary)
+            if found_family is not None:
+                ripped_family = found_family
+            if isinstance(binary, ProcessMemoryBinary) and binary.image is not None:
+                found_family = extract_config(binary.image)
+                if found_family is not None:
+                    ripped_family = found_family
+        return ripped_family
 
     @property
     def config(self):
@@ -194,22 +259,24 @@ def on_extractor_error(self, exc, extractor, method_name):
         """
         self.parent.on_extractor_error(exc, extractor, method_name)
 
-    def push_procmem(self, p):
+    def push_procmem(self, p, _matches=None):
         """
         Pushes ProcessMemory object for extraction
 
         :param p: ProcessMemory object
         :type p: :class:`malduck.procmem.ProcessMemory`
+        :param _matches: YaraMatches object (used internally)
+        :type _matches: :class:`malduck.yara.YaraMatches`
         """
-        matched = p.yarav(self.parent.rules)
+        matches = _matches or p.yarav(self.parent.rules)
         # For each extractor...
         for ext_class in self.parent.extractors:
             extractor = ext_class(self)
             # For each rule identifier in extractor.yara_rules...
             for rule in extractor.yara_rules:
-                if rule in matched:
+                if rule in matches:
                     try:
-                        extractor.handle_yara(p, matched[rule])
+                        extractor.handle_yara(p, matches[rule])
                     except Exception as exc:
                         self.parent.on_error(exc, extractor)
 
@@ -226,24 +293,30 @@ def push_config(self, config, extractor):
         :param extractor: Extractor object reference
         :type extractor: :class:`malduck.extractor.Extractor`
         """
-        if "family" in config:
-            if not self.family or (
-                    self.family != extractor.family and
-                    self.family in extractor.overrides):
-                self.family = config["family"]
+        try:
+            json.dumps(config)
+        except (TypeError, OverflowError):
+            raise RuntimeError("Config must be JSON-encodable")
+
+        config = sanitize_config(config)
+
+        if not config:
+            return
 
-        new_config = dict(self.collected_config)
+        log.debug("%s found the following config parts: %s", extractor.__class__.__name__, sorted(config.keys()))
 
-        merge_configs(new_config, config)
+        self.collected_config = merge_configs(self.collected_config, config)
 
-        if self.family:
-            new_config["family"] = self.family
-        self.collected_config = new_config
+        if "family" in config and (
+                not self.family or (self.family != extractor.family and self.family in extractor.overrides)):
+            self.family = config["family"]
+            log.debug("%s tells it's %s", extractor.__class__.__name__, self.family)
 
     @property
     def config(self):
         """
-        Returns collected config, but if family is not matched - returns empty dict
+        Returns collected config, but if family is not matched - returns empty dict.
+        Family is not included in config itself, look at :py:attr:`ProcmemExtractManager.family`.
         """
         if self.family is None:
             return {}