Merge pull request #411 from target/yara-compiled-fix

Safely Account for a YARA Compilation Failure

src/python/strelka/scanners/scan_yara.py

@@ -58,7 +58,6 @@ def scan(self, data, file, options, expire_at):
             self.load_yara_rules(options)
             if not self.compiled_yara:
                 self.flags.append("no_rules_loaded")
-                return
 
         # Set the total rules loaded
         self.event["rules_loaded"] = self.rules_loaded
@@ -79,35 +78,36 @@ def scan(self, data, file, options, expire_at):
         self.event["hex"] = []
 
         # Match the data against the YARA rules.
-        yara_matches = self.compiled_yara.match(data=data)
-        for match in yara_matches:
-            # Append rule matches and update tags.
-            self.event["matches"].append(match.rule)
-            self.event["tags"].extend(match.tags)
-
-            # Extract hex representation if configured to store offsets.
-            if self.store_offset and self.offset_meta_key:
-                if match.meta.get(self.offset_meta_key):
-                    for string_data in match.strings:
-                        for instance in string_data.instances:
-                            offset = instance.offset
-                            matched_string = instance.matched_data
-                            self.extract_match_hex(
-                                match.rule,
-                                offset,
-                                matched_string,
-                                data,
-                                self.offset_padding,
-                            )
-
-            # Append meta information if configured to do so.
-            for k, v in match.meta.items():
-                self.event["meta"].append(
-                    {"rule": match.rule, "identifier": k, "value": v}
-                )
+        if self.compiled_yara:
+            yara_matches = self.compiled_yara.match(data=data)
+            for match in yara_matches:
+                # Append rule matches and update tags.
+                self.event["matches"].append(match.rule)
+                self.event["tags"].extend(match.tags)
+
+                # Extract hex representation if configured to store offsets.
+                if self.store_offset and self.offset_meta_key:
+                    if match.meta.get(self.offset_meta_key):
+                        for string_data in match.strings:
+                            for instance in string_data.instances:
+                                offset = instance.offset
+                                matched_string = instance.matched_data
+                                self.extract_match_hex(
+                                    match.rule,
+                                    offset,
+                                    matched_string,
+                                    data,
+                                    self.offset_padding,
+                                )
+
+                # Append meta information if configured to do so.
+                for k, v in match.meta.items():
+                    self.event["meta"].append(
+                        {"rule": match.rule, "identifier": k, "value": v}
+                    )
 
-        # De-duplicate tags.
-        self.event["tags"] = list(set(self.event["tags"]))
+            # De-duplicate tags.
+            self.event["tags"] = list(set(self.event["tags"]))
 
     def load_yara_rules(self, options):
         """Loads YARA rules based on the provided path.
@@ -159,7 +159,8 @@ def load_yara_rules(self, options):
             self.flags.append(f"compiling_error_syntax_{e}")
 
         # Set the total rules loaded.
-        self.rules_loaded = len(list(self.compiled_yara))
+        if self.compiled_yara:
+            self.rules_loaded = len(list(self.compiled_yara))
 
     def extract_match_hex(self, rule, offset, matched_string, data, offset_padding=32):
         """

src/python/strelka/tests/fixtures/test_elk_linux_torte.yara

@@ -0,0 +1,48 @@
+/*
+    This Yara ruleset is under the GNU-GPLv2 license (http://www.gnu.org/licenses/gpl-2.0.html) and open to any user or organization, as
+    long as you use it under this license.
+*/
+
+rule ELF_Linux_Torte : Linux ELF
+
+{
+    meta:
+		author = "@mmorenog,@yararules"
+		description = "Detects ELF Linux/Torte infection"
+		ref = "http://blog.malwaremustdie.org/2016/01/mmd-0050-2016-incident-report-elf.html"
+		hash1 = "1faf27f6b8e8a9cadb611f668a01cf73"
+		hash2 = "cb0477445fef9c5f1a5b6689bbfb941e"
+
+    strings:
+        $s0 = "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.6)"
+        $s1 = "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.7.6)"
+        $s2 = "?sessd="
+        $s3 = "&sessc="
+        $s4 = "&sessk="
+        $s5 = "3a08fe7b8c4da6ed09f21c3ef97efce2"
+        $s6 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
+        $s7 = "_ZN11CThreadPool10getBatchesERSt6vectorISt4pairISsiESaIS2_EE"
+        $s8 = "_ZNSs4_Rep10_M_destroyERKSaIcE@@GLIBCXX_3.4"
+        $s9 = "_ZNSt6vectorImSaImEE13_M_insert_auxEN9__gnu_cxx17__normal_iteratorIPmS1_EERKm"
+        $s10 = "_ZNSt6vectorISt4pairISsiESaIS1_EE13_M_insert_auxEN9__gnu_cxx17__normal_iteratorIPS1_S3_EERKS1_"
+        $s11 = "_ZSt20__throw_out_of_rangePKc@@GLIBCXX_3.4"
+
+        condition:
+        is__elf and all of ($s*)
+}
+
+
+rule ELF_Linux_Torte_domains {
+	meta:
+		author = "@mmorenog,@yararules"
+		description = "Detects ELF Linux/Torte infection"
+		ref1 = "http://blog.malwaremustdie.org/2016/01/mmd-0050-2016-incident-report-elf.html"
+	strings:
+		$1 = "pages.touchpadz.com" ascii wide nocase
+		$2 = "bat.touchpadz.com" ascii wide nocase
+		$3 = "stat.touchpadz.com" ascii wide nocase
+		$4 = "sk2.touchpadz.com" ascii wide nocase
+
+	condition:
+		any of them
+}

src/python/strelka/tests/test_scan_yara.py

@@ -38,6 +38,45 @@ def test_scan_yara(mocker):
     TestCase().assertDictEqual(test_scan_event, scanner_event)
 
 
+def test_scan_bad_yara(mocker):
+    """
+    This test was implemented to test a more complex and unsupported rule. A bug was observed that was
+    not triggered by the basic YARA test.
+    Src: https://github.com/target/strelka/issues/410
+    Pass: Sample event matches output of scanner.
+    Failure: Unable to load file or sample event fails to match.
+    """
+
+    test_scan_event = {
+        "elapsed": mock.ANY,
+        "flags": [
+            'compiling_error_general_/strelka/strelka/tests/fixtures/test_elk_linux_torte.yara(31): undefined identifier "is__elf"',
+            "no_rules_loaded",
+        ],
+        "matches": [],
+        "rules_loaded": 0,
+        "meta": mock.ANY,
+        "tags": [],
+        "hex": [],
+    }
+
+    scanner_event = run_test_scan(
+        mocker=mocker,
+        scan_class=ScanUnderTest,
+        fixture_path=Path(__file__).parent / "fixtures/test.txt",
+        options={
+            "location": str(Path(Path(__file__).parent / "fixtures/")),
+            "compiled": {
+                "enabled": False,
+                "filename": "rules.compiled",
+            },
+        },
+    )
+
+    TestCase.maxDiff = None
+    TestCase().assertDictEqual(test_scan_event, scanner_event)
+
+
 def test_scan_yara_hex_extraction(mocker):
     """
     Pass: Sample event matches output of scanner.