Support multiple STIX versions

.gitignore

@@ -109,3 +109,6 @@ venv.bak/
 
 # PyCharm
 .idea/
+
+# Vim
+*.swp

.pre-commit-config.yaml

@@ -2,12 +2,12 @@
     sha: ea227f024bd89d638aea319c92806737e3375979
     hooks:
     -   id: trailing-whitespace
-        exclude: stix2patterns/grammars/*
+        exclude: grammars
     -   id: flake8
+        exclude: grammars
         args:
         - --ignore=F403,F405
         - --max-line-length=160
-        - --exclude=stix2patterns/grammars/*
     -   id: check-merge-conflict
 -   repo: https://github.com/FalconSocial/pre-commit-python-sorter
     sha: b57843b0b874df1d16eb0bef00b868792cb245c2

MANIFEST.in

@@ -5,7 +5,8 @@ include CONTRIBUTING.md
 include dev-requirements.txt
 include LICENSE
 include tox.ini
-include stix2patterns/test/spec_examples.txt
+include stix2patterns/test/v20/spec_examples.txt
+include stix2patterns/test/v21/spec_examples.txt
 
 recursive-include docs *
 prune docs/_build

setup.py

@@ -43,7 +43,8 @@
         'typing ; python_version<"3.5" and python_version>="3"',
     ],
     package_data={
-        'stix2patterns.test': ['spec_examples.txt'],
+        'stix2patterns.test.v20': ['spec_examples.txt'],
+        'stix2patterns.test.v21': ['spec_examples.txt'],
     },
     entry_points={
         'console_scripts': [

stix2patterns/__init__.py

@@ -0,0 +1 @@
+DEFAULT_VERSION = '2.0'  # Default version should always be the latest STIX 2.X version

stix2patterns/exceptions.py

@@ -0,0 +1,28 @@
+from antlr4.error.ErrorListener import ErrorListener
+
+
+class STIXPatternErrorListener(ErrorListener):
+    """
+    Modifies ErrorListener to collect error message and set flag to False when
+    invalid pattern is encountered.
+    """
+    def __init__(self):
+        super(STIXPatternErrorListener, self).__init__()
+        self.err_strings = []
+
+    def syntaxError(self, recognizer, offendingSymbol, line, column, msg, e):
+        self.err_strings.append("FAIL: Error found at line %d:%d. %s" %
+                                (line, column, msg))
+
+
+class ParserErrorListener(ErrorListener):
+    """
+    Simple error listener which just remembers the last error message received.
+    """
+    def syntaxError(self, recognizer, offendingSymbol, line, column, msg, e):
+        self.error_message = u"{}:{}: {}".format(line, column, msg)
+
+
+class ParseException(Exception):
+    """Represents a parse error."""
+    pass

stix2patterns/inspector.py

@@ -1,7 +1,5 @@
 import collections
 
-from stix2patterns.grammars.STIXPatternListener import STIXPatternListener
-
 
 class InspectionException(Exception):
     """Represents a error that occurred during inspection."""
@@ -26,150 +24,3 @@ def _string_literal_to_string(string_literal_token):
     token_text = string_literal_token.getText()
     return token_text[1:-1].replace(u"\\'", u"'"). \
         replace(u"\\\\", u"\\")
-
-
-class InspectionListener(STIXPatternListener):
-    """This listener collects info about a pattern and puts it
-    in a python structure.  It is intended to assist apps which wish to
-    look "inside" a pattern and know what's in there.
-    """
-
-    def __init__(self):
-        self.__comparison_data = {}
-        self.__qualifiers = set()
-        self.__observation_ops = set()
-        self.__obj_type = None
-        self.__obj_path = None
-
-    def pattern_data(self):
-        return _PatternData(self.__comparison_data, self.__observation_ops,
-                            self.__qualifiers)
-
-    def __add_prop_tuple(self, obj_type, obj_path, op, value):
-        if obj_type not in self.__comparison_data:
-            self.__comparison_data[obj_type] = []
-
-        self.__comparison_data[obj_type].append((obj_path, op, value))
-
-    def exitObservationExpressions(self, ctx):
-        if ctx.FOLLOWEDBY():
-            self.__observation_ops.add(u"FOLLOWEDBY")
-
-    def exitObservationExpressionOr(self, ctx):
-        if ctx.OR():
-            self.__observation_ops.add(u"OR")
-
-    def exitObservationExpressionAnd(self, ctx):
-        if ctx.AND():
-            self.__observation_ops.add(u"AND")
-
-    def exitStartStopQualifier(self, ctx):
-        self.__qualifiers.add(
-            u"START {0} STOP {1}".format(
-                ctx.StringLiteral(0), ctx.StringLiteral(1)
-            )
-        )
-
-    def exitWithinQualifier(self, ctx):
-        self.__qualifiers.add(
-            u"WITHIN {0} SECONDS".format(
-                ctx.IntPosLiteral() or ctx.FloatPosLiteral()
-            )
-        )
-
-    def exitRepeatedQualifier(self, ctx):
-        self.__qualifiers.add(
-            u"REPEATS {0} TIMES".format(
-                ctx.IntPosLiteral()
-            )
-        )
-
-    def exitPropTestEqual(self, ctx):
-        op_tok = ctx.EQ() or ctx.NEQ()
-        op_str = u"NOT " if ctx.NOT() else u""
-        op_str += op_tok.getText()
-
-        value = ctx.primitiveLiteral().getText()
-
-        self.__add_prop_tuple(self.__obj_type, self.__obj_path, op_str,
-                              value)
-
-    def exitPropTestOrder(self, ctx):
-        op_tok = ctx.GT() or ctx.LT() or ctx.GE() or ctx.LE()
-        op_str = u"NOT " if ctx.NOT() else u""
-        op_str += op_tok.getText()
-
-        value = ctx.orderableLiteral().getText()
-
-        self.__add_prop_tuple(self.__obj_type, self.__obj_path, op_str,
-                              value)
-
-    def exitPropTestSet(self, ctx):
-        op_str = u"NOT " if ctx.NOT() else u""
-        op_str += u"IN"
-
-        value = ctx.setLiteral().getText()
-
-        self.__add_prop_tuple(self.__obj_type, self.__obj_path, op_str,
-                              value)
-
-    def exitPropTestLike(self, ctx):
-        op_str = u"NOT " if ctx.NOT() else u""
-        op_str += u"LIKE"
-
-        value = ctx.StringLiteral().getText()
-
-        self.__add_prop_tuple(self.__obj_type, self.__obj_path, op_str,
-                              value)
-
-    def exitPropTestRegex(self, ctx):
-        op_str = u"NOT " if ctx.NOT() else u""
-        op_str += u"MATCHES"
-
-        value = ctx.StringLiteral().getText()
-
-        self.__add_prop_tuple(self.__obj_type, self.__obj_path, op_str,
-                              value)
-
-    def exitPropTestIsSubset(self, ctx):
-        op_str = u"NOT " if ctx.NOT() else u""
-        op_str += u"ISSUBSET"
-
-        value = ctx.StringLiteral().getText()
-
-        self.__add_prop_tuple(self.__obj_type, self.__obj_path, op_str,
-                              value)
-
-    def exitPropTestIsSuperset(self, ctx):
-        op_str = u"NOT " if ctx.NOT() else u""
-        op_str += u"ISSUPERSET"
-
-        value = ctx.StringLiteral().getText()
-
-        self.__add_prop_tuple(self.__obj_type, self.__obj_path, op_str,
-                              value)
-
-    def exitObjectType(self, ctx):
-        self.__obj_type = ctx.getText()
-
-    def exitFirstPathComponent(self, ctx):
-        if ctx.StringLiteral():
-            path_component = _string_literal_to_string(ctx.StringLiteral())
-        else:
-            path_component = ctx.getText()
-
-        self.__obj_path = [path_component]
-
-    def exitKeyPathStep(self, ctx):
-        if ctx.IdentifierWithoutHyphen():
-            path_component = ctx.IdentifierWithoutHyphen().getText()
-        else:  # A StringLiteral
-            path_component = _string_literal_to_string(ctx.StringLiteral())
-
-        self.__obj_path.append(path_component)
-
-    def exitIndexPathStep(self, ctx):
-        if ctx.ASTERISK():
-            self.__obj_path.append(INDEX_STAR)
-        else:
-            self.__obj_path.append(int(ctx.IntPosLiteral().getText()))

stix2patterns/test/test_inspector.py → stix2patterns/test/v20/test_inspector.py

@@ -1,7 +1,7 @@
 import pytest
 
 from stix2patterns.inspector import INDEX_STAR
-from stix2patterns.pattern import Pattern
+from stix2patterns.v20.pattern import Pattern
 
 
 @pytest.mark.parametrize(u"pattern,expected_qualifiers", [

stix2patterns/test/v21/spec_examples.txt

@@ -0,0 +1,23 @@
+[file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f']
+[email-message:from_ref.value MATCHES '.+\\@example\\.com$' AND email-message:body_multipart[*].body_raw_ref.name MATCHES '^Final Report.+\\.exe$']
+[file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f' AND file:mime_type = 'application/x-pdf']
+[file:hashes.'SHA-256' = 'bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c' OR file:hashes.MD5 = 'cead3f77f6cda6ec00f57d76c9a6879f'] AND [file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f']
+([file:hashes.MD5 = '79054025255fb1a26e4bc422aef54eb4'] FOLLOWEDBY [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\foo\\bar']) WITHIN 300 SECONDS
+[user-account:account_type = 'unix' AND user-account:user_id = '1007' AND user-account:account_login = 'Peter'] AND [user-account:account_type = 'unix' AND user-account:user_id = '1008' AND user-account:account_login = 'Paul'] AND [user-account:account_type = 'unix' AND user-account:user_id = '1009' AND user-account:account_login = 'Mary']
+[artifact:mime_type = 'application/vnd.tcpdump.pcap' AND artifact:payload_bin MATCHES '\\xd4\\xc3\\xb2\\xa1\\x02\\x00\\x04\\x00']
+[file:name = 'foo.dll' AND file:parent_directory_ref.path = 'C:\\Windows\\System32']
+[file:extensions.'windows-pebinary-ext'.sections[*].entropy > 7.0]
+[file:mime_type = 'image/bmp' AND file:magic_number_hex = h'ffd8']
+[network-traffic:dst_ref.type = 'ipv4-addr' AND network-traffic:dst_ref.value = '203.0.113.33/32']
+[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] REPEATS 5 TIMES WITHIN 1800 SECONDS
+[domain-name:value = 'www.5z8.info' AND domain-name:resolves_to_refs[*].value = '198.51.100.1/32']
+[url:value = 'http://example.com/foo' OR url:value = 'http://example.com/bar']
+[x509-certificate:issuer = 'CN=WEBMAIL' AND x509-certificate:serial_number = '4c:0b:1d:19:74:86:a7:66:b4:1a:bf:40:27:21:76:28']
+[windows-registry-key:key = 'HKEY_CURRENT_USER\\Software\\CryptoLocker\\Files' OR windows-registry-key:key = 'HKEY_CURRENT_USER\\Software\\Microsoft\\CurrentVersion\\Run\\CryptoLocker_0388']
+[(file:name = 'pdf.exe' OR file:size = '371712') AND file:created = t'2014-01-13T07:03:17Z']
+[email-message:sender_ref.value = '[email protected]' AND email-message:subject = 'Conference Info']
+[x-usb-device:usbdrive.serial_number = '575833314133343231313937']
+[process:command_line MATCHES '^.+>-add GlobalSign.cer -c -s -r localMachine Root$'] FOLLOWEDBY [process:command_line MATCHES'^.+>-add GlobalSign.cer -c -s -r localMachineTrustedPublisher$'] WITHIN 300 SECONDS
+[network-traffic:dst_ref.value ISSUBSET '2001:0db8:dead:beef:0000:0000:0000:0000/64']
+([file:name = 'foo.dll'] AND [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\foo\\bar']) OR [process:name = 'fooproc' OR process:name = 'procfoo']
+[file:hashes.MD5 = 'cead3f77f6cda6ec00f57d76c9a69faa']

stix2patterns/test/v21/test_inspector.py

@@ -0,0 +1,71 @@
+import pytest
+
+from stix2patterns.inspector import INDEX_STAR
+from stix2patterns.v21.pattern import Pattern
+
+
+@pytest.mark.parametrize(u"pattern,expected_qualifiers", [
+    (u"[foo:bar = 1]", set()),
+    (u"[foo:bar = 1] REPEATS 5 TIMES", set([u"REPEATS 5 TIMES"])),
+    (u"[foo:bar = 1] WITHIN 10.3 SECONDS", set([u"WITHIN 10.3 SECONDS"])),
+    (u"[foo:bar = 1] WITHIN 123 SECONDS", set([u"WITHIN 123 SECONDS"])),
+    (u"[foo:bar = 1] START t'1932-11-12T15:42:15Z' STOP t'1964-10-23T21:12:26Z'",
+        set([u"START t'1932-11-12T15:42:15Z' STOP t'1964-10-23T21:12:26Z'"])),
+    (u"[foo:bar = 1] REPEATS 1 TIMES REPEATS 2 TIMES",
+        set([u"REPEATS 1 TIMES", u"REPEATS 2 TIMES"])),
+    (u"[foo:bar = 1] REPEATS 1 TIMES AND [foo:baz = 2] WITHIN 1.23 SECONDS",
+        set([u"REPEATS 1 TIMES", u"WITHIN 1.23 SECONDS"])),
+    (u"([foo:bar = 1] START t'1932-11-12T15:42:15Z' STOP t'1964-10-23T21:12:26Z' AND [foo:abc < h'12ab']) WITHIN 22 SECONDS "
+     u"OR [frob:baz NOT IN (1,2,3)] REPEATS 31 TIMES",
+        set([u"START t'1932-11-12T15:42:15Z' STOP t'1964-10-23T21:12:26Z'",
+            u"WITHIN 22 SECONDS", u"REPEATS 31 TIMES"]))
+])
+def test_qualifiers(pattern, expected_qualifiers):
+    compiled_pattern = Pattern(pattern)
+    pattern_data = compiled_pattern.inspect()
+
+    assert pattern_data.qualifiers == expected_qualifiers
+
+
+@pytest.mark.parametrize(u"pattern,expected_obs_ops", [
+    (u"[foo:bar = 1]", set()),
+    (u"[foo:bar = 1] AND [foo:baz > 25.2]", set([u"AND"])),
+    (u"[foo:bar = 1] OR [foo:baz != 'hello']", set([u"OR"])),
+    (u"[foo:bar = 1] FOLLOWEDBY [foo:baz IN (1,2,3)]", set([u"FOLLOWEDBY"])),
+    (u"[foo:bar = 1] AND [foo:baz = 22] OR [foo:abc = '123']", set([u"AND", u"OR"])),
+    (u"[foo:bar = 1] OR ([foo:baz = false] FOLLOWEDBY [frob:abc LIKE '123']) WITHIN 46.1 SECONDS",
+        set([u"OR", u"FOLLOWEDBY"]))
+])
+def test_observation_ops(pattern, expected_obs_ops):
+    compiled_pattern = Pattern(pattern)
+    pattern_data = compiled_pattern.inspect()
+
+    assert pattern_data.observation_ops == expected_obs_ops
+
+
+@pytest.mark.parametrize(u"pattern,expected_comparisons", [
+    (u"[foo:bar = 1]", {u"foo": [([u"bar"], u"=", u"1")]}),
+    (u"[foo:bar=1 AND foo:baz=2]", {u"foo": [([u"bar"], u"=", u"1"), ([u"baz"], u"=", u"2")]}),
+    (u"[foo:bar NOT !=1 OR bar:foo<12.3]", {
+        u"foo": [([u"bar"], u"NOT !=", u"1")],
+        u"bar": [([u"foo"], u"<", u"12.3")]
+    }),
+    (u"[foo:bar=1] OR [foo:baz MATCHES '123\\\\d+']", {
+        u"foo": [([u"bar"], u"=", u"1"), ([u"baz"], u"MATCHES", u"'123\\\\d+'")]
+    }),
+    (u"[foo:bar=1 AND bar:foo NOT >33] REPEATS 12 TIMES OR "
+     u"  ([baz:bar ISSUBSET '1234'] FOLLOWEDBY [baz:quux NOT LIKE 'a_cd'])",
+     {
+         u"foo": [([u"bar"], u"=", u"1")],
+         u"bar": [([u"foo"], u"NOT >", u"33")],
+         u"baz": [([u"bar"], u"ISSUBSET", u"'1234'"), ([u"quux"], u"NOT LIKE", u"'a_cd'")]
+     }),
+    (u"[obj-type:a.b[*][1].'c-d' NOT ISSUPERSET '1.2.3.4/16']", {
+        u"obj-type": [([u"a", u"b", INDEX_STAR, 1, u"c-d"], u"NOT ISSUPERSET", u"'1.2.3.4/16'")]
+    }),
+])
+def test_comparisons(pattern, expected_comparisons):
+    compiled_pattern = Pattern(pattern)
+    pattern_data = compiled_pattern.inspect()
+
+    assert pattern_data.comparisons == expected_comparisons