This repository has been archived by the owner on Jul 9, 2021. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathoutliers.conf
165 lines (132 loc) · 4.83 KB
/
outliers.conf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
###############################################
# Dnsbeat Proof-of-Concept Statistical Models #
###############################################
# TERMS - RARE RECORD
[terms_rare_record]
es_query_filter=NOT dns.type: ("RRSIG" OR "SOA")
aggregator=event.dataset
target=event.original
target_count_method=within_aggregator
trigger_on=low
trigger_method=pct_of_max_value
trigger_sensitivity=10
outlier_type=Rare Value
outlier_reason=Rare Record
outlier_summary=Rare record for '{event.dataset}'
run_model=1
test_model=0
# TERMS - RARE CNAME TARGET
[terms_rare_cname_target]
es_query_filter=dns.type: "CNAME" AND _exists_: dns.rdata.sld AND _exists_: dns.sld
aggregator=dns.sld
target=dns.rdata.sld
target_count_method=within_aggregator
trigger_on=low
trigger_method=pct_of_max_value
trigger_sensitivity=10
outlier_type=Rare Value
outlier_reason=Rare Target
outlier_summary=Rare target '{dns.rdata.sld}' for '{dns.sld}'
run_model=1
test_model=0
#TERMS - RARE MAIL RECORD
[terms_rare_mail_record]
es_query_filter=dns.type: "MX" OR (dns.type: "TXT" and dns.rdata.txt: ("*spf*" OR "*DKIM*" OR "*DMARC*"))
aggregator=event.dataset
target=event.original
target_count_method=within_aggregator
trigger_on=low
trigger_method=pct_of_max_value
trigger_sensitivity=10
outlier_type=Rare Value
outlier_reason=Rare Mail Record
outlier_summary=Rare '{dns.type}' mail-related record for '{event.dataset}'
run_model=1
test_model=0
##################################
# ee-outliers Base Configuration #
##################################
[general]
# Elasticsearch parameters
es_url=http://es01:9200
es_index_pattern=dnsbeat-*
es_scan_size=10000
es_scroll_time=25m
es_timeout=300
# The field name representing the event timestamp in Elasticsearch
timestamp_field=@timestamp
# Save outlier detection results to Elasticsearch (if set to 0, Elasticsearch events won't be touched - great for testing)
es_save_results=1
# Print outlier matches to the console. For testing purposes, it's advised to enable this so that the analyst can directly see on the command line which outliers are detected
print_outliers_to_console=1
# How far back should we process events and look for outliers?
# Both values are combined (for example the below will look back 7 days and 12 hours, up until right now).
history_window_days=7
history_window_hours=12
# Wipe all existing outliers that fall in the history window upon first run
es_wipe_all_existing_outliers=0
# Should existing outliers be checked (and removed) for a match with the configuration whitelist
es_wipe_all_whitelisted_outliers=0
# General flags to run, test or train models
# Training is only used by the Machine Learning models (word2vec)
run_models=1
test_models=0
train_models=0
# 0 for no progress info, 1-4 for progressively more output, 5+ for all the log output
log_verbosity=5
# CRITICAL ERROR WARNING INFO DEBUG
log_level=DEBUG
log_file=/mappedvolumes/logs/outliers.log
##############################
# ASSET FIELDS
##############################
[assets]
##############################
# NOTIFIER
##############################
[notifier]
email_notifier=0
smtp_user=<USERNAME>
smtp_pass=<PASSWORD>
smtp_server=<SMTP SERVER>
smtp_port=465
# Number of element keep in memory to avoid twice alerts for same notification
max_cache_ignore=1000
##############################
# DAEMON
##############################
[daemon]
# Only used when running ee-outliers in daemon mode. Below schedule will run each day at 00:00.
schedule=* * * * *
##############################
# TERMS PARAMETERS
##############################
[terms]
# Define how many events should be processed at the same time, before looking for outliers.
# More often means better results, but will result in increased memory usage.
terms_batch_eval_size=100000
##############################
# METRICS PARAMETERS
##############################
[metrics]
# Define how many events should be processed at the same time, before looking for outliers.
# More often means better results, but will result in increased memory usage.
metrics_batch_eval_size=100000
##############################
# MACHINE LEARNING PARAMETERS
##############################
[machine_learning]
# Defaults to 0, so all logs are shown. Set TF_CPP_MIN_LOG_LEVEL to 1 to filter out INFO logs, 2 to additionall filter out WARNING, 3 to additionally filter out ERROR.
tensorflow_log_level=2
word2vec_batch_eval_size=1000
word2vec_use_cache=1
word2vec_use_test_data=0
training_data_size_pct=100
training_steps=100000
models_directory=/tmp/ee-outliers/trained_models/
######################################################################################################################################################
# WHITELISTS
######################################################################################################################################################
[whitelist_literals]
[whitelist_regexps]