mirror of https://github.com/OWASP/Nettacker.git
494 lines
15 KiB
Python
494 lines
15 KiB
Python
import copy
|
|
import ctypes
|
|
import datetime
|
|
import hashlib
|
|
import importlib
|
|
import math
|
|
import multiprocessing
|
|
import random
|
|
import re
|
|
import string
|
|
import sys
|
|
import time
|
|
from itertools import product
|
|
|
|
from nettacker import logger
|
|
from nettacker.core.utils.dsl_matcher import dsl_matcher
|
|
|
|
log = logger.get_logger()
|
|
|
|
|
|
def replace_dependent_response(log, response_dependent):
|
|
"""The `response_dependent` is needed for `eval` below."""
|
|
if str(log):
|
|
key_name = re.findall(re.compile("response_dependent\\['\\S+\\]"), log)
|
|
for i in key_name:
|
|
try:
|
|
key_value = eval(i)
|
|
except Exception:
|
|
key_value = "response dependent error"
|
|
log = log.replace(i, " ".join(key_value))
|
|
return log
|
|
|
|
|
|
def merge_logs_to_list(result, log_list=[]):
|
|
if isinstance(result, dict):
|
|
for i in result:
|
|
if "log" == i:
|
|
log_list.append(result["log"])
|
|
else:
|
|
merge_logs_to_list(result[i], log_list)
|
|
return list(set(log_list))
|
|
|
|
|
|
def reverse_and_regex_condition(regex, reverse):
|
|
if regex:
|
|
if reverse:
|
|
return []
|
|
return list(set(regex))
|
|
else:
|
|
if reverse:
|
|
return True
|
|
return []
|
|
|
|
|
|
def wait_for_threads_to_finish(threads, maximum=None, terminable=False, sub_process=False):
|
|
while threads:
|
|
try:
|
|
for thread in threads:
|
|
if not thread.is_alive():
|
|
threads.remove(thread)
|
|
if maximum and len(threads) < maximum:
|
|
break
|
|
time.sleep(0.01)
|
|
except KeyboardInterrupt:
|
|
if terminable:
|
|
for thread in threads:
|
|
terminate_thread(thread)
|
|
if sub_process:
|
|
for thread in threads:
|
|
thread.kill()
|
|
return False
|
|
return True
|
|
|
|
|
|
def terminate_thread(thread, verbose=True):
|
|
"""
|
|
kill a thread https://stackoverflow.com/a/15274929
|
|
Args:
|
|
thread: an alive thread
|
|
verbose: verbose mode/boolean
|
|
Returns:
|
|
True/None
|
|
"""
|
|
|
|
if verbose:
|
|
log.info("killing {0}".format(thread.name))
|
|
if not thread.is_alive():
|
|
return
|
|
|
|
exc = ctypes.py_object(SystemExit)
|
|
res = ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(thread.ident), exc)
|
|
if res == 0:
|
|
raise ValueError("nonexistent thread id")
|
|
elif res > 1:
|
|
# if it returns a number greater than one, you're in trouble,
|
|
# and you should call it again with exc=NULL to revert the effect
|
|
ctypes.pythonapi.PyThreadState_SetAsyncExc(thread.ident, None)
|
|
raise SystemError("PyThreadState_SetAsyncExc failed")
|
|
return True
|
|
|
|
|
|
def get_http_header_key(http_header):
|
|
"""
|
|
Return the HTTP header key based on the full string entered by the user
|
|
Args:
|
|
http_header: a string entered by the user following the -H flag
|
|
Returns:
|
|
1. The HTTP header key if http_header is a key-value pair
|
|
2. The http_header itself if http_header is NOT a key_value pair (i.e. http_header is a plain string)
|
|
3. An empty string if http_header is empty
|
|
Example:
|
|
http_header: "Authorization: Bearer abcdefgh"
|
|
Returns -> "Authorization"
|
|
"""
|
|
# Split only at the first ":"
|
|
return http_header.split(":", 1)[0].strip()
|
|
|
|
|
|
def get_http_header_value(http_header):
|
|
"""
|
|
Return the HTTP header value based on the full string entered by the user
|
|
Args:
|
|
http_header: a string entered by the user following the -H flag
|
|
Returns:
|
|
1. The HTTP header value if http_header is a key-value pair
|
|
2. None if the http_header is empty or NOT a key-value pair
|
|
Example:
|
|
http_header: "Authorization: Bearer abcdefgh"
|
|
Returns -> "Bearer abcdefgh"
|
|
"""
|
|
if not http_header or ":" not in http_header:
|
|
return None
|
|
# Split only at the first ":"
|
|
value = http_header.split(":", 1)[1].strip()
|
|
return value if value else None
|
|
|
|
|
|
def remove_sensitive_header_keys(event):
|
|
"""
|
|
Removes the sensitive headers that the user might add
|
|
Args:
|
|
event: The json event which contains the headers
|
|
Returns:
|
|
event: The json event without the sensitive headers
|
|
"""
|
|
from nettacker.config import sensitive_headers
|
|
|
|
if not isinstance(event, dict):
|
|
return event
|
|
|
|
if "headers" in event:
|
|
if not isinstance(event["headers"], dict):
|
|
return event
|
|
for key in list(event["headers"].keys()):
|
|
if key.lower() in sensitive_headers:
|
|
del event["headers"][key]
|
|
|
|
return event
|
|
|
|
|
|
def find_args_value(args_name):
|
|
try:
|
|
return sys.argv[sys.argv.index(args_name) + 1]
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def re_address_repeaters_key_name(key_name):
|
|
return "".join(["['" + _key + "']" for _key in key_name.split("/")[:-1]])
|
|
|
|
|
|
def generate_new_sub_steps(sub_steps, data_matrix, arrays):
|
|
original_sub_steps = copy.deepcopy(sub_steps)
|
|
steps_array = []
|
|
for array in data_matrix:
|
|
array_name_position = 0
|
|
for array_name in arrays:
|
|
for sub_step in sub_steps:
|
|
exec(
|
|
"original_sub_steps{key_name} = {matrix_value}".format(
|
|
key_name=re_address_repeaters_key_name(array_name),
|
|
matrix_value=(
|
|
'"' + str(array[array_name_position]) + '"'
|
|
if isinstance(array[array_name_position], int)
|
|
or isinstance(array[array_name_position], str)
|
|
else array[array_name_position]
|
|
),
|
|
)
|
|
)
|
|
array_name_position += 1
|
|
steps_array.append(copy.deepcopy(original_sub_steps))
|
|
return steps_array
|
|
|
|
|
|
def find_repeaters(sub_content, root, arrays):
|
|
if isinstance(sub_content, dict) and "nettacker_fuzzer" not in sub_content:
|
|
temporary_content = copy.deepcopy(sub_content)
|
|
original_root = root
|
|
for key in sub_content:
|
|
root = original_root
|
|
root += key + "/"
|
|
temporary_content[key], _root, arrays = find_repeaters(sub_content[key], root, arrays)
|
|
sub_content = copy.deepcopy(temporary_content)
|
|
root = original_root
|
|
if (not isinstance(sub_content, (bool, int, float))) and (
|
|
isinstance(sub_content, list) or "nettacker_fuzzer" in sub_content
|
|
):
|
|
arrays[root] = sub_content
|
|
return (sub_content, root, arrays) if root != "" else arrays
|
|
|
|
|
|
class value_to_class:
|
|
def __init__(self, value):
|
|
self.value = value
|
|
|
|
|
|
def class_to_value(arrays):
|
|
original_arrays = copy.deepcopy(arrays)
|
|
array_index = 0
|
|
for array in arrays:
|
|
value_index = 0
|
|
for value in array:
|
|
if isinstance(value, value_to_class):
|
|
original_arrays[array_index][value_index] = value.value
|
|
value_index += 1
|
|
array_index += 1
|
|
return original_arrays
|
|
|
|
|
|
def generate_and_replace_md5(content):
|
|
# todo: make it betetr and document it
|
|
md5_content = content.split("NETTACKER_MD5_GENERATOR_START")[1].split(
|
|
"NETTACKER_MD5_GENERATOR_STOP"
|
|
)[0]
|
|
md5_content_backup = md5_content
|
|
if isinstance(md5_content, str):
|
|
md5_content = md5_content.encode()
|
|
md5_hash = hashlib.md5(md5_content).hexdigest()
|
|
return content.replace(
|
|
"NETTACKER_MD5_GENERATOR_START" + md5_content_backup + "NETTACKER_MD5_GENERATOR_STOP",
|
|
md5_hash,
|
|
)
|
|
|
|
|
|
def generate_target_groups(targets, set_hardware_usage):
|
|
"""
|
|
Split a list of targets into smaller sublists based on a specified size.
|
|
"""
|
|
if not targets:
|
|
return targets
|
|
|
|
targets_total = len(targets)
|
|
split_size = min(set_hardware_usage, targets_total)
|
|
|
|
# Calculate the size of each chunk.
|
|
chunk_size = (targets_total + split_size - 1) // split_size
|
|
|
|
return [targets[i : i + chunk_size] for i in range(0, targets_total, chunk_size)]
|
|
|
|
|
|
def arrays_to_matrix(arrays):
|
|
"""
|
|
Generate a Cartesian product of input arrays as a list of lists.
|
|
"""
|
|
return [list(item) for item in product(*[arrays[array_name] for array_name in arrays])]
|
|
|
|
|
|
def string_to_bytes(string):
|
|
return string.encode()
|
|
|
|
|
|
AVAILABLE_DATA_FUNCTIONS = {
|
|
"passwords": {"read_from_file"},
|
|
"paths": {"read_from_file"},
|
|
"urls": {"read_from_file"},
|
|
}
|
|
|
|
|
|
def fuzzer_function_read_file_as_array(filename):
|
|
from nettacker.config import PathConfig
|
|
|
|
return open(PathConfig().payloads_dir / filename).read().split("\n")
|
|
|
|
|
|
def apply_data_functions(data):
|
|
def apply_data_functions_new():
|
|
if item not in AVAILABLE_DATA_FUNCTIONS:
|
|
return
|
|
|
|
for fn_name in data[item]:
|
|
if fn_name in AVAILABLE_DATA_FUNCTIONS[item]:
|
|
fn = getattr(importlib.import_module("nettacker.core.fuzzer"), fn_name)
|
|
if fn is not None:
|
|
original_data[item] = fn(data[item][fn_name])
|
|
|
|
def apply_data_functions_old():
|
|
function_results = {}
|
|
globals().update(locals())
|
|
exec(
|
|
"fuzzer_function = {fuzzer_function}".format(fuzzer_function=data[item]),
|
|
globals(),
|
|
function_results,
|
|
)
|
|
original_data[item] = function_results["fuzzer_function"]
|
|
|
|
original_data = copy.deepcopy(data)
|
|
for item in data:
|
|
if isinstance((data[item]), str) and data[item].startswith("fuzzer_function"):
|
|
apply_data_functions_old()
|
|
else:
|
|
apply_data_functions_new()
|
|
|
|
return original_data
|
|
|
|
|
|
def fuzzer_repeater_perform(arrays):
|
|
original_arrays = copy.deepcopy(arrays)
|
|
for array_name in arrays:
|
|
if "nettacker_fuzzer" not in arrays[array_name]:
|
|
continue
|
|
|
|
data = arrays[array_name]["nettacker_fuzzer"]["data"]
|
|
data_matrix = arrays_to_matrix(apply_data_functions(data))
|
|
prefix = arrays[array_name]["nettacker_fuzzer"]["prefix"]
|
|
input_format = arrays[array_name]["nettacker_fuzzer"]["input_format"]
|
|
interceptors = copy.deepcopy(arrays[array_name]["nettacker_fuzzer"]["interceptors"])
|
|
if interceptors:
|
|
interceptors = interceptors.split(",")
|
|
suffix = arrays[array_name]["nettacker_fuzzer"]["suffix"]
|
|
processed_array = []
|
|
|
|
for sub_data in data_matrix:
|
|
formatted_data = {}
|
|
index_input = 0
|
|
for value in sub_data:
|
|
formatted_data[list(data.keys())[index_input]] = value
|
|
index_input += 1
|
|
interceptors_function = ""
|
|
interceptors_function_processed = ""
|
|
|
|
if interceptors:
|
|
interceptors_function += "interceptors_function_processed = "
|
|
for interceptor in interceptors[::-1]:
|
|
interceptors_function += "{interceptor}(".format(interceptor=interceptor)
|
|
interceptors_function += "input_format.format(**formatted_data)" + str(
|
|
")" * interceptors_function.count("(")
|
|
)
|
|
expected_variables = {}
|
|
globals().update(locals())
|
|
exec(interceptors_function, globals(), expected_variables)
|
|
interceptors_function_processed = expected_variables[
|
|
"interceptors_function_processed"
|
|
]
|
|
else:
|
|
interceptors_function_processed = input_format.format(**formatted_data)
|
|
|
|
processed_sub_data = interceptors_function_processed
|
|
if prefix:
|
|
processed_sub_data = prefix + processed_sub_data
|
|
if suffix:
|
|
processed_sub_data = processed_sub_data + suffix
|
|
processed_array.append(copy.deepcopy(processed_sub_data))
|
|
original_arrays[array_name] = processed_array
|
|
|
|
return original_arrays
|
|
|
|
|
|
def expand_module_steps(content):
|
|
return [expand_protocol(x) for x in copy.deepcopy(content)]
|
|
|
|
|
|
def expand_protocol(protocol):
|
|
protocol["steps"] = [expand_step(x) for x in protocol["steps"]]
|
|
return protocol
|
|
|
|
|
|
def expand_step(step):
|
|
arrays = fuzzer_repeater_perform(find_repeaters(step, "", {}))
|
|
if arrays:
|
|
return generate_new_sub_steps(step, class_to_value(arrays_to_matrix(arrays)), arrays)
|
|
else:
|
|
# Minimum 1 step in array
|
|
return [step]
|
|
|
|
|
|
def generate_random_token(length=10):
|
|
return "".join(random.choice(string.ascii_lowercase) for _ in range(length))
|
|
|
|
|
|
def now(format="%Y-%m-%d %H:%M:%S"):
|
|
"""
|
|
get now date and time
|
|
Args:
|
|
format: the date and time model, default is "%Y-%m-%d %H:%M:%S"
|
|
|
|
Returns:
|
|
the date and time of now
|
|
"""
|
|
return datetime.datetime.now().strftime(format)
|
|
|
|
|
|
def select_maximum_cpu_core(mode):
|
|
cpu_count = multiprocessing.cpu_count()
|
|
|
|
if cpu_count - 1 == 0:
|
|
return 1
|
|
|
|
mode_core_mapping = {
|
|
"maximum": cpu_count - 1,
|
|
"high": cpu_count / 2,
|
|
"normal": cpu_count / 4,
|
|
"low": cpu_count / 8,
|
|
}
|
|
rounded = math.ceil if mode == "high" else math.floor
|
|
|
|
return int(max((rounded(mode_core_mapping.get(mode, 1)), 1)))
|
|
|
|
|
|
def sort_dictionary(dictionary):
|
|
etc_flag = "..." in dictionary
|
|
if etc_flag:
|
|
del dictionary["..."]
|
|
sorted_dictionary = {}
|
|
for key in sorted(dictionary):
|
|
sorted_dictionary[key] = dictionary[key]
|
|
if etc_flag:
|
|
sorted_dictionary["..."] = {}
|
|
return sorted_dictionary
|
|
|
|
|
|
def sanitize_path(path):
|
|
"""
|
|
Sanitize the file path to preven unathorized access
|
|
Args:
|
|
path: filepath(user input)
|
|
|
|
Returns:
|
|
sanitized_path
|
|
"""
|
|
return "_".join(
|
|
[
|
|
component
|
|
for component in re.split(r"[/\\]", path)
|
|
if re.match(r"^[a-zA-Z0-9_-]+(\.[a-zA-Z0-9_-]+)?$", component)
|
|
]
|
|
)
|
|
|
|
|
|
def generate_compare_filepath(scan_id):
|
|
return "/report_compare_{date_time}_{scan_id}.json".format(
|
|
date_time=now(format="%Y_%m_%d_%H_%M_%S"),
|
|
scan_id=scan_id,
|
|
)
|
|
|
|
|
|
def version_matches_dsl(detected_version, dsl_expression):
|
|
"""
|
|
Check if a detected version matches a DSL expression.
|
|
|
|
Supports various version comparison operators:
|
|
- Range operators: >=, <=, >, <, ==, !=
|
|
- Semantic operators: ~ (tilde), ^ (caret)
|
|
- Logical operators: ||, &&, or, and
|
|
- Range syntax: "1.0 to 2.0", "1.0 - 2.0"
|
|
- Wildcards: *, ?
|
|
|
|
Args:
|
|
detected_version: Version string detected from target
|
|
dsl_expression: DSL expression to match against
|
|
|
|
Returns:
|
|
bool: True if version matches the expression
|
|
|
|
Examples:
|
|
version_matches_dsl("2.4.51", ">=2.4.0, <2.4.54") # True
|
|
version_matches_dsl("1.2.3", "~1.2.0") # True (patch-level match)
|
|
version_matches_dsl("8.18.0", ">=8.0 && <9.0") # True
|
|
"""
|
|
return dsl_matcher.parse_dsl_expression(dsl_expression, detected_version)
|
|
|
|
|
|
def extract_version_from_content(content, patterns):
|
|
"""
|
|
Extract version from content using regex patterns.
|
|
|
|
Args:
|
|
content: String content to search
|
|
patterns: List of regex patterns to try
|
|
|
|
Returns:
|
|
str: Extracted version or None
|
|
"""
|
|
return dsl_matcher.extract_version_from_response(content, patterns)
|