diff --git a/config.py b/config.py index d24bec3d..41a63453 100644 --- a/config.py +++ b/config.py @@ -117,6 +117,7 @@ def nettacker_user_application_config(): "time_sleep_between_requests": 0.0, "scan_ip_range": False, "scan_subdomains": False, + "skip_service_discovery": False, "thread_per_host": 100, "parallel_module_scan": 1, "socks_proxy": None, diff --git a/core/args_loader.py b/core/args_loader.py index 4a6be24d..d9cf2190 100644 --- a/core/args_loader.py +++ b/core/args_loader.py @@ -259,6 +259,13 @@ def load_all_args(): dest="scan_subdomains", help=messages("subdomains"), ) + modules.add_argument( + "--skip-service-discovery", + action="store_true", + default=nettacker_global_configuration['nettacker_user_application_config']["skip_service_discovery"], + dest="skip_service_discovery", + help=messages("skip_service_discovery") + ) modules.add_argument( "-t", "--thread-per-host", diff --git a/core/load_modules.py b/core/load_modules.py index 3cb24ad9..97902d8c 100644 --- a/core/load_modules.py +++ b/core/load_modules.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- - +import copy import os import socket +import yaml import time +import json from glob import glob from io import StringIO @@ -58,6 +60,16 @@ class NettackerModules: self.module_thread_number = None self.total_module_thread_number = None self.module_inputs = {} + self.skip_service_discovery = None + self.discovered_services = None + self.service_discovery_signatures = list(set(yaml.load( + StringIO( + open(nettacker_paths()['modules_path'] + '/scan/port.yaml').read().format( + **{'target': 'dummy'} + ) + ), + Loader=yaml.FullLoader + )['payloads'][0]['steps'][0]['response']['conditions'].keys())) self.libraries = [ module_protocol.split('.py')[0] for module_protocol in os.listdir(nettacker_paths()['module_protocols_path']) if @@ -65,9 +77,9 @@ class NettackerModules: ] def load(self): - import yaml from config import nettacker_paths from core.utility import find_and_replace_configuration_keys + from database.db import find_events self.module_content = find_and_replace_configuration_keys( yaml.load( StringIO( @@ -87,6 +99,33 @@ class NettackerModules: ), self.module_inputs ) + if not self.skip_service_discovery: + services = {} + for service in find_events(self.target, 'port_scan', self.scan_unique_id): + service_event = json.loads(service.json_event) + port = service_event['ports'] + protocols = service_event['response']['conditions_results'].keys() + for protocol in protocols: + if protocol in self.libraries and protocol: + if protocol in services: + services[protocol].append(port) + else: + services[protocol] = [port] + self.discovered_services = copy.deepcopy(services) + for payload in copy.deepcopy(self.module_content['payloads']): + if payload['library'] not in self.discovered_services and \ + payload['library'] in self.service_discovery_signatures: + del self.module_content['payloads'][self.module_content['payloads'].index(payload)] + else: + for step in copy.deepcopy( + self.module_content['payloads'][self.module_content['payloads'].index(payload)]['steps'] + ): + backup_step = copy.deepcopy(step) + step['ports'] = self.discovered_services[payload['library']] + self.module_content['payloads'][self.module_content['payloads'].index(payload)]['steps'][ + self.module_content['payloads'][self.module_content['payloads'].index(payload)][ + 'steps'].index(backup_step) + ] = step def generate_loops(self): from core.utility import expand_module_steps @@ -270,6 +309,7 @@ def perform_scan(options, target, module_name, scan_unique_id, process_number, t socket.socket, socket.getaddrinfo = set_socks_proxy(options.socks_proxy) options.target = target validate_module = NettackerModules() + validate_module.skip_service_discovery = options.skip_service_discovery validate_module.module_name = module_name validate_module.process_number = process_number validate_module.module_thread_number = thread_number diff --git a/core/module_protocols/socket.py b/core/module_protocols/socket.py index 7ad7ed9c..765579d4 100644 --- a/core/module_protocols/socket.py +++ b/core/module_protocols/socket.py @@ -48,33 +48,38 @@ def create_tcp_socket(host, ports, timeout): socket_connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM) socket_connection.settimeout(timeout) socket_connection.connect((host, int(ports))) + ssl_flag = False try: socket_connection = ssl.wrap_socket(socket_connection) + ssl_flag = True except Exception: socket_connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM) socket_connection.settimeout(timeout) socket_connection.connect((host, int(ports))) - return socket_connection + return socket_connection, ssl_flag class NettackerSocket: def tcp_connect_only(host, ports, timeout): - socket_connection = create_tcp_socket(host, ports, timeout) + socket_connection, ssl_flag = create_tcp_socket(host, ports, timeout) peer_name = socket_connection.getpeername() socket_connection.close() return { "peer_name": peer_name, - "service": socket.getservbyport(int(ports)) + "service": socket.getservbyport(int(ports)), + "ssl_flag": ssl_flag } def tcp_connect_send_and_receive(host, ports, timeout): - socket_connection = create_tcp_socket(host, ports, timeout) + socket_connection, ssl_flag = create_tcp_socket(host, ports, timeout) peer_name = socket_connection.getpeername() try: - socket_connection.send(b"ABC\x00\r\n" * 10) + socket_connection.send(b"ABC\x00\r\n\r\n\r\n" * 10) response = socket_connection.recv(1024 * 1024 * 10) + print(response) socket_connection.close() - except Exception: + except Exception as e: + print(e) try: socket_connection.close() response = b"" @@ -83,7 +88,8 @@ class NettackerSocket: return { "peer_name": peer_name, "service": socket.getservbyport(int(ports)), - "response": response.decode(errors='ignore') + "response": response.decode(errors='ignore'), + "ssl_flag": ssl_flag } def socket_icmp(host, timeout): @@ -219,7 +225,8 @@ class NettackerSocket: socket_connection.close() return { "host": host, - "response_time": delay + "response_time": delay, + "ssl_flag": False } @@ -260,6 +267,7 @@ class Engine: response = [] sub_step['method'] = backup_method sub_step['response'] = backup_response + sub_step['response']['ssl_flag'] = response['ssl_flag'] if type(response) == dict else False sub_step['response']['conditions_results'] = response_conditions_matched(sub_step, response) return process_conditions( sub_step, diff --git a/core/scan_targers.py b/core/scan_targers.py index 7f55a443..49541116 100644 --- a/core/scan_targers.py +++ b/core/scan_targers.py @@ -52,26 +52,16 @@ def parallel_scan_process(options, targets, scan_unique_id, process_number): return True -def start_scan_processes(options): - """ - preparing for attacks and managing multi-processing for host - - Args: - options: all options - - Returns: - True when it ends - """ - scan_unique_id = generate_random_token(32) - # find total number of targets + types + expand (subdomain, IPRanges, etc) - # optimize CPU usage - info(messages("regrouping_targets")) - options.targets = expand_targets(options, scan_unique_id) +def multi_processor(options, scan_unique_id): + if not options.targets: + info(messages("no_live_service_found")) + return True number_of_total_targets = len(options.targets) options.targets = [ targets.tolist() for targets in numpy.array_split( options.targets, - options.set_hardware_usage if options.set_hardware_usage <= len(options.targets) else len(options.targets) + options.set_hardware_usage if options.set_hardware_usage <= len(options.targets) + else number_of_total_targets ) ] info(messages("removing_old_db_records")) @@ -104,6 +94,28 @@ def start_scan_processes(options): ) process.start() active_processes.append(process) - exit_code = wait_for_threads_to_finish(active_processes, sub_process=True) - create_report(options, scan_unique_id) + return wait_for_threads_to_finish(active_processes, sub_process=True) + + +def start_scan_processes(options): + """ + preparing for attacks and managing multi-processing for host + + Args: + options: all options + + Returns: + True when it ends + """ + scan_unique_id = generate_random_token(32) + # find total number of targets + types + expand (subdomain, IPRanges, etc) + # optimize CPU usage + info(messages("regrouping_targets")) + options.targets = expand_targets(options, scan_unique_id) + if options.targets: + exit_code = multi_processor(options, scan_unique_id) + create_report(options, scan_unique_id) + else: + info(messages("no_live_service_found")) + exit_code = True return exit_code diff --git a/core/targets.py b/core/targets.py index 519fc09a..67f19169 100644 --- a/core/targets.py +++ b/core/targets.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- import copy import json +import os from core.ip import (get_ip_range, generate_ip_range, is_single_ipv4, @@ -13,6 +14,13 @@ from core.ip import (get_ip_range, from database.db import find_events +def filter_target_by_event(targets, scan_unique_id, module_name): + for target in copy.deepcopy(targets): + if not find_events(target, module_name, scan_unique_id): + targets.remove(target) + return targets + + def expand_targets(options, scan_unique_id): """ analysis and calulcate targets. @@ -24,7 +32,7 @@ def expand_targets(options, scan_unique_id): Returns: a generator """ - from core.load_modules import perform_scan + from core.scan_targers import multi_processor targets = [] for target in options.targets: if '://' in target: @@ -40,35 +48,59 @@ def expand_targets(options, scan_unique_id): # IP ranges elif is_ipv4_range(target) or is_ipv6_range(target) or is_ipv4_cidr(target) or is_ipv6_cidr(target): targets += generate_ip_range(target) - # domains - elif options.scan_subdomains: - targets.append(target) - perform_scan( - options, - target, - 'subdomain_scan', - scan_unique_id, - 'pre_process', - 'pre_process_thread', - 'unknown' - ) - for row in find_events(target, 'subdomain_scan', scan_unique_id): - for sub_domain in json.loads(row.json_event)['response']['conditions_results']['content']: - if sub_domain not in targets: - targets.append(sub_domain) + # domains probably else: targets.append(target) + options.targets = targets + + # subdomain_scan + if options.scan_subdomains: + selected_modules = options.selected_modules + options.selected_modules = ['subdomain_scan'] + multi_processor( + copy.deepcopy(options), + scan_unique_id + ) + options.selected_modules = selected_modules + if 'subdomain_scan' in options.selected_modules: + options.selected_modules.remove('subdomain_scan') + + for target in copy.deepcopy(options.targets): + for row in find_events(target, 'subdomain_scan', scan_unique_id): + for sub_domain in json.loads(row.json_event)['response']['conditions_results']['content']: + if sub_domain not in options.targets: + options.targets.append(sub_domain) + # icmp_scan if options.ping_before_scan: - for target in copy.deepcopy(targets): - perform_scan( - options, - target, - 'icmp_scan', - scan_unique_id, - 'pre_process', - 'pre_process_thread', - 'unknown' + if os.geteuid() == 0: + selected_modules = options.selected_modules + options.selected_modules = ['icmp_scan'] + multi_processor( + copy.deepcopy(options), + scan_unique_id ) - if not find_events(target, 'icmp_scan', scan_unique_id): - targets.remove(target) - return list(set(targets)) + options.selected_modules = selected_modules + if 'icmp_scan' in options.selected_modules: + options.selected_modules.remove('icmp_scan') + options.targets = filter_target_by_event(targets, scan_unique_id, 'icmp_scan') + else: + from core.alert import warn + from core.alert import messages + warn(messages("icmp_need_root_access")) + if 'icmp_scan' in options.selected_modules: + options.selected_modules.remove('icmp_scan') + # port_scan + if not options.skip_service_discovery: + options.skip_service_discovery = True + selected_modules = options.selected_modules + options.selected_modules = ['port_scan'] + multi_processor( + copy.deepcopy(options), + scan_unique_id + ) + options.selected_modules = selected_modules + if 'port_scan' in options.selected_modules: + options.selected_modules.remove('port_scan') + options.targets = filter_target_by_event(targets, scan_unique_id, 'port_scan') + options.skip_service_discovery = False + return list(set(options.targets)) diff --git a/lib/messages/en.yaml b/lib/messages/en.yaml index c7d8196b..a4b52038 100644 --- a/lib/messages/en.yaml +++ b/lib/messages/en.yaml @@ -10,6 +10,9 @@ API_key: " * API is accessible from https://nettacker-api.z3r0d4y.com:{0}/ via A API_options: API options API_port: API port number Method: Method +skip_service_discovery: skip service discovery before scan and enforce all modules to scan anyway +no_live_service_found: no any live service found to scan. +icmp_need_root_access: to use icmp_scan module or --ping-before-scan you need to run the script as root! available_graph: "build a graph of all activities and information, you must use HTML output. available graphs: {0}" browser_session_killed: your browser session killed browser_session_valid: your browser session is valid diff --git a/modules/scan/port.yaml b/modules/scan/port.yaml index 889bb47b..d0cdeb37 100644 --- a/modules/scan/port.yaml +++ b/modules/scan/port.yaml @@ -1031,7 +1031,7 @@ payloads: regex: "" reverse: false http: - regex: "HTTP\\/[\\d.]+\\s+[\\d]+|Server: |Content-Length: \\d+|Content-Type: |Access-Control-Request-Headers: |Forwarded: |Proxy-Authorization: |User-Agent: |X-Forwarded-Host: |Content-MD5: |Access-Control-Request-Method: |Accept-Language: " + regex: "HTTPStatus.BAD_REQUEST|HTTP\\/[\\d.]+\\s+[\\d]+|Server: |Content-Length: \\d+|Content-Type: |Access-Control-Request-Headers: |Forwarded: |Proxy-Authorization: |User-Agent: |X-Forwarded-Host: |Content-MD5: |Access-Control-Request-Method: |Accept-Language: " reverse: false ftp: regex: "220-You are user number|530 USER and PASS required|Invalid command: try being more creative|220 \\S+ FTP (Service|service|Server|server)|220 FTP Server ready|Directory status|Service closing control connection|Requested file action|Connection closed; transfer aborted|Directory not empty"