Added Scan Compare feature

This commit is contained in:
Captain-T2004 2024-08-26 04:02:07 +05:30
parent 8c86f6239b
commit de4e02c2b1
13 changed files with 575 additions and 5 deletions

View File

@ -27,6 +27,7 @@ from nettacker.api.helpers import structure
from nettacker.config import Config
from nettacker.core.app import Nettacker
from nettacker.core.die import die_failure
from nettacker.core.graph import create_compare_report
from nettacker.core.messages import messages as _
from nettacker.core.utils.common import now
from nettacker.database.db import (
@ -212,6 +213,38 @@ def new_scan():
return jsonify(vars(nettacker_app.arguments)), 200
@app.route("/compare/scans", methods=["POST"])
def compare_scans():
"""
compare two scans through the API
Returns:
Success if the comparision is successfull and report is saved and error if not.
"""
api_key_is_valid(app, flask_request)
scan_id_first = get_value(flask_request, "scan_id_first")
scan_id_second = get_value(flask_request, "scan_id_second")
if not scan_id_first or not scan_id_second:
return jsonify(structure(status="error", msg="Invalid Scan IDs")), 400
compare_report_path_filename = get_value(flask_request, "compare_report_path")
if not compare_report_path_filename:
compare_report_path_filename = nettacker_application_config["compare_report_path_filename"]
try:
result = create_compare_report(scan_id_first, scan_id_second, compare_report_path_filename)
if result:
return jsonify(
structure(
status="success",
msg="scan_comparison_completed",
)
), 200
return jsonify(structure(status="error", msg="Scan ID not found")), 404
except (FileNotFoundError, PermissionError, IOError):
return jsonify(structure(status="error", msg="Not a valid filepath")), 500
@app.route("/session/check", methods=["GET"])
def session_check():
"""

View File

@ -148,6 +148,12 @@ class DefaultSettings(ConfigBase):
usernames_list = None
verbose_event = False
verbose_mode = False
scan_compare_id = None
compare_report_path_filename = "{results_path}/results_{date_time}_{random_chars}.json".format(
results_path=PathConfig.results_dir,
date_time=now(format="%Y_%m_%d_%H_%M_%S"),
random_chars=generate_random_token(10),
)
class Config:

View File

@ -12,7 +12,7 @@ from nettacker import logger
from nettacker.config import Config, version_info
from nettacker.core.arg_parser import ArgParser
from nettacker.core.die import die_failure
from nettacker.core.graph import create_report
from nettacker.core.graph import create_report, create_compare_report
from nettacker.core.ip import (
get_ip_range,
generate_ip_range,
@ -216,6 +216,12 @@ class Nettacker(ArgParser):
exit_code = self.start_scan(scan_id)
create_report(self.arguments, scan_id)
if self.arguments.scan_compare_id is not None:
create_compare_report(
scan_id,
self.arguments.scan_compare_id,
self.arguments.compare_report_path_filename,
)
log.info(_("done"))
return exit_code
@ -243,6 +249,7 @@ class Nettacker(ArgParser):
"target": target,
"module_name": module_name,
"scan_id": scan_id,
"scan_compare_id": self.arguments.scan_compare_id,
}
)

View File

@ -390,6 +390,22 @@ class ArgParser(ArgumentParser):
default=Config.settings.ping_before_scan,
help=_("ping_before_scan"),
)
method_options.add_argument(
"-K",
"--scan-compare",
action="store",
dest="scan_compare_id",
default=Config.settings.scan_compare_id,
help=_("compare_scans"),
)
method_options.add_argument(
"-J",
"--compare-report-path",
action="store",
dest="compare_report_path_filename",
default=Config.settings.compare_report_path_filename,
help=_("compare_report_path_filename"),
)
# API Options
api_options = self.add_argument_group(_("API"), _("API_options"))

View File

@ -11,7 +11,7 @@ from nettacker.config import version_info
from nettacker.core.die import die_failure
from nettacker.core.messages import messages as _
from nettacker.core.utils.common import merge_logs_to_list, now
from nettacker.database.db import get_logs_by_scan_id, submit_report_to_db
from nettacker.database.db import get_logs_by_scan_id, submit_report_to_db, get_options_by_scan_id
log = logger.get_logger()
@ -42,6 +42,27 @@ def build_graph(graph_name, events):
return start(events)
def build_compare_report(compare_results):
"""
build the compare report
Args:
compare_results: Final result of the comparision(dict)
Returns:
report in html format
"""
log.info(_("build_compare_report"))
try:
build_report = getattr(
importlib.import_module("nettacker.lib.compare_report.engine"),
"build_report",
)
except ModuleNotFoundError:
die_failure(_("graph_module_unavailable").format("compare_report"))
log.info(_("finish_build_report"))
return build_report(compare_results)
def build_text_table(events):
"""
value['date'], value["target"], value['module_name'], value['scan_id'],
@ -77,6 +98,20 @@ def build_text_table(events):
)
def create_compare_text_table(results):
_table = texttable.Texttable()
table_headers = list(results.keys())
_table.add_rows([table_headers])
_table.add_rows(
[
table_headers,
[results[col] for col in table_headers],
]
)
_table.set_cols_width([len(i) for i in table_headers])
return _table.draw() + "\n\n"
def create_report(options, scan_id):
"""
sort all events, create log file in HTML/TEXT/JSON and remove old logs
@ -168,3 +203,76 @@ def create_report(options, scan_id):
log.info(_("file_saved").format(report_path_filename))
return True
def create_compare_report(scan_id, comp_id, filepath):
"""
if compare_id is given then create the report of comparision b/w scans
Args:
options: parsing options
scan_id: scan unique id
Returns:
True if success otherwise None
"""
scan_log_curr = get_logs_by_scan_id(scan_id)
scan_logs_comp = get_logs_by_scan_id(comp_id)
if not scan_log_curr:
log.info(_("no_events_for_report"))
return None
if not scan_logs_comp:
log.info(_("no_scan_to_compare"))
return None
scan_opts_curr = get_options_by_scan_id(scan_id)
scan_opts_comp = get_options_by_scan_id(comp_id)
def get_targets_set(item):
return tuple(json.loads(item["options"])["targets"])
curr_target_set = set(get_targets_set(item) for item in scan_opts_curr)
comp_target_set = set(get_targets_set(item) for item in scan_opts_comp)
def get_modules_ports(item):
return (item["target"], item["module_name"], item["port"])
curr_modules_ports = set(get_modules_ports(item) for item in scan_log_curr)
comp_modules_ports = set(get_modules_ports(item) for item in scan_logs_comp)
compare_results = {
"curr_scan_details": (scan_id, scan_log_curr[0]["date"]),
"comp_scan_details": (comp_id, scan_logs_comp[0]["date"]),
"curr_target_set": tuple(curr_target_set),
"comp_target_set": tuple(comp_target_set),
"curr_scan_result": tuple(curr_modules_ports),
"comp_scan_result": tuple(comp_modules_ports),
"new_targets_discovered": tuple(curr_modules_ports - comp_modules_ports),
"old_targets_not_detected": tuple(comp_modules_ports - curr_modules_ports),
}
compare_report_path_filename = filepath
if (
len(compare_report_path_filename) >= 5 and compare_report_path_filename[-5:] == ".html"
) or (len(compare_report_path_filename) >= 4 and compare_report_path_filename[-4:] == ".htm"):
html_report = build_compare_report(compare_results)
with open(compare_report_path_filename, "w", encoding="utf-8") as compare_report:
compare_report.write(html_report + "\n")
compare_report.close()
elif len(compare_report_path_filename) >= 5 and compare_report_path_filename[-5:] == ".json":
with open(compare_report_path_filename, "w", encoding="utf-8") as compare_report:
compare_report.write(str(json.dumps(compare_results)) + "\n")
compare_report.close()
elif len(compare_report_path_filename) >= 5 and compare_report_path_filename[-4:] == ".csv":
keys = compare_results.keys()
with open(compare_report_path_filename, "a") as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=keys)
if csvfile.tell() == 0:
writer.writeheader()
writer.writerow(compare_results)
csvfile.close()
else:
with open(compare_report_path_filename, "w", encoding="utf-8") as compare_report:
compare_report.write(create_compare_text_table(compare_results))
log.write(create_compare_text_table(compare_results))
log.info(_("compare_report_saved").format(compare_report_path_filename))
return True

View File

@ -117,6 +117,8 @@ def remove_old_logs(options):
HostsLog.target == options["target"],
HostsLog.module_name == options["module_name"],
HostsLog.scan_unique_id != options["scan_id"],
HostsLog.scan_unique_id != options["scan_compare_id"],
# Don't remove old logs if they are to be used for the scan reports
).delete(synchronize_session=False)
return send_submit_query(session)
@ -362,6 +364,21 @@ def get_logs_by_scan_id(scan_id):
]
def get_options_by_scan_id(scan_id):
"""
select all stored options of the scan by scan id hash
Args:
scan_id: scan id hash
Returns:
an array with a dict with stored options or an empty array
"""
session = create_connection()
return [
{"options": log.options}
for log in session.query(Report).filter(Report.scan_unique_id == scan_id).all()
]
def logs_to_report_json(target):
"""
select all reports of a host

View File

View File

@ -0,0 +1,21 @@
import json
from nettacker.config import Config
def build_report(compare_result):
"""
generate a report based on result of comparision b/w scans
Args:
compare_result: dict with result of the compare
Returns:
Compare report in HTML
"""
data = (
open(Config.path.web_static_dir / "report/compare_report.html")
.read()
.replace("__data_will_locate_here__", json.dumps(compare_result))
)
return data

View File

@ -117,4 +117,9 @@ username_list: username(s) list, separate with ","
verbose_mode: verbose mode level (0-5) (default 0)
wrong_hardware_usage: "You must select one of these profiles for hardware usage. (low, normal, high, maximum)"
invalid_scan_id: your scan id is not valid!
compare_scans: compare current scan to old scans using the unique scan_id
compare_report_path_filename: the file-path to store the compare_scan report
no_scan_to_compare: the scan_id to be compared not found
compare_report_saved: "compare results saved in {0}"
build_compare_report: "building compare report"
finish_build_report: "Finished building compare report"

View File

@ -2,7 +2,7 @@ body {
background: url("/img/background.jpeg")
}
#new_scan, #home, #get_results,#crawler_area {
#new_scan, #home, #get_results,#crawler_area,#compare_area{
background: #FEFCFF;
padding: 20px;
border-radius: 5px;

View File

@ -104,6 +104,11 @@
<li id="new_scan_ul">
<button id="new_scan_btn" class="btn btn-warning navbar-btn">New Scan</button>
</li>
<li id="compare_btn_ul">
<button id="compare_btn" class="btn btn-info navbar-btn">
Compare
</button>
</li>
<li id="logout_btn" class="hidden">
<button type="submit" action="javascript:do_logout()"
class="btn btn-info navbar-btn"
@ -371,6 +376,44 @@
</ul>
<br><br><br>
</div>
<script>
var compare_page = 1;
</script>
<div id="compare_area" class="hidden">
<h2>Compare Scan Results</h2>
<ul class="nav nav-tabs">
</ul><br>
<form id="compare_form">
<div class="input-group col-xs-5">
<span class="input-group-addon">scan ID</span>
<input id="scan_id_first" type="text" class="form-control" placeholder="Enter first scan ID">
</div>
<br>
<div class="input-group col-xs-5">
<span class="input-group-addon">scan ID</span>
<input id="scan_id_second" type="text" class="form-control" placeholder="Enter second scan ID">
</div>
<h3>Output(HTML/JSON/CSV/TXT)</h3>
<div class="input-group col-xs-5">
<span class="input-group-addon">filename</span>
<input id="compare_report_path" type="text" class="form-control" placeholder="Additional Info"
value="{% autoescape off %}{{filename}}{% endautoescape %}">
</div>
<br>
<ul id="create_compare_report" class="btn btn-primary" action="javascript:create_compare_report()">
Submit
</ul>
</form>
<br><br>
<div id="success_report" class="alert alert-success hidden text-justify">
<strong>Success!</strong> Compare report saved at the filepath<br>
</div>
<div id="failed_report" class="alert alert-danger hidden shake animated text-justify">
<strong>Failed!</strong><br>
<p id="report_error_msg"></p>
</div>
<br><br>
</div>
</body>
<br><br>
</div>

View File

@ -85,6 +85,7 @@ $(document).ready(function () {
$("#new_scan").addClass("hidden");
$("#get_results").addClass("hidden");
$("#crawler_area").addClass("hidden");
$("#compare_area").addClass("hidden");
$("#home").removeClass("hidden");
});
@ -100,6 +101,7 @@ $(document).ready(function () {
$("#get_results").addClass("hidden");
$("#crawler_area").addClass("hidden");
$("#login_first").addClass("hidden");
$("#compare_area").addClass("hidden");
$("#new_scan").removeClass("hidden");
})
.fail(function (jqXHR, textStatus, errorThrown) {
@ -107,6 +109,7 @@ $(document).ready(function () {
$("#get_results").addClass("hidden");
$("#crawler_area").addClass("hidden");
$("#new_scan").addClass("hidden");
$("#compare_area").addClass("hidden");
$("#login_first").removeClass("hidden");
});
});
@ -116,6 +119,7 @@ $(document).ready(function () {
$("#home").addClass("hidden");
$("#new_scan").addClass("hidden");
$("#crawler_area").addClass("hidden");
$("#compare_area").addClass("hidden");
$("#get_results").removeClass("hidden");
});
@ -124,9 +128,90 @@ $(document).ready(function () {
$("#home").addClass("hidden");
$("#new_scan").addClass("hidden");
$("#get_results").addClass("hidden");
$("#compare_area").addClass("hidden");
$("#crawler_area").removeClass("hidden");
});
// Compare scans
$("#compare_btn").click(function() {
$("#home").addClass("hidden");
$("#new_scan").addClass("hidden");
$("#get_results").addClass("hidden");
$("#crawler_area").addClass("hidden");
$("#compare_area").removeClass("hidden");
});
// Show the scan compare area
$("#compare_btn").click(function() {
$.ajax({
type: "GET",
url: "/session/check",
dataType: "text",
})
.done(function (res) {
$("#home").addClass("hidden");
$("#new_scan").addClass("hidden");
$("#get_results").addClass("hidden");
$("#crawler_area").addClass("hidden");
$("#login_first").addClass("hidden");
$("#compare_area").removeClass("hidden");
})
.fail(function (jqXHR, textStatus, errorThrown) {
$("#home").addClass("hidden");
$("#get_results").addClass("hidden");
$("#crawler_area").addClass("hidden");
$("#new_scan").addClass("hidden");
$("#compare_area").addClass("hidden");
$("#login_first").removeClass("hidden");
});
});
// Create the compare report
$("#create_compare_report").click(function() {
var tmp_data = {
scan_id_first: $("#scan_id_first").val(),
scan_id_second: $("#scan_id_second").val(),
compare_report_path: $("#compare_report_path").val(),
};
var key = "";
var data = {};
for (key in tmp_data) {
if (
tmp_data[key] != "" &&
tmp_data[key] != false &&
tmp_data[key] != null
) {
data[key] = tmp_data[key];
}
}
$.ajax({
type: "POST",
url: "/compare/scans",
data: data,
})
.done(function (response, textStatus, jqXHR) {
if (response.status === "success") {
$("#success_report").removeClass("hidden");
setTimeout('$("#success_report").addClass("animated fadeOut");', 5000);
setTimeout('$("#success_report").addClass("hidden");', 6000);
$("#success_report").removeClass("animated fadeOut");
}
else {
document.getElementById("report_error_msg").innerHTML = response.message;
$("#failed_report").removeClass("hidden");
setTimeout('$("#failed_report").addClass("hidden");', 5000);
}})
.fail(function (jqXHR, textStatus, errorThrown) {
var errorMessage = "An error occurred while comparing scans.";
if(jqXHR.responseJSON && jqXHR.responseJSON.msg){
errorMessage = jqXHR.responseJSON.msg;
}
document.getElementById("report_error_msg").innerHTML = errorMessage;
$("#failed_report").removeClass("hidden");
setTimeout('$("#failed_report").addClass("hidden");', 5000);
});
});
// start tutorial
$("#tutorial_btn").click(function () {
if ($("#logout_btn").is(":hidden")) {
@ -179,7 +264,7 @@ $(document).ready(function () {
position: "right",
},
{
element: document.querySelectorAll("#report_path_filename")[0],
element: document.querySelectorAll("#output_file")[0],
intro:
"Enter the location of the file you want your output in or leave it to the default value.",
position: "right",
@ -211,6 +296,12 @@ $(document).ready(function () {
"Click here to view all the results sorted by the target on which it was performed.",
position: "right",
},
{
element: document.querySelectorAll("#compare_btn_ul")[0],
intro:
"Click here to compare two scans and generate a compare report",
position: "right",
},
{
element: document.querySelectorAll("#logout_btn")[0],
intro: "Click here to destroy your session.",
@ -525,6 +616,7 @@ $(document).ready(function () {
$("#nxt_prv_btn").addClass("hidden");
$("#home").addClass("hidden");
$("#crawler_area").addClass("hidden");
$("#compare_area").addClass("hidden");
} else {
$("#login_first").addClass("hidden");
$("#scan_results").removeClass("hidden");
@ -894,6 +986,7 @@ function filter_large_content(content, filter_rate){
$("#crw_nxt_prv_btn").addClass("hidden");
$("#home").addClass("hidden");
$("#crawler_area").addClass("hidden");
$("#compare_area").addClass("hidden");
} else {
$("#login_first").addClass("hidden");
$("#crawl_results").removeClass("hidden");
@ -960,6 +1053,7 @@ function filter_large_content(content, filter_rate){
$("#crw_nxt_prv_btn").addClass("hidden");
$("#home").addClass("hidden");
$("#crawler_area").addClass("hidden");
$("#compare_area").addClass("hidden");
} else {
$("#login_first").addClass("hidden");
$("#crawl_results").removeClass("hidden");

View File

@ -0,0 +1,220 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Nettacker Scan Comparison Report</title>
<link href="https://fonts.googleapis.com/css2?family=Roboto:wght@400;700&family=Montserrat:wght@700&display=swap" rel="stylesheet">
<style>
:root {
--primary-color: #3498db;
--secondary-color: #2c3e50;
--background-color: #ecf0f1;
--text-color: #34495e;
--border-color: #bdc3c7;
}
body {
font-family: 'Roboto', sans-serif;
font-size: 16px;
line-height: 1.6;
color: var(--text-color);
background-color: var(--background-color);
margin: 0;
padding: 0;
}
.container {
max-width: 800px;
margin: 0 auto;
padding: 20px;
}
h1 {
font-family: 'Montserrat', sans-serif;
color: var(--secondary-color);
text-align: center;
font-size: 26px;
margin-bottom: 20px;
text-transform: uppercase;
letter-spacing: 1px;
border-bottom: 2px solid var(--primary-color);
padding-bottom: 10px;
}
h2 {
font-family: 'Montserrat', sans-serif;
color: var(--primary-color);
font-size: 20px;
margin-top: 20px;
margin-bottom: 10px;
}
.section {
background-color: #ffffff;
border: 1px solid var(--border-color);
border-radius: 8px;
padding: 15px;
margin-bottom: 15px;
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
}
.item {
margin-bottom: 10px;
padding: 10px;
background-color: #f8f9fa;
border: 1px solid var(--border-color);
border-radius: 5px;
}
.label {
font-weight: 700;
color: var(--primary-color);
margin-bottom: 3px;
font-size: 13px;
}
.value {
margin-left: 15px;
font-size: 15px;
}
.list {
list-style-type: none;
padding-left: 0;
margin: 0;
}
#list_yellow {
background-color: #ffe359;
font-weight: 10px;
}
#list_red {
font-weight: 10px;
background-color: red;
}
.list-item {
margin-bottom: 5px;
font-size: 15px;
}
@media (max-width: 600px) {
.container {
padding: 10px;
}
h1 {
font-size: 22px;
}
h2 {
font-size: 18px;
}
.section {
padding: 10px;
}
.item {
padding: 8px;
}
.label, .value, .list-item {
font-size: 14px;
}
}
</style>
</head>
<body>
<div class="container">
<h1>Nettacker Scan Comparison Report</h1>
<div id="report-container"></div>
</div>
<script>
const jsonData = __data_will_locate_here__;
const reportContainer = document.getElementById('report-container');
function generateReport(data) {
let reportContent = '';
// Scan Details Section
reportContent += `
<div class="section">
<h2>Scan Details</h2>
<div class="item">
<div class="label">Current Scan:</div>
<div class="value">ID: ${data.curr_scan_details[0]}</div>
<div class="value">Date: ${data.curr_scan_details[1]}</div>
</div>
<div class="item">
<div class="label">Comparison Scan:</div>
<div class="value">ID: ${data.comp_scan_details[0]}</div>
<div class="value">Date: ${data.comp_scan_details[1]}</div>
</div>
</div>`;
// Target Sets Section
reportContent += `
<div class="section">
<h2>Target Sets</h2>
<div class="item">
<div class="label">Current Targets:</div>
<ul class="list">
${data.curr_target_set[0].map(target => `<li class="list-item">${target}</li>`).join('')}
</ul>
</div>
<div class="item">
<div class="label">Comparison Targets:</div>
<ul class="list">
${data.comp_target_set[0].map(target => `<li class="list-item">${target}</li>`).join('')}
</ul>
</div>
</div>`;
// Scan Results Section
reportContent += `
<div class="section">
<h2>Scan Results</h2>
<div class="item">
<div class="label">Current Scan Results:</div>
<ul class="list">
${data.curr_scan_result.map(result => `<li class="list-item">Target: ${result[0]}, Scan Type: ${result[1]}, Port: ${result[2]}</li>`).join('')}
</ul>
</div>
<div class="item">
<div class="label">Comparison Scan Results:</div>
<ul class="list">
${data.comp_scan_result.map(result => `<li class="list-item">Target: ${result[0]}, Scan Type: ${result[1]}, Port: ${result[2]}</li>`).join('')}
</ul>
</div>
</div>`;
// Changes Section
reportContent += `
<div class="section">
<h2>Changes Detected</h2>
<div class="item">
<div class="label">New Targets Discovered:</div>
<ul class="list">
${data.new_targets_discovered.map(target => `<li class="list-item" id="list_yellow">Target: ${target[0]}, Scan Type: ${target[1]}, Port: ${target[2]}</li>`).join('')}
</ul>
</div>
<div class="item">
<div class="label">Old Targets Not Detected:</div>
<ul class="list" id="list_red">
${data.old_targets_not_detected.map(target => `<li class="list-item" id="list_red">Target: ${target[0]}, Scan Type: ${target[1]}, Port: ${target[2]}</li>`).join('')}
</ul>
</div>
</div>`;
reportContainer.innerHTML = reportContent;
}
generateReport(jsonData);
</script>
</body>
</html>