#!/usr/bin/env python
"""
ww - weaknesses walker
----------------------
Choose your tool between the possible ones:
- flawfinder
- rats
- its4
- vdb
Then, select the path you want to run them. The output format will be
the one used by the SATE project: http://samate.nist.gov/index.php/SATE
Disclaimer:
This tool doesn't mean to be that useful, it's only to run different type of tools
and get the answer with a common format.
ww Version 0.1 - Romain Gaucher - http://rgaucher.info
"""
import os,sys,re,popen2
from xml.sax import * # Need PyXML [http://pyxml.sourceforge.net/]
files_list = []
# lower case
extensions_list = ["c","cc","cpp","cxx","h","hh","hpp","hxx"]
tools_list = ["flawfinder","rats","its4","vdb"]
format_list = ["plain","original","sate"]
# basic configuration
t_name = ""
t_version = ""
file_out = "out.xml"
format = "both"
directory = "./"
is_file = False
xml_store = []
# regular expression for matching the outputs
re_ff = re.compile("^(.+):([\d]+): \[([\d]+)\] \((.*)\) (.*) $", re.I)
re_it = re.compile("^(.+):([\d]+):\((.*)\) (.+)$",re.I)
def strip_characters(str_buffer):
return str_buffer.replace('\n','')
def good_extension(file_name):
"""
Check whether the file is a possible source code (.c,.h,.cpp, etc.)
"""
if '.' not in file_name:
return False
file_name = file_name.lower()
file_ext = file_name[file_name.rfind('.')+1:]
if file_ext in extensions_list:
return True
return False
def get_list_files(directory):
"""
Retrieve the list of files with the good extension; will look inside subdirectories
"""
global files_list
names = os.listdir(directory)
for n in names:
srcname = os.path.join(directory,n)
try:
if os.path.isdir(srcname):
get_list_files(srcname)
elif os.path.isfile(srcname) and good_extension(srcname):
if srcname not in files_list:
files_list.append(srcname)
except (IOError, os.error), error_name:
print "get_list_files(%s) --> " % directory, error_name
def launch_process(cmd_str):
"""
Launch a processus and return the stdout/stderr output
"""
r,w,e = popen2.popen3(cmd_str)
b1 = e.readlines()
b2 = r.readlines()
r.close()
e.close()
w.close()
return (b1,b2)
def write_sate_xml_ff(ff_id,ff_fname,ff_nb,ff_severity,ff_type,ff_description):
xml = """
%s"""
return xml % (ff_id,ff_type,ff_fname,ff_nb,ff_severity,ff_description)
def run_flawfinder():
"""
Run flawfinder on a given project directory.
Flawfinder will do the scan of files itself
"""
ff_format = ""
if format == "sate":
ff_format = "--html"
cmd_str = "flawfinder %s %s" % (ff_format,directory)
(stderr,stdout) = launch_process(cmd_str)
if len(stdout) > 0 or len(stderr) > 0:
if len(stdout) == 0:
# just manage the error
for l in stderr:
print "",l
else:
# handling the normal output...
if format == "sate":
buffer = ''.join(stdout)
# looking between
output = strip_characters(buffer[buffer.find('
')+4 : buffer.rfind('
')])
# to convert, a vulnerability start with
and finish to the
output = output.split('
')
out_xml_str = ""
for l in output:
if re_ff.match(l):
out = re_ff.search(l)
ff_fname = out.group(1)
ff_line = out.group(2)
ff_severity = out.group(3)
ff_type = out.group(4)
ff_description = out.group(5)
out_xml_str += write_sate_xml_ff(hash(l), ff_fname, ff_line,ff_severity,ff_type,ff_description)
(foo,get_version) = launch_process("flawfinder --version")
ff_version = strip_characters(''.join(get_version))
return "\n" + out_xml_str + ""
else:
return '\n'.join(stdout)
else:
return False
return False
# Handle the RATS-XML file with a SAX Parser
class RATSXMLHandler(ContentHandler):
def __init__(self):
global xml_store
self.in_vulnerability = False
self.in_severity = False
self.in_type = False
self.in_message = False
self.in_file = False
self.in_name = False
self.in_line = False
self.string = ""
self.rt_dict = {}
self.name = ""
xml_store = []
def startElement(self, name, attrs):
if name == 'vulnerability':
self.in_vulnerability = True
elif name == 'severity':
self.in_severity = True
self.string = ""
elif name == "type":
self.in_type = True
self.string = ""
elif name == "message":
self.in_message = True
self.string = ""
elif name == "file":
self.in_file = True
elif name == "name":
self.in_name = True
self.string = ""
elif name == "line":
self.in_line = True
self.string = ""
def characters(self, ch):
self.string += ch
def endElement(self, name):
global xml_store
if name == 'vulnerability':
self.in_vulnerability = False
xml_store.append(self.rt_dict)
self.rt_dict = {}
elif name == 'severity' and self.in_vulnerability:
self.in_severity = False
self.rt_dict["grade"] = get_grade_from_rats(self.string)
self.string = ""
elif name == "type" and self.in_vulnerability:
self.in_type = False
self.rt_dict["name"] = self.string
self.string = ""
elif name == "message" and self.in_vulnerability:
self.in_message = False
if "description" not in self.rt_dict:
self.rt_dict["description"] = self.string
else:
self.rt_dict["description"] += "\n" + self.string
self.string = ""
elif name == "file" and self.in_vulnerability:
self.in_file = False
self.name = ""
elif name == "name" and self.in_file:
self.in_name = False
if "location" not in self.rt_dict:
self.rt_dict["location"] = []
self.name = self.string
self.string = ""
elif name == "line":
self.in_line = False
self.rt_dict["location"].append((self.name,self.string))
self.string = ""
def get_grade_from_rats(rt_string):
rt_string = rt_string.replace(' ','')
rt_string = rt_string.lower()
if rt_string == "high":
return "3"
elif rt_string == "medium":
return "2"
return "1"
def write_sate_xml_dict(elmt):
"""
Output the SATE XML form of the xml_store dict form
"""
xml = """
%s
%s
"""
rt_id = hash('|'.join(str(elmt.values())))
rt_type = elmt["name"]
rt_grade = elmt["grade"]
rt_description= elmt["description"]
rt_loc = ""
for e in elmt["location"]:
rt_loc += " \n" % (e[0],e[1])
return xml % (rt_id,rt_type,rt_loc,rt_grade,rt_description)
def run_rats():
"""
Run flawfinder on a given project directory.
Flawfinder will do the scan of files itself
"""
rt_format = ""
if format == "sate":
rt_format = "--xml"
cmd_str = "rats --resultsonly %s %s" % (rt_format,directory)
(stderr,stdout) = launch_process(cmd_str)
if len(stdout) > 0 or len(stderr) > 0:
if len(stdout) == 0:
# just manage the error
for l in stderr:
print "",l
else:
# handling the normal output...
if format == "sate":
buffer = ''.join(stdout)
# looking between
output = buffer
parser = make_parser()
rt_handler = RATSXMLHandler()
# Tell the parser to use our handler
parser.setContentHandler(rt_handler)
parser.feed(output)
out_xml_str = ""
for e in xml_store:
out_xml_str += write_sate_xml_dict(e)
(foo,get_version) = launch_process("rats -h")
rt_version = strip_characters(''.join(get_version[0]))
rt_version = rt_version.replace('RATS v','')
rt_version = rt_version.replace(' - Rough Auditing Tool for Security','')
return "\n" + out_xml_str + "\n"
else:
return '\n'.join(stdout)
else:
return False
return False
return
def get_grade_from_its(it_string):
it_string = it_string.replace(' ','')
it_string = it_string.lower()
if it_string == "urgent":
return "3"
elif it_string == "risky":
return "2"
return "1"
def run_its4():
global xml_store
"""
Run ITS4 on a given project directory.
Using the ww to look at the source code files; TS4 works on a single file
"""
it_buff = {}
it_join = " "
if not is_file:
get_list_files(directory)
for f in files_list:
cmd_str = "its4 %s" % (f)
(stderr,stdout) = launch_process(cmd_str)
if f not in it_buff:
it_buff[f] = strip_characters(it_join.join(stdout))
else:
cmd_str = "its4 %s" % (directory)
(stderr,stdout) = launch_process(cmd_str)
it_buff[directory] = strip_characters(it_join.join(stdout))
if format == "sate":
# Transform the buffers into XML SATE files
it_out_xml_buffer = ""
xml_store = []
for k in it_buff:
# analyze the report for each file
it_out_buffer = it_buff[k].split("----------------")
for r in it_out_buffer:
it_vuln_dict = {}
it_file_line = []
it_description = ""
lines = r.split(" ")
for l in lines:
if re_it.match(l):
# description line
out = re_it.search(l)
it_name = out.group(1)
it_line = out.group(2)
it_severity = out.group(3)
it_type = out.group(4)
it_file_line.append((it_name, it_line))
else:
it_description += l
if len(it_file_line) > 0:
it_vuln_dict["name"] = it_type
it_vuln_dict["location"] = it_file_line
it_vuln_dict["grade"] = get_grade_from_its(it_severity)
it_vuln_dict["description"]= it_description
xml_store.append(it_vuln_dict)
out_xml_str = ""
for e in xml_store:
out_xml_str += write_sate_xml_dict(e)
(foo,get_version) = launch_process("its4 --version")
it_version = strip_characters(''.join(get_version[0]))
it_version = it_version.replace('It\'s the software, stupid! (Security Scanner) Version ','')
it_version = it_version[:it_version.find(',')+1]
return "\n" + out_xml_str + "\n"
else:
# Just return the concatenated buffers
it_out_buffer = ""
for k in it_buff:
it_out_buffer += "------------------------------------------\n"
it_out_buffer += "File: %s\n" % k
it_out_buffer += "\n%s" % it_buff[k].replace(" ","\n")
return it_out_buffer
return ""
# Levenstein distance
# stolen from m.l. hetland
def ld(a, b):
n, m = len(a), len(b)
if n > m:
a,b = b,a
n,m = m,n
current = xrange(n+1)
for i in xrange(1,m+1):
previous, current = current, [i]+[0] * m
for j in xrange(1, n+1):
add, delete = previous[j] + 1, current[j-1] + 1
change = previous[j-1]
if a[j-1] != b[i-1]:
change +=1
current[j] = min(add, delete, change)
return current[n]
def run_vdb():
global t_name, t_version
"""
Looking at the NVD and trying to find vulnerabilities for the given product/version
The following code is absolutely not optimized...
"""
from wwwCall import wwwCall
http = wwwCall('http://nvd.nist.gov')
handler = http.get("http://nvd.nist.gov/nvd.cfm?advancedsearch&productstart=10")
t_name = t_name.lower()
html = handler.read().lower()
if t_name not in html:
# Look for the product with the closest name
html = re.sub("<.*?>","",html)
html_list = html.split(' ')
min_value = 42
min_word = "__false__"
for w in html_list:
c_value = ld(t_name,w)
if c_value < min_value:
min_word = w
min_value= c_value
if min_word == "__false__":
print " ww cannot find such a project in the current NVD"
return "\n\n" % t_name
print " NVD: ww couldn't find the exact project name in the NVD and choose to look for '%s'" % min_word
# The product exists in the NVD
handler = http.get("http://nvd.nist.gov/nvd.cfm?advancedsearch&product_command="+t_name)
html = handler.read().lower()
buffer = html[html.find("")]
html = re.sub("<.*?>","",buffer)
if t_version in html:
# Retrieve the vulnerabilities
handler = http.post("http://nvd.nist.gov/nvd.cfm?startrow=1",{"Search":"Search","product":t_name,"version":t_version,"resources":"cve"})
if handler:
html = handler.read()
match= "
"
html = html[html.find(match)+len(match):]
html = html[:html.find("
")]
html = html[:html.rfind("
")]
list_vulns = html.split(match)
out_xml_sate = ""
xml_fmt = """\n\t\n\t\n\n"""
for vuln_instance in list_vulns:
vuln_instance = vuln_instance[vuln_instance.find("Summary:"):vuln_instance.rfind(' ')]
vuln_instance = re.sub("<.*?>","",vuln_instance)
v_id = hash(vuln_instance)
v_description = vuln_instance[vuln_instance.find("Summary:")+9:vuln_instance.find("Published")]
v_severity = vuln_instance[vuln_instance.find("CVSS Severity:")+14:vuln_instance.rfind("CVE")]
v_description = ' '.join(v_description.split())
v_severity = re.sub("[^a-zA-Z]+","",v_severity)
if v_severity == "High":
v_severity = "3"
elif v_severity == "Medium":
v_severity = "2"
else:
v_severity = "1"
out_xml_sate += xml_fmt % (v_id,v_severity,v_description)
return "\n"+ out_xml_sate +"\n"
else:
print " ww cannot find such a project/version in the current NVD"
return "\n\n" % (t_name,t_version)
return "\n\n"
def help():
print " Weaknesses Walker: "
print " Usage: ./ww.py --tool [rats|flawfinder|its4] --file [fname] --format [sate|original] [directory]"
print " ./ww.py --vdb [project name] [version] --file [fname]"
print " 'ww' will look at the directory for file names then, run the selected tool against the files or the"
print " directory, depending on how the tools work. You can keep the original format or use the SATE ones, "
print " depending on what you want."
print " The --vdb options is sets to look at the NVD and retrieve the vulnerabilites in SATE format."
def tool_name(n):
"""
Check whether the name of the tool is correct or not; if so, return the name (lowered)
"""
n = n.lower()
if n in tools_list:
return n
return None
def format_name(n):
"""
Check whether the format is correct or not; if so, return the format name (lowered)
"""
n = n.lower()
if n in format_list:
return n
"""
Main dispatcher
"""
if __name__ == "__main__":
nargs = len(sys.argv)
if nargs == 8:
for i in range(nargs):
s = sys.argv[i]
if s == "--tool":
t_name = tool_name(sys.argv[i+1])
elif s == "--file":
file_out = sys.argv[i+1]
elif s == "--format":
format = format_name(sys.argv[i+1])
directory = sys.argv[nargs-1]
# You can also test with a single file
if not os.path.isdir(directory) and os.path.isfile(directory):
is_file = True
# print the information of the current scan
print " Scanning %s with %s; the output (%s) will be in %s format" % (directory,t_name,file_out,format)
# cleaning the listing
files_list = []
# basic dispatcher
out = None
try:
if t_name == "flawfinder":
out = run_flawfinder()
elif t_name == "rats":
out = run_rats()
elif t_name == "its4":
out = run_its4()
else:
out = run_vdb()
except KeyboardInterrupt:
print " Stopped by user"
try:
f_out = open(file_out,"w")
f_out.write(out)
f_out.close()
except IOError:
print " Error, cannot write the output file:",file_out
elif nargs == 6:
# looking at the NVD
for i in range(nargs):
s = sys.argv[i]
if s == "--vdb":
t_name = sys.argv[i+1]
t_version = sys.argv[i+2]
elif s == "--file":
file_out = sys.argv[i+1]
try:
out = run_vdb()
except KeyboardInterrupt:
print " Stopped by user"
try:
f_out = open(file_out,"w")
f_out.write(out)
f_out.close()
except IOError:
print " Error, cannot write the output file:",file_out
else:
help()