|
@@ -7,6 +7,7 @@
|
|
|
|
|
|
import logging
|
|
import logging
|
|
import re
|
|
import re
|
|
|
|
+import os
|
|
import pytablewriter
|
|
import pytablewriter
|
|
|
|
|
|
import mpp.api
|
|
import mpp.api
|
|
@@ -17,167 +18,24 @@ DIGIT_COUNT = 8
|
|
|
|
|
|
class Plugin(mpp.api.Plugin, mpp.api.IConfigurable, mpp.api.IRunable):
|
|
class Plugin(mpp.api.Plugin, mpp.api.IConfigurable, mpp.api.IRunable):
|
|
|
|
|
|
- MODE_NEW = 0x01
|
|
|
|
- MODE_TREND = 0x03
|
|
|
|
- MODE_TOUCHED = 0x07
|
|
|
|
- MODE_ALL = 0x15
|
|
|
|
-
|
|
|
|
def declare_configuration(self, parser):
|
|
def declare_configuration(self, parser):
|
|
self.parser = parser
|
|
self.parser = parser
|
|
parser.add_option("--output-dir", "--od", default='./metrixpp/',
|
|
parser.add_option("--output-dir", "--od", default='./metrixpp/',
|
|
help="Set the output folder. [default: %default].")
|
|
help="Set the output folder. [default: %default].")
|
|
- parser.add_option("--format", "--ft", default='txt', choices=['txt', 'md', 'html', 'rst', 'latex', 'doxygen'],
|
|
|
|
|
|
+ parser.add_option("--format", "--ft", default='txt', choices=['txt', 'md', 'html', 'rst', 'latex', 'xlsx', 'doxygen'],
|
|
help="Format of the output data. "
|
|
help="Format of the output data. "
|
|
- "Possible values are 'txt', 'md', 'html', 'rst', 'latex' or 'doxygen' [default: %default]")
|
|
|
|
- parser.add_option("--disable-suppressions", "--ds", action="store_true", default=False,
|
|
|
|
- help = "If not set (none), all suppressions are ignored"
|
|
|
|
- " and associated warnings are printed. [default: %default]")
|
|
|
|
- parser.add_option("--warn-mode", "--wm", default='all', choices=['new', 'trend', 'touched', 'all'],
|
|
|
|
- help="Defines the warnings mode. "
|
|
|
|
- "'all' - all warnings active, "
|
|
|
|
- "'new' - warnings for new regions/files only, "
|
|
|
|
- "'trend' - warnings for new regions/files and for bad trend of modified regions/files, "
|
|
|
|
- "'touched' - warnings for new and modified regions/files "
|
|
|
|
- "[default: %default]")
|
|
|
|
- parser.add_option("--min-limit", "--min", action="multiopt",
|
|
|
|
- help="A threshold per 'namespace:field' metric in order to select regions, "
|
|
|
|
- "which have got metric value less than the specified limit. "
|
|
|
|
- "This option can be specified multiple times, if it is necessary to apply several limits. "
|
|
|
|
- "Should be in the format: <namespace>:<field>:<limit-value>[:region_type[,region_type]], for example: "
|
|
|
|
- "'std.code.lines:comments:1', or 'std.code.lines:comments:1:function,class'. "
|
|
|
|
- "Region types is optional specifier, and if not defined the limit is applied to regions of all types.")
|
|
|
|
- parser.add_option("--max-limit", "--max", action="multiopt",
|
|
|
|
- help="A threshold per 'namespace:field' metric in order to select regions, "
|
|
|
|
- "which have got metric value more than the specified limit. "
|
|
|
|
- "This option can be specified multiple times, if it is necessary to apply several limits. "
|
|
|
|
- "Should be in the format: <namespace>:<field>:<limit-value>[:region_type[,region_type]], for example: "
|
|
|
|
- "'std.code.complexity:cyclomatic:7', or 'std.code.complexity:maxdepth:5:function'. "
|
|
|
|
- "Region types is optional specifier, and if not defined the limit is applied to regions of all types.")
|
|
|
|
|
|
+ "Possible values are 'txt', 'md', 'html', 'rst', 'latex', 'xlsx' or 'doxygen' [default: %default]")
|
|
|
|
|
|
def configure(self, options):
|
|
def configure(self, options):
|
|
self.out_dir = options.__dict__['output_dir']
|
|
self.out_dir = options.__dict__['output_dir']
|
|
self.out_format = options.__dict__['format']
|
|
self.out_format = options.__dict__['format']
|
|
- self.no_suppress = options.__dict__['disable_suppressions']
|
|
|
|
-
|
|
|
|
- if options.__dict__['warn_mode'] == 'new':
|
|
|
|
- self.mode = self.MODE_NEW
|
|
|
|
- elif options.__dict__['warn_mode'] == 'trend':
|
|
|
|
- self.mode = self.MODE_TREND
|
|
|
|
- elif options.__dict__['warn_mode'] == 'touched':
|
|
|
|
- self.mode = self.MODE_TOUCHED
|
|
|
|
- elif options.__dict__['warn_mode'] == 'all':
|
|
|
|
- self.mode = self.MODE_ALL
|
|
|
|
-
|
|
|
|
- if self.mode != self.MODE_ALL and options.__dict__['db_file_prev'] == None:
|
|
|
|
- self.parser.error("option --warn-mode: The mode '" + options.__dict__['warn_mode'] + "' requires '--db-file-prev' option set")
|
|
|
|
-
|
|
|
|
- class Limit(object):
|
|
|
|
- def __init__(self, limit_type, limit, namespace, field, db_filter, region_types, original):
|
|
|
|
- self.type = limit_type
|
|
|
|
- self.limit = limit
|
|
|
|
- self.namespace = namespace
|
|
|
|
- self.field = field
|
|
|
|
- self.filter = db_filter
|
|
|
|
- self.region_types = region_types
|
|
|
|
- self.original = original
|
|
|
|
-
|
|
|
|
- def __repr__(self):
|
|
|
|
- return "'{0}:{1}' {2} {3} [applied to '{4}' region type(s)]".format(
|
|
|
|
- self.namespace, self.field, self.filter[1], self.limit,
|
|
|
|
- mpp.api.Region.T().to_str(self.region_types))
|
|
|
|
-
|
|
|
|
- self.limits = []
|
|
|
|
- pattern = re.compile(r'''([^:]+)[:]([^:]+)[:]([-+]?[0-9]+(?:[.][0-9]+)?)(?:[:](.+))?''')
|
|
|
|
- if options.__dict__['max_limit'] != None:
|
|
|
|
- for each in options.__dict__['max_limit']:
|
|
|
|
- match = re.match(pattern, each)
|
|
|
|
- if match == None:
|
|
|
|
- self.parser.error("option --max-limit: Invalid format: " + each)
|
|
|
|
- region_types = 0x00
|
|
|
|
- if match.group(4) != None:
|
|
|
|
- for region_type in match.group(4).split(','):
|
|
|
|
- region_type = region_type.strip()
|
|
|
|
- group_id = mpp.api.Region.T().from_str(region_type)
|
|
|
|
- if group_id == None:
|
|
|
|
- self.parser.error(
|
|
|
|
- "option --max-limit: uknown region type (allowed: global, class, struct, namespace, function, interface, any): " + region_type)
|
|
|
|
- region_types |= group_id
|
|
|
|
- else:
|
|
|
|
- region_types = mpp.api.Region.T().ANY
|
|
|
|
- limit = Limit("max", float(match.group(3)), match.group(1), match.group(2),
|
|
|
|
- (match.group(2), '>', float(match.group(3))), region_types, each)
|
|
|
|
- self.limits.append(limit)
|
|
|
|
- if options.__dict__['min_limit'] != None:
|
|
|
|
- for each in options.__dict__['min_limit']:
|
|
|
|
- match = re.match(pattern, each)
|
|
|
|
- if match == None:
|
|
|
|
- self.parser.error("option --min-limit: Invalid format: " + each)
|
|
|
|
- region_types = 0x00
|
|
|
|
- if match.group(4) != None:
|
|
|
|
- for region_type in match.group(4).split(','):
|
|
|
|
- region_type = region_type.strip()
|
|
|
|
- group_id = mpp.api.Region.T().from_str(region_type)
|
|
|
|
- if group_id == None:
|
|
|
|
- self.parser.error(
|
|
|
|
- "option --max-limit: uknown region type (allowed: global, class, struct, namespace, function, interface, any): " + region_type)
|
|
|
|
- region_types |= group_id
|
|
|
|
- else:
|
|
|
|
- region_types = mpp.api.Region.T().ANY
|
|
|
|
- limit = Limit("min", float(match.group(3)), match.group(1), match.group(2),
|
|
|
|
- (match.group(2), '<', float(match.group(3))), region_types, each)
|
|
|
|
- self.limits.append(limit)
|
|
|
|
|
|
|
|
def initialize(self):
|
|
def initialize(self):
|
|
super(Plugin, self).initialize()
|
|
super(Plugin, self).initialize()
|
|
- db_loader = self.get_plugin('mpp.dbf').get_loader()
|
|
|
|
- self._verify_namespaces(db_loader.iterate_namespace_names())
|
|
|
|
- for each in db_loader.iterate_namespace_names():
|
|
|
|
- self._verify_fields(each, db_loader.get_namespace(each).iterate_field_names())
|
|
|
|
-
|
|
|
|
- def _verify_namespaces(self, valid_namespaces):
|
|
|
|
- valid = []
|
|
|
|
- for each in valid_namespaces:
|
|
|
|
- valid.append(each)
|
|
|
|
- for each in self.limits:
|
|
|
|
- if each.namespace not in valid:
|
|
|
|
- self.parser.error("option --{0}-limit: metric '{1}:{2}' is not available in the database file.".
|
|
|
|
- format(each.type, each.namespace, each.field))
|
|
|
|
-
|
|
|
|
- def _verify_fields(self, namespace, valid_fields):
|
|
|
|
- valid = []
|
|
|
|
- for each in valid_fields:
|
|
|
|
- valid.append(each)
|
|
|
|
- for each in self.limits:
|
|
|
|
- if each.namespace == namespace:
|
|
|
|
- if each.field not in valid:
|
|
|
|
- self.parser.error("option --{0}-limit: metric '{1}:{2}' is not available in the database file.".
|
|
|
|
- format(each.type, each.namespace, each.field))
|
|
|
|
-
|
|
|
|
- def iterate_limits(self):
|
|
|
|
- for each in self.limits:
|
|
|
|
- yield each
|
|
|
|
-
|
|
|
|
- def is_mode_matched(self, limit, value, diff, is_modified):
|
|
|
|
- if is_modified == None:
|
|
|
|
- # means new region, True in all modes
|
|
|
|
- return True
|
|
|
|
- if self.mode == self.MODE_ALL:
|
|
|
|
- return True
|
|
|
|
- if self.mode == self.MODE_TOUCHED and is_modified == True:
|
|
|
|
- return True
|
|
|
|
- if self.mode == self.MODE_TREND and is_modified == True:
|
|
|
|
- if limit < value and diff > 0:
|
|
|
|
- return True
|
|
|
|
- if limit > value and diff < 0:
|
|
|
|
- return True
|
|
|
|
- return False
|
|
|
|
|
|
|
|
def run(self, args):
|
|
def run(self, args):
|
|
return main(self, args)
|
|
return main(self, args)
|
|
|
|
|
|
- def run(self, args):
|
|
|
|
- return main(self, args)
|
|
|
|
-
|
|
|
|
def loadSubdirs(loader, path, subdirs, subfiles):
|
|
def loadSubdirs(loader, path, subdirs, subfiles):
|
|
|
|
|
|
aggregated_data = loader.load_aggregated_data(path)
|
|
aggregated_data = loader.load_aggregated_data(path)
|
|
@@ -198,7 +56,7 @@ def main(plugin, args):
|
|
|
|
|
|
exit_code = 0
|
|
exit_code = 0
|
|
|
|
|
|
- data = {"fileMetrixList" : [],
|
|
|
|
|
|
+ data = {"fileMetrixList" : {},
|
|
"regionMetrixList" : [],
|
|
"regionMetrixList" : [],
|
|
"files" : []}
|
|
"files" : []}
|
|
|
|
|
|
@@ -211,20 +69,24 @@ def main(plugin, args):
|
|
else:
|
|
else:
|
|
paths = args
|
|
paths = args
|
|
|
|
|
|
- # Try to optimise iterative change scans
|
|
|
|
- modified_file_ids = None
|
|
|
|
- if plugin.mode != plugin.MODE_ALL:
|
|
|
|
- modified_file_ids = get_list_of_modified_files(loader, loader_prev)
|
|
|
|
-
|
|
|
|
for path in paths:
|
|
for path in paths:
|
|
path = mpp.utils.preprocess_path(path)
|
|
path = mpp.utils.preprocess_path(path)
|
|
|
|
|
|
aggregated_data = loader.load_aggregated_data(path)
|
|
aggregated_data = loader.load_aggregated_data(path)
|
|
file_data = loader.load_file_data(path)
|
|
file_data = loader.load_file_data(path)
|
|
|
|
|
|
|
|
+ for namespace in aggregated_data.iterate_namespaces():
|
|
|
|
+ for field in aggregated_data.iterate_fields(namespace):
|
|
|
|
+ print(field)
|
|
|
|
+
|
|
for key in aggregated_data.data:
|
|
for key in aggregated_data.data:
|
|
if not key in data["fileMetrixList"]:
|
|
if not key in data["fileMetrixList"]:
|
|
- data["fileMetrixList"].append(key)
|
|
|
|
|
|
+ metric = { "name" : key,
|
|
|
|
+ "submetrics" : []}
|
|
|
|
+ data["fileMetrixList"][key] = metric
|
|
|
|
+ for subkey in aggregated_data.data[key]:
|
|
|
|
+ if not subkey in data["fileMetrixList"][key]:
|
|
|
|
+ data["fileMetrixList"][key]["submetrics"].append(subkey)
|
|
|
|
|
|
file = {"path" : path,
|
|
file = {"path" : path,
|
|
"file_id" : file_data.file_id,
|
|
"file_id" : file_data.file_id,
|
|
@@ -244,106 +106,42 @@ def main(plugin, args):
|
|
for key in region["data"]:
|
|
for key in region["data"]:
|
|
if not key in data["regionMetrixList"]:
|
|
if not key in data["regionMetrixList"]:
|
|
data["regionMetrixList"].append(key)
|
|
data["regionMetrixList"].append(key)
|
|
|
|
+ for subkey in key:
|
|
|
|
+ if not subkey in data["regionMetrixList"][key]:
|
|
|
|
+ data["regionMetrixList"][key].append(subkey)
|
|
|
|
|
|
- for limit in plugin.iterate_limits():
|
|
|
|
- warns_count = 0
|
|
|
|
- logging.info("Applying limit: " + str(limit))
|
|
|
|
- filters = [limit.filter]
|
|
|
|
- if modified_file_ids != None:
|
|
|
|
- filters.append(('file_id', 'IN', modified_file_ids))
|
|
|
|
- sort_by = None
|
|
|
|
- limit_by = None
|
|
|
|
-
|
|
|
|
- selected_data = loader.load_selected_data(limit.namespace,
|
|
|
|
- fields=[limit.field],
|
|
|
|
- path=path,
|
|
|
|
- filters=filters,
|
|
|
|
- sort_by=sort_by,
|
|
|
|
- limit_by=limit_by)
|
|
|
|
- if selected_data == None:
|
|
|
|
- mpp.utils.report_bad_path(path)
|
|
|
|
- exit_code += 1
|
|
|
|
- continue
|
|
|
|
-
|
|
|
|
- is_modified = None
|
|
|
|
- diff = None
|
|
|
|
- file_data = loader.load_file_data(selected_data.get_path())
|
|
|
|
- file_data_prev = loader_prev.load_file_data(selected_data.get_path())
|
|
|
|
- if file_data_prev != None:
|
|
|
|
- if file_data.get_checksum() == file_data_prev.get_checksum():
|
|
|
|
- diff = 0
|
|
|
|
- is_modified = False
|
|
|
|
- else:
|
|
|
|
- matcher = mpp.utils.FileRegionsMatcher(file_data, file_data_prev)
|
|
|
|
- prev_id = matcher.get_prev_id(selected_data.get_region().get_id())
|
|
|
|
- if matcher.is_matched(selected_data.get_region().get_id()):
|
|
|
|
- if matcher.is_modified(selected_data.get_region().get_id()):
|
|
|
|
- is_modified = True
|
|
|
|
- else:
|
|
|
|
- is_modified = False
|
|
|
|
- diff = mpp.api.DiffData(selected_data,
|
|
|
|
- file_data_prev.get_region(prev_id)).get_data(limit.namespace,
|
|
|
|
- limit.field)
|
|
|
|
-
|
|
|
|
- if (plugin.is_mode_matched(limit.limit,
|
|
|
|
- selected_data.get_data(limit.namespace, limit.field),
|
|
|
|
- diff,
|
|
|
|
- is_modified) == False):
|
|
|
|
- continue
|
|
|
|
-
|
|
|
|
- is_sup = is_metric_suppressed(limit.namespace, limit.field, loader, selected_data)
|
|
|
|
-
|
|
|
|
- # add a warning flag to the metric
|
|
|
|
- for file in data["files"]:
|
|
|
|
- if file["path"] == selected_data.get_path():
|
|
|
|
- for region in file["regions"]:
|
|
|
|
- if region["region_id"] == selected_data.region_id:
|
|
|
|
- for metric in region["data"]:
|
|
|
|
- if metric == limit.namespace:
|
|
|
|
- region["data"][metric]["warning"] = True
|
|
|
|
- region["data"][metric]["suppressed"] = is_sup
|
|
|
|
-
|
|
|
|
- if is_sup == True and plugin.no_suppress == False:
|
|
|
|
- continue
|
|
|
|
-
|
|
|
|
- warns_count += 1
|
|
|
|
- exit_code += 1
|
|
|
|
-
|
|
|
|
- writer = pytablewriter.SpaceAlignedTableWriter()
|
|
|
|
|
|
+ writer = pytablewriter.ExcelXlsxTableWriter()
|
|
|
|
+ writer.open("index.xlsx")
|
|
|
|
+ writer.make_worksheet("files")
|
|
writer.headers = ["file"] + data["fileMetrixList"]
|
|
writer.headers = ["file"] + data["fileMetrixList"]
|
|
|
|
|
|
matrix = [];
|
|
matrix = [];
|
|
|
|
|
|
for file in data["files"]:
|
|
for file in data["files"]:
|
|
line = []
|
|
line = []
|
|
- line.append(file["path"])
|
|
|
|
|
|
+ line.append("=HYPERLINK(\"#{0}!A1\",\"{0}\")".format(os.path.basename(file["path"])))
|
|
for metric in data["fileMetrixList"]:
|
|
for metric in data["fileMetrixList"]:
|
|
if metric in file["data"]:
|
|
if metric in file["data"]:
|
|
for value in file["data"][metric].values():
|
|
for value in file["data"][metric].values():
|
|
- values = []
|
|
|
|
- values.append(value["min"])
|
|
|
|
- values.append(value["max"])
|
|
|
|
- values.append(value["avg"])
|
|
|
|
- values.append(value["total"])
|
|
|
|
- values.append(value["count"])
|
|
|
|
- line.append(values)
|
|
|
|
|
|
+ line.append(value["total"])
|
|
break
|
|
break
|
|
else:
|
|
else:
|
|
line.append("---")
|
|
line.append("---")
|
|
matrix.append(line)
|
|
matrix.append(line)
|
|
|
|
|
|
- writer.table_name = file["path"]
|
|
|
|
- writer.value_matrix = matrix
|
|
|
|
- writer.write_table()
|
|
|
|
-
|
|
|
|
- writer = pytablewriter.SpaceAlignedTableWriter()
|
|
|
|
- writer.headers = ["line", "name"] + data["regionMetrixList"]
|
|
|
|
|
|
+ #writer.table_name = file["path"]
|
|
|
|
+ #writer.value_matrix = matrix
|
|
|
|
+ #writer.write_table()
|
|
|
|
|
|
- matrix = [];
|
|
|
|
|
|
+ #writer = pytablewriter.ExcelXlsxTableWriter()
|
|
|
|
+ writer.headers = ["file", "line", "name"] + data["regionMetrixList"]
|
|
|
|
|
|
for file in data["files"]:
|
|
for file in data["files"]:
|
|
|
|
+ writer.make_worksheet(os.path.basename(file["path"]))
|
|
|
|
+ matrix = [];
|
|
for region in file["regions"]:
|
|
for region in file["regions"]:
|
|
line = []
|
|
line = []
|
|
|
|
+ line.append(file["path"])
|
|
line.append(str(region["line_begin"]))
|
|
line.append(str(region["line_begin"]))
|
|
line.append(region["name"])
|
|
line.append(region["name"])
|
|
for metric in data["regionMetrixList"]:
|
|
for metric in data["regionMetrixList"]:
|
|
@@ -359,41 +157,6 @@ def main(plugin, args):
|
|
writer.value_matrix = matrix
|
|
writer.value_matrix = matrix
|
|
writer.write_table()
|
|
writer.write_table()
|
|
|
|
|
|
- return exit_code
|
|
|
|
|
|
+ writer.close()
|
|
|
|
|
|
-def get_list_of_modified_files(loader, loader_prev):
|
|
|
|
- logging.info("Identifying changed files...")
|
|
|
|
-
|
|
|
|
- old_files_map = {}
|
|
|
|
- for each in loader_prev.iterate_file_data():
|
|
|
|
- old_files_map[each.get_path()] = each.get_checksum()
|
|
|
|
- if len(old_files_map) == 0:
|
|
|
|
- return None
|
|
|
|
-
|
|
|
|
- modified_file_ids = []
|
|
|
|
- for each in loader.iterate_file_data():
|
|
|
|
- if len(modified_file_ids) > 1000: # If more than 1000 files changed, skip optimisation
|
|
|
|
- return None
|
|
|
|
- if (each.get_path() not in list(old_files_map.keys())) or old_files_map[
|
|
|
|
- each.get_path()] != each.get_checksum():
|
|
|
|
- modified_file_ids.append(str(each.get_id()))
|
|
|
|
-
|
|
|
|
- old_files_map = None
|
|
|
|
-
|
|
|
|
- if len(modified_file_ids) != 0:
|
|
|
|
- modified_file_ids = " , ".join(modified_file_ids)
|
|
|
|
- modified_file_ids = "(" + modified_file_ids + ")"
|
|
|
|
- return modified_file_ids
|
|
|
|
-
|
|
|
|
- return None
|
|
|
|
-
|
|
|
|
-def is_metric_suppressed(metric_namespace, metric_field, loader, select_data):
|
|
|
|
- data = loader.load_file_data(select_data.get_path())
|
|
|
|
- if select_data.get_region() != None:
|
|
|
|
- data = data.get_region(select_data.get_region().get_id())
|
|
|
|
- sup_data = data.get_data('std.suppress', 'list')
|
|
|
|
- else:
|
|
|
|
- sup_data = data.get_data('std.suppress.file', 'list')
|
|
|
|
- if sup_data != None and sup_data.find('[' + metric_namespace + ':' + metric_field + ']') != -1:
|
|
|
|
- return True
|
|
|
|
- return False
|
|
|
|
|
|
+ return exit_code
|