|
@@ -12,6 +12,8 @@ import mpp.api
|
|
import mpp.utils
|
|
import mpp.utils
|
|
import mpp.cout
|
|
import mpp.cout
|
|
|
|
|
|
|
|
+DIGIT_COUNT = 8
|
|
|
|
+
|
|
class Plugin(mpp.api.Plugin, mpp.api.IConfigurable, mpp.api.IRunable):
|
|
class Plugin(mpp.api.Plugin, mpp.api.IConfigurable, mpp.api.IRunable):
|
|
|
|
|
|
MODE_NEW = 0x01
|
|
MODE_NEW = 0x01
|
|
@@ -61,7 +63,7 @@ class Plugin(mpp.api.Plugin, mpp.api.IConfigurable, mpp.api.IRunable):
|
|
paths = [""]
|
|
paths = [""]
|
|
else:
|
|
else:
|
|
paths = args
|
|
paths = args
|
|
-
|
|
|
|
|
|
+
|
|
(result, exit_code) = export_to_str(self.out_format,
|
|
(result, exit_code) = export_to_str(self.out_format,
|
|
paths,
|
|
paths,
|
|
loader,
|
|
loader,
|
|
@@ -69,7 +71,7 @@ class Plugin(mpp.api.Plugin, mpp.api.IConfigurable, mpp.api.IRunable):
|
|
self.nest_regions,
|
|
self.nest_regions,
|
|
self.dist_columns,
|
|
self.dist_columns,
|
|
self.mode)
|
|
self.mode)
|
|
- print result
|
|
|
|
|
|
+ print(result)
|
|
return exit_code
|
|
return exit_code
|
|
|
|
|
|
def export_to_str(out_format, paths, loader, loader_prev, nest_regions, dist_columns, mode):
|
|
def export_to_str(out_format, paths, loader, loader_prev, nest_regions, dist_columns, mode):
|
|
@@ -90,8 +92,8 @@ def export_to_str(out_format, paths, loader, loader_prev, nest_regions, dist_col
|
|
subfiles = []
|
|
subfiles = []
|
|
if aggregated_data != None:
|
|
if aggregated_data != None:
|
|
aggregated_data_tree = aggregated_data.get_data_tree()
|
|
aggregated_data_tree = aggregated_data.get_data_tree()
|
|
- subdirs = aggregated_data.get_subdirs()
|
|
|
|
- subfiles = aggregated_data.get_subfiles()
|
|
|
|
|
|
+ subdirs = sorted(aggregated_data.get_subdirs())
|
|
|
|
+ subfiles = sorted(aggregated_data.get_subfiles())
|
|
else:
|
|
else:
|
|
mpp.utils.report_bad_path(path)
|
|
mpp.utils.report_bad_path(path)
|
|
exit_code += 1
|
|
exit_code += 1
|
|
@@ -121,7 +123,7 @@ def export_to_str(out_format, paths, loader, loader_prev, nest_regions, dist_col
|
|
if out_format == 'txt':
|
|
if out_format == 'txt':
|
|
cout_txt(data, loader)
|
|
cout_txt(data, loader)
|
|
elif out_format == 'xml':
|
|
elif out_format == 'xml':
|
|
- result += mpp.utils.serialize_to_xml(data, root_name = "data") + "\n"
|
|
|
|
|
|
+ result += mpp.utils.serialize_to_xml(data, root_name = "data", digitCount = DIGIT_COUNT) + "\n"
|
|
elif out_format == 'python':
|
|
elif out_format == 'python':
|
|
postfix = ""
|
|
postfix = ""
|
|
if ind < len(paths) - 1:
|
|
if ind < len(paths) - 1:
|
|
@@ -192,8 +194,8 @@ def load_aggregated_data_with_mode(loader, loader_prev, path, mode):
|
|
assert(self.in_processing_mode == True)
|
|
assert(self.in_processing_mode == True)
|
|
sup_data = orig_data.get_data('std.suppress', 'list')
|
|
sup_data = orig_data.get_data('std.suppress', 'list')
|
|
data = orig_data.get_data_tree()
|
|
data = orig_data.get_data_tree()
|
|
- for namespace in data.keys():
|
|
|
|
- for field in data[namespace].keys():
|
|
|
|
|
|
+ for namespace in list(data.keys()):
|
|
|
|
+ for field in list(data[namespace].keys()):
|
|
aggr_data = self.get_data(namespace, field)
|
|
aggr_data = self.get_data(namespace, field)
|
|
metric_value = data[namespace][field]
|
|
metric_value = data[namespace][field]
|
|
if isinstance(metric_value, str):
|
|
if isinstance(metric_value, str):
|
|
@@ -206,7 +208,7 @@ def load_aggregated_data_with_mode(loader, loader_prev, path, mode):
|
|
aggr_data['count'] += 1
|
|
aggr_data['count'] += 1
|
|
aggr_data['total'] += metric_value
|
|
aggr_data['total'] += metric_value
|
|
# average is calculated later on get_data_tree
|
|
# average is calculated later on get_data_tree
|
|
- if metric_value not in aggr_data['distribution-bars'].keys():
|
|
|
|
|
|
+ if metric_value not in list(aggr_data['distribution-bars'].keys()):
|
|
aggr_data['distribution-bars'][metric_value] = 0
|
|
aggr_data['distribution-bars'][metric_value] = 0
|
|
aggr_data['distribution-bars'][metric_value] += 1
|
|
aggr_data['distribution-bars'][metric_value] += 1
|
|
if sup_data != None:
|
|
if sup_data != None:
|
|
@@ -340,16 +342,16 @@ def append_diff(main_tree, prev_tree):
|
|
assert(main_tree != None)
|
|
assert(main_tree != None)
|
|
assert(prev_tree != None)
|
|
assert(prev_tree != None)
|
|
|
|
|
|
- for name in main_tree.keys():
|
|
|
|
- if name not in prev_tree.keys():
|
|
|
|
|
|
+ for name in list(main_tree.keys()):
|
|
|
|
+ if name not in list(prev_tree.keys()):
|
|
continue
|
|
continue
|
|
- for field in main_tree[name].keys():
|
|
|
|
- if field not in prev_tree[name].keys():
|
|
|
|
|
|
+ for field in list(main_tree[name].keys()):
|
|
|
|
+ if field not in list(prev_tree[name].keys()):
|
|
continue
|
|
continue
|
|
if isinstance(main_tree[name][field], dict) and isinstance(prev_tree[name][field], dict):
|
|
if isinstance(main_tree[name][field], dict) and isinstance(prev_tree[name][field], dict):
|
|
diff = {}
|
|
diff = {}
|
|
- for key in main_tree[name][field].keys():
|
|
|
|
- if key not in prev_tree[name][field].keys():
|
|
|
|
|
|
+ for key in list(main_tree[name][field].keys()):
|
|
|
|
+ if key not in list(prev_tree[name][field].keys()):
|
|
continue
|
|
continue
|
|
main_val = main_tree[name][field][key]
|
|
main_val = main_tree[name][field][key]
|
|
prev_val = prev_tree[name][field][key]
|
|
prev_val = prev_tree[name][field][key]
|
|
@@ -373,7 +375,7 @@ def append_diff_list(main_list, prev_list):
|
|
for bar in main_list:
|
|
for bar in main_list:
|
|
merged_list[bar['metric']] = {'count': bar['count'], '__diff__':bar['count'], 'ratio': bar['ratio']}
|
|
merged_list[bar['metric']] = {'count': bar['count'], '__diff__':bar['count'], 'ratio': bar['ratio']}
|
|
for bar in prev_list:
|
|
for bar in prev_list:
|
|
- if bar['metric'] in merged_list.keys():
|
|
|
|
|
|
+ if bar['metric'] in list(merged_list.keys()):
|
|
merged_list[bar['metric']]['__diff__'] = \
|
|
merged_list[bar['metric']]['__diff__'] = \
|
|
merged_list[bar['metric']]['count'] - bar['count']
|
|
merged_list[bar['metric']]['count'] - bar['count']
|
|
else:
|
|
else:
|
|
@@ -389,8 +391,8 @@ def append_diff_list(main_list, prev_list):
|
|
def append_suppressions(path, data, loader, mode):
|
|
def append_suppressions(path, data, loader, mode):
|
|
if mode == Plugin.MODE_ALL:
|
|
if mode == Plugin.MODE_ALL:
|
|
# in other modes, suppressions are appended during data loading
|
|
# in other modes, suppressions are appended during data loading
|
|
- for namespace in data.keys():
|
|
|
|
- for field in data[namespace].keys():
|
|
|
|
|
|
+ for namespace in list(data.keys()):
|
|
|
|
+ for field in list(data[namespace].keys()):
|
|
selected_data = loader.load_selected_data('std.suppress',
|
|
selected_data = loader.load_selected_data('std.suppress',
|
|
fields = ['list'],
|
|
fields = ['list'],
|
|
path=path,
|
|
path=path,
|
|
@@ -409,8 +411,8 @@ def compress_dist(data, columns):
|
|
if columns == 0:
|
|
if columns == 0:
|
|
return data
|
|
return data
|
|
|
|
|
|
- for namespace in data.keys():
|
|
|
|
- for field in data[namespace].keys():
|
|
|
|
|
|
+ for namespace in list(data.keys()):
|
|
|
|
+ for field in list(data[namespace].keys()):
|
|
metric_data = data[namespace][field]
|
|
metric_data = data[namespace][field]
|
|
distr = metric_data['distribution-bars']
|
|
distr = metric_data['distribution-bars']
|
|
columns = float(columns) # to trigger floating calculations
|
|
columns = float(columns) # to trigger floating calculations
|
|
@@ -422,8 +424,8 @@ def compress_dist(data, columns):
|
|
remaining_count = metric_data['count']
|
|
remaining_count = metric_data['count']
|
|
next_consume = None
|
|
next_consume = None
|
|
next_bar = None
|
|
next_bar = None
|
|
- max_count = -sys.maxint - 1
|
|
|
|
- min_count = sys.maxint
|
|
|
|
|
|
+ max_count = -sys.maxsize - 1
|
|
|
|
+ min_count = sys.maxsize
|
|
sum_ratio = 0
|
|
sum_ratio = 0
|
|
for (ind, bar) in enumerate(distr):
|
|
for (ind, bar) in enumerate(distr):
|
|
if next_bar == None:
|
|
if next_bar == None:
|
|
@@ -432,15 +434,15 @@ def compress_dist(data, columns):
|
|
'ratio': bar['ratio'],
|
|
'ratio': bar['ratio'],
|
|
'metric_s': bar['metric'],
|
|
'metric_s': bar['metric'],
|
|
'metric_f': bar['metric']}
|
|
'metric_f': bar['metric']}
|
|
- if '__diff__' in bar.keys():
|
|
|
|
|
|
+ if '__diff__' in list(bar.keys()):
|
|
next_bar['__diff__'] = bar['__diff__']
|
|
next_bar['__diff__'] = bar['__diff__']
|
|
- next_consume = int(round(remaining_count/ (columns - len(new_dist))))
|
|
|
|
|
|
+ next_consume = int(remaining_count/ (columns - len(new_dist)))
|
|
else:
|
|
else:
|
|
# merge to existing bar
|
|
# merge to existing bar
|
|
next_bar['count'] += bar['count']
|
|
next_bar['count'] += bar['count']
|
|
next_bar['ratio'] += bar['ratio']
|
|
next_bar['ratio'] += bar['ratio']
|
|
next_bar['metric_f'] = bar['metric']
|
|
next_bar['metric_f'] = bar['metric']
|
|
- if '__diff__' in bar.keys():
|
|
|
|
|
|
+ if '__diff__' in list(bar.keys()):
|
|
next_bar['__diff__'] += bar['__diff__']
|
|
next_bar['__diff__'] += bar['__diff__']
|
|
|
|
|
|
next_consume -= bar['count']
|
|
next_consume -= bar['count']
|
|
@@ -490,7 +492,7 @@ def compress_dist(data, columns):
|
|
'ratio': bar['ratio'],
|
|
'ratio': bar['ratio'],
|
|
'metric_s': next_end_limit,
|
|
'metric_s': next_end_limit,
|
|
'metric_f': bar['metric']}
|
|
'metric_f': bar['metric']}
|
|
- if '__diff__' in bar.keys():
|
|
|
|
|
|
+ if '__diff__' in list(bar.keys()):
|
|
next_bar['__diff__'] = bar['__diff__']
|
|
next_bar['__diff__'] = bar['__diff__']
|
|
next_end_limit += step
|
|
next_end_limit += step
|
|
else:
|
|
else:
|
|
@@ -498,7 +500,7 @@ def compress_dist(data, columns):
|
|
next_bar['count'] += bar['count']
|
|
next_bar['count'] += bar['count']
|
|
next_bar['ratio'] += bar['ratio']
|
|
next_bar['ratio'] += bar['ratio']
|
|
next_bar['metric_f'] = bar['metric']
|
|
next_bar['metric_f'] = bar['metric']
|
|
- if '__diff__' in bar.keys():
|
|
|
|
|
|
+ if '__diff__' in list(bar.keys()):
|
|
next_bar['__diff__'] += bar['__diff__']
|
|
next_bar['__diff__'] += bar['__diff__']
|
|
|
|
|
|
if (next_bar['metric_f'] >= next_end_limit # consumed enough
|
|
if (next_bar['metric_f'] >= next_end_limit # consumed enough
|
|
@@ -530,15 +532,15 @@ def cout_txt_regions(path, regions, indent = 0):
|
|
('Line numbers', str(region['info']['line_begin']) + "-" + str(region['info']['line_end'])),
|
|
('Line numbers', str(region['info']['line_begin']) + "-" + str(region['info']['line_end'])),
|
|
('Modified', str(region['info']['modified']))
|
|
('Modified', str(region['info']['modified']))
|
|
]
|
|
]
|
|
- for namespace in region['data'].keys():
|
|
|
|
|
|
+ for namespace in sorted(list(region['data'].keys())):
|
|
diff_data = {}
|
|
diff_data = {}
|
|
- if '__diff__' in region['data'][namespace].keys():
|
|
|
|
|
|
+ if '__diff__' in list(region['data'][namespace].keys()):
|
|
diff_data = region['data'][namespace]['__diff__']
|
|
diff_data = region['data'][namespace]['__diff__']
|
|
- for field in region['data'][namespace].keys():
|
|
|
|
|
|
+ for field in sorted(list(region['data'][namespace].keys())):
|
|
diff_str = ""
|
|
diff_str = ""
|
|
if field == '__diff__':
|
|
if field == '__diff__':
|
|
continue
|
|
continue
|
|
- if field in diff_data.keys():
|
|
|
|
|
|
+ if field in list(diff_data.keys()):
|
|
diff_str = " [" + ("+" if diff_data[field] >= 0 else "") + str(diff_data[field]) + "]"
|
|
diff_str = " [" + ("+" if diff_data[field] >= 0 else "") + str(diff_data[field]) + "]"
|
|
details.append((namespace + ":" + field, str(region['data'][namespace][field]) + diff_str))
|
|
details.append((namespace + ":" + field, str(region['data'][namespace][field]) + diff_str))
|
|
mpp.cout.notify(path,
|
|
mpp.cout.notify(path,
|
|
@@ -547,25 +549,25 @@ def cout_txt_regions(path, regions, indent = 0):
|
|
"Metrics per '" + region['info']['name']+ "' region",
|
|
"Metrics per '" + region['info']['name']+ "' region",
|
|
details,
|
|
details,
|
|
indent=indent)
|
|
indent=indent)
|
|
- if 'subregions' in region.keys():
|
|
|
|
|
|
+ if 'subregions' in list(region.keys()):
|
|
cout_txt_regions(path, region['subregions'], indent=indent+1)
|
|
cout_txt_regions(path, region['subregions'], indent=indent+1)
|
|
|
|
|
|
def cout_txt(data, loader):
|
|
def cout_txt(data, loader):
|
|
|
|
|
|
details = []
|
|
details = []
|
|
- for key in data['file-data'].keys():
|
|
|
|
|
|
+ for key in list(data['file-data'].keys()):
|
|
if key == 'regions':
|
|
if key == 'regions':
|
|
cout_txt_regions(data['info']['path'], data['file-data'][key])
|
|
cout_txt_regions(data['info']['path'], data['file-data'][key])
|
|
else:
|
|
else:
|
|
namespace = key
|
|
namespace = key
|
|
diff_data = {}
|
|
diff_data = {}
|
|
- if '__diff__' in data['file-data'][namespace].keys():
|
|
|
|
|
|
+ if '__diff__' in list(data['file-data'][namespace].keys()):
|
|
diff_data = data['file-data'][namespace]['__diff__']
|
|
diff_data = data['file-data'][namespace]['__diff__']
|
|
- for field in data['file-data'][namespace].keys():
|
|
|
|
|
|
+ for field in sorted(list(data['file-data'][namespace].keys())):
|
|
diff_str = ""
|
|
diff_str = ""
|
|
if field == '__diff__':
|
|
if field == '__diff__':
|
|
continue
|
|
continue
|
|
- if field in diff_data.keys():
|
|
|
|
|
|
+ if field in list(diff_data.keys()):
|
|
diff_str = " [" + ("+" if diff_data[field] >= 0 else "") + str(diff_data[field]) + "]"
|
|
diff_str = " [" + ("+" if diff_data[field] >= 0 else "") + str(diff_data[field]) + "]"
|
|
details.append((namespace + ":" + field, str(data['file-data'][namespace][field]) + diff_str))
|
|
details.append((namespace + ":" + field, str(data['file-data'][namespace][field]) + diff_str))
|
|
if len(details) > 0:
|
|
if len(details) > 0:
|
|
@@ -580,25 +582,32 @@ def cout_txt(data, loader):
|
|
'min': 'Minimum',
|
|
'min': 'Minimum',
|
|
'max': 'Maximum',
|
|
'max': 'Maximum',
|
|
}
|
|
}
|
|
- for namespace in data['aggregated-data'].keys():
|
|
|
|
- for field in data['aggregated-data'][namespace].keys():
|
|
|
|
|
|
+ for namespace in sorted(list(data['aggregated-data'].keys())):
|
|
|
|
+ for field in sorted(list(data['aggregated-data'][namespace].keys())):
|
|
details = []
|
|
details = []
|
|
diff_data = {}
|
|
diff_data = {}
|
|
- if '__diff__' in data['aggregated-data'][namespace][field].keys():
|
|
|
|
|
|
+ if '__diff__' in list(data['aggregated-data'][namespace][field].keys()):
|
|
diff_data = data['aggregated-data'][namespace][field]['__diff__']
|
|
diff_data = data['aggregated-data'][namespace][field]['__diff__']
|
|
for attr in ['avg', 'min', 'max', 'total']:
|
|
for attr in ['avg', 'min', 'max', 'total']:
|
|
diff_str = ""
|
|
diff_str = ""
|
|
- if attr in diff_data.keys():
|
|
|
|
- diff_str = " [" + ("+" if diff_data[attr] >= 0 else "") + str(diff_data[attr]) + "]"
|
|
|
|
|
|
+ if attr in list(diff_data.keys()):
|
|
|
|
+ if isinstance(diff_data[attr], float):
|
|
|
|
+ diff_str = " [" + ("+" if diff_data[attr] >= 0 else "") + str(round(diff_data[attr], DIGIT_COUNT)) + "]"
|
|
|
|
+ else:
|
|
|
|
+ diff_str = " [" + ("+" if diff_data[attr] >= 0 else "") + str(diff_data[attr]) + "]"
|
|
if attr == 'avg' and data['aggregated-data'][namespace][field]['nonzero'] == True:
|
|
if attr == 'avg' and data['aggregated-data'][namespace][field]['nonzero'] == True:
|
|
diff_str += " (excluding zero metric values)"
|
|
diff_str += " (excluding zero metric values)"
|
|
- details.append((attr_map[attr], str(data['aggregated-data'][namespace][field][attr]) + diff_str))
|
|
|
|
|
|
+ if isinstance(data['aggregated-data'][namespace][field][attr], float):
|
|
|
|
+ # round the data to reach same results on platforms with different precision
|
|
|
|
+ details.append((attr_map[attr], str(round(data['aggregated-data'][namespace][field][attr], DIGIT_COUNT)) + diff_str))
|
|
|
|
+ else:
|
|
|
|
+ details.append((attr_map[attr], str(data['aggregated-data'][namespace][field][attr]) + diff_str))
|
|
|
|
|
|
measured = data['aggregated-data'][namespace][field]['count']
|
|
measured = data['aggregated-data'][namespace][field]['count']
|
|
- if 'count' in diff_data.keys():
|
|
|
|
|
|
+ if 'count' in list(diff_data.keys()):
|
|
diff_str = ' [{0:{1}}]'.format(diff_data['count'], '+' if diff_data['count'] >= 0 else '')
|
|
diff_str = ' [{0:{1}}]'.format(diff_data['count'], '+' if diff_data['count'] >= 0 else '')
|
|
sup_diff_str = ""
|
|
sup_diff_str = ""
|
|
- if 'sup' in diff_data.keys():
|
|
|
|
|
|
+ if 'sup' in list(diff_data.keys()):
|
|
sup_diff_str = ' [{0:{1}}]'.format(diff_data['sup'], '+' if diff_data['sup'] >= 0 else '')
|
|
sup_diff_str = ' [{0:{1}}]'.format(diff_data['sup'], '+' if diff_data['sup'] >= 0 else '')
|
|
elem_name = 'regions'
|
|
elem_name = 'regions'
|
|
if loader.get_namespace(namespace).are_regions_supported() == False:
|
|
if loader.get_namespace(namespace).are_regions_supported() == False:
|
|
@@ -615,7 +624,7 @@ def cout_txt(data, loader):
|
|
for bar in data['aggregated-data'][namespace][field]['distribution-bars']:
|
|
for bar in data['aggregated-data'][namespace][field]['distribution-bars']:
|
|
sum_ratio += bar['ratio']
|
|
sum_ratio += bar['ratio']
|
|
diff_str = ""
|
|
diff_str = ""
|
|
- if '__diff__' in bar.keys():
|
|
|
|
|
|
+ if '__diff__' in list(bar.keys()):
|
|
if bar['__diff__'] >= 0:
|
|
if bar['__diff__'] >= 0:
|
|
diff_str = ' [+{0:<{1}}]'.format(bar['__diff__'], count_str_len)
|
|
diff_str = ' [+{0:<{1}}]'.format(bar['__diff__'], count_str_len)
|
|
else:
|
|
else:
|
|
@@ -630,16 +639,16 @@ def cout_txt(data, loader):
|
|
count_str = ((" " * (count_str_len - len(count_str))) + count_str + diff_str + "\t")
|
|
count_str = ((" " * (count_str_len - len(count_str))) + count_str + diff_str + "\t")
|
|
details.append((metric_str,
|
|
details.append((metric_str,
|
|
"{0:.3f}".format(bar['ratio']) + " : " + "{0:.3f}".format(sum_ratio) + " : " +
|
|
"{0:.3f}".format(bar['ratio']) + " : " + "{0:.3f}".format(sum_ratio) + " : " +
|
|
- count_str + ('|' * int(round(bar['ratio']*100)))))
|
|
|
|
|
|
+ count_str + ('|' * int(bar['ratio']*100))))
|
|
mpp.cout.notify(data['info']['path'],
|
|
mpp.cout.notify(data['info']['path'],
|
|
'', # no line number
|
|
'', # no line number
|
|
mpp.cout.SEVERITY_INFO,
|
|
mpp.cout.SEVERITY_INFO,
|
|
"Overall metrics for '" + namespace + ":" + field + "' metric",
|
|
"Overall metrics for '" + namespace + ":" + field + "' metric",
|
|
details)
|
|
details)
|
|
details = []
|
|
details = []
|
|
- for each in data['subdirs']:
|
|
|
|
|
|
+ for each in sorted(data['subdirs']):
|
|
details.append(('Directory', each))
|
|
details.append(('Directory', each))
|
|
- for each in data['subfiles']:
|
|
|
|
|
|
+ for each in sorted(data['subfiles']):
|
|
details.append(('File', each))
|
|
details.append(('File', each))
|
|
if len(details) > 0:
|
|
if len(details) > 0:
|
|
mpp.cout.notify(data['info']['path'],
|
|
mpp.cout.notify(data['info']['path'],
|
|
@@ -647,4 +656,4 @@ def cout_txt(data, loader):
|
|
mpp.cout.SEVERITY_INFO,
|
|
mpp.cout.SEVERITY_INFO,
|
|
"Directory content:",
|
|
"Directory content:",
|
|
details)
|
|
details)
|
|
-
|
|
|
|
|
|
+
|