|
|
@@ -17,6 +17,166 @@
|
|
|
# along with Metrix++. If not, see <http://www.gnu.org/licenses/>.
|
|
|
#
|
|
|
|
|
|
+import logging
|
|
|
+
|
|
|
+class Data(object):
|
|
|
+
|
|
|
+ def __init__(self):
|
|
|
+ self.data = {}
|
|
|
+
|
|
|
+ def get_data(self, namespace, field):
|
|
|
+ if namespace not in self.data.keys():
|
|
|
+ return None
|
|
|
+ if field not in self.data[namespace].keys():
|
|
|
+ return None
|
|
|
+ return self.data[namespace][field]
|
|
|
+
|
|
|
+ def set_data(self, namespace, field, value):
|
|
|
+ if namespace not in self.data:
|
|
|
+ self.data[namespace] = {}
|
|
|
+ self.data[namespace][field] = value
|
|
|
+
|
|
|
+ def iterate_namespaces(self):
|
|
|
+ for namespace in self.data.keys():
|
|
|
+ yield namespace
|
|
|
+
|
|
|
+ def iterate_fields(self, namespace):
|
|
|
+ for field in self.data[namespace].keys():
|
|
|
+ yield (field, self.data[namespace][field])
|
|
|
+
|
|
|
+ def get_data_tree(self, namespaces=None):
|
|
|
+ return self.data
|
|
|
+
|
|
|
+ def __repr__(self):
|
|
|
+ return object.__repr__(self) + " with data " + self.data.__repr__()
|
|
|
+
|
|
|
+
|
|
|
+class LoadableData(Data):
|
|
|
+
|
|
|
+ def __init__(self, loader, file_id, region_id):
|
|
|
+ Data.__init__(self)
|
|
|
+ self.loader = loader
|
|
|
+ self.file_id = file_id
|
|
|
+ self.region_id = region_id
|
|
|
+ self.loaded_namespaces = []
|
|
|
+ self.changed_namespaces = []
|
|
|
+
|
|
|
+ def load_namespace(self, namespace):
|
|
|
+ try:
|
|
|
+ row = self.loader.db.get_row(namespace, self.file_id, self.region_id)
|
|
|
+ except Exception:
|
|
|
+ logging.debug("No data in the database for namespace: " + namespace)
|
|
|
+ return
|
|
|
+ if row == None:
|
|
|
+ return
|
|
|
+ for column_name in row.keys():
|
|
|
+ packager = self.loader.get_namespace(namespace).get_field_packager(column_name)
|
|
|
+ if packager == None:
|
|
|
+ continue
|
|
|
+ if row[column_name] == None:
|
|
|
+ continue
|
|
|
+ Data.set_data(self, namespace, column_name, packager.unpack(row[column_name]))
|
|
|
+
|
|
|
+ def set_data(self, namespace, field, value):
|
|
|
+ if namespace not in self.changed_namespaces:
|
|
|
+ self.changed_namespaces.append(namespace)
|
|
|
+ return Data.set_data(self, namespace, field, value)
|
|
|
+
|
|
|
+ def get_data(self, namespace, field):
|
|
|
+ if namespace not in self.loaded_namespaces:
|
|
|
+ self.loaded_namespaces.append(namespace)
|
|
|
+ self.load_namespace(namespace)
|
|
|
+ return Data.get_data(self, namespace, field)
|
|
|
+
|
|
|
+ def is_namespace_updated(self, namespace):
|
|
|
+ return namespace in self.changed_namespaces
|
|
|
+
|
|
|
+ def is_namespace_loaded(self, namespace):
|
|
|
+ return namespace in self.loaded_namespaces
|
|
|
+
|
|
|
+ def get_data_tree(self, namespaces=None):
|
|
|
+ if namespaces == None:
|
|
|
+ namespaces = self.loader.iterate_namespace_names()
|
|
|
+ for each in namespaces:
|
|
|
+ self.load_namespace(each)
|
|
|
+ return Data.get_data_tree(self)
|
|
|
+
|
|
|
+class FileRegionData(LoadableData):
|
|
|
+
|
|
|
+ class T(object):
|
|
|
+ NONE = 0x00
|
|
|
+ GLOBAL = 0x01
|
|
|
+ CLASS = 0x02
|
|
|
+ STRUCT = 0x04
|
|
|
+ NAMESPACE = 0x08
|
|
|
+ FUNCTION = 0x10
|
|
|
+ INTERFACE = 0x20
|
|
|
+ ANY = 0xFF
|
|
|
+
|
|
|
+ def to_str(self, group):
|
|
|
+ if group == self.NONE:
|
|
|
+ return "none"
|
|
|
+ elif group == self.GLOBAL:
|
|
|
+ return "global"
|
|
|
+ elif group == self.CLASS:
|
|
|
+ return "class"
|
|
|
+ elif group == self.STRUCT:
|
|
|
+ return "struct"
|
|
|
+ elif group == self.NAMESPACE:
|
|
|
+ return "namespace"
|
|
|
+ elif group == self.FUNCTION:
|
|
|
+ return "function"
|
|
|
+ elif group == self.INTERFACE:
|
|
|
+ return "interface"
|
|
|
+ else:
|
|
|
+ assert(False)
|
|
|
+
|
|
|
+ def __init__(self, loader, file_id, region_id, region_name, offset_begin, offset_end, line_begin, line_end, cursor_line, group, checksum):
|
|
|
+ LoadableData.__init__(self, loader, file_id, region_id)
|
|
|
+ self.name = region_name
|
|
|
+ self.begin = offset_begin
|
|
|
+ self.end = offset_end
|
|
|
+ self.line_begin = line_begin
|
|
|
+ self.line_end = line_end
|
|
|
+ self.cursor = cursor_line
|
|
|
+ self.group = group
|
|
|
+ self.checksum = checksum
|
|
|
+
|
|
|
+ self.children = []
|
|
|
+
|
|
|
+ def get_id(self):
|
|
|
+ return self.region_id
|
|
|
+
|
|
|
+ def get_name(self):
|
|
|
+ return self.name
|
|
|
+
|
|
|
+ def get_offset_begin(self):
|
|
|
+ return self.begin
|
|
|
+
|
|
|
+ def get_offset_end(self):
|
|
|
+ return self.end
|
|
|
+
|
|
|
+ def get_line_begin(self):
|
|
|
+ return self.line_begin
|
|
|
+
|
|
|
+ def get_line_end(self):
|
|
|
+ return self.line_end
|
|
|
+
|
|
|
+ def get_cursor(self):
|
|
|
+ return self.cursor
|
|
|
+
|
|
|
+ def get_type(self):
|
|
|
+ return self.group
|
|
|
+
|
|
|
+ def get_checksum(self):
|
|
|
+ return self.checksum
|
|
|
+
|
|
|
+ def register_subregion_id(self, child_id):
|
|
|
+ self.children.append(child_id)
|
|
|
+
|
|
|
+ def iterate_subregion_ids(self):
|
|
|
+ return self.children
|
|
|
+
|
|
|
class Marker(object):
|
|
|
class T(object):
|
|
|
NONE = 0x00
|
|
|
@@ -55,6 +215,246 @@ class Marker(object):
|
|
|
def get_type(self):
|
|
|
return self.group
|
|
|
|
|
|
+class FileData(LoadableData):
|
|
|
+
|
|
|
+ def __init__(self, loader, path, file_id, checksum, content):
|
|
|
+ LoadableData.__init__(self, loader, file_id, None)
|
|
|
+ self.path = path
|
|
|
+ self.checksum = checksum
|
|
|
+ self.content = content
|
|
|
+ self.regions = None
|
|
|
+ self.markers = None
|
|
|
+ self.loader = loader
|
|
|
+ self.loading_tmp = []
|
|
|
+
|
|
|
+ def get_id(self):
|
|
|
+ return self.file_id
|
|
|
+
|
|
|
+ def get_path(self):
|
|
|
+ return self.path
|
|
|
+
|
|
|
+ def get_checksum(self):
|
|
|
+ return self.checksum
|
|
|
+
|
|
|
+ def get_content(self):
|
|
|
+ return self.content
|
|
|
+
|
|
|
+ def internal_append_region(self, region):
|
|
|
+ # here we apply some magic - we rely on special ordering of coming regions,
|
|
|
+ # which is supported by code parsers
|
|
|
+ prev_id = None
|
|
|
+ while True:
|
|
|
+ if len(self.loading_tmp) == 0:
|
|
|
+ break
|
|
|
+ prev_id = self.loading_tmp.pop()
|
|
|
+ if self.get_region(prev_id).get_offset_end() > region.get_offset_begin():
|
|
|
+ self.loading_tmp.append(prev_id) # return back
|
|
|
+ break
|
|
|
+ self.loading_tmp.append(region.get_id())
|
|
|
+ if prev_id != None:
|
|
|
+ self.get_region(prev_id).register_subregion_id(region.get_id())
|
|
|
+ self.regions.append(region)
|
|
|
+
|
|
|
+ def load_regions(self):
|
|
|
+ if self.regions == None:
|
|
|
+ self.regions = []
|
|
|
+ for each in self.loader.db.iterate_regions(self.get_id()):
|
|
|
+ self.internal_append_region(FileRegionData(self.loader,
|
|
|
+ self.get_id(),
|
|
|
+ each.region_id,
|
|
|
+ each.name,
|
|
|
+ each.begin,
|
|
|
+ each.end,
|
|
|
+ each.line_begin,
|
|
|
+ each.line_end,
|
|
|
+ each.cursor,
|
|
|
+ each.group,
|
|
|
+ each.checksum))
|
|
|
+ assert(len(self.regions) == each.region_id)
|
|
|
+
|
|
|
+ def add_region(self, region_name, offset_begin, offset_end, line_begin, line_end, cursor_line, group, checksum):
|
|
|
+ if self.regions == None:
|
|
|
+ self.regions = [] # do not load in time of collection
|
|
|
+ new_id = len(self.regions) + 1
|
|
|
+ self.internal_append_region(FileRegionData(self.loader, self.get_id(), new_id, region_name, offset_begin, offset_end, line_begin, line_end, cursor_line, group, checksum))
|
|
|
+ self.loader.db.create_region(self.file_id, new_id, region_name, offset_begin, offset_end, line_begin, line_end, cursor_line, group, checksum)
|
|
|
+ return new_id
|
|
|
+
|
|
|
+ def get_region(self, region_id):
|
|
|
+ self.load_regions()
|
|
|
+ return self.regions[region_id - 1]
|
|
|
+
|
|
|
+ def get_region_types(self):
|
|
|
+ return FileRegionData.T
|
|
|
+
|
|
|
+ def iterate_regions(self, filter_group = FileRegionData.T.ANY):
|
|
|
+ self.load_regions()
|
|
|
+ for each in self.regions:
|
|
|
+ if each.group & filter_group:
|
|
|
+ yield each
|
|
|
+
|
|
|
+ def are_regions_loaded(self):
|
|
|
+ return self.regions != None
|
|
|
+
|
|
|
+ def load_markers(self):
|
|
|
+ if self.markers == None:
|
|
|
+ self.markers = []
|
|
|
+ for each in self.loader.db.iterate_markers(self.get_id()):
|
|
|
+ self.markers.append(Marker(each.begin, each.end, each.group))
|
|
|
+
|
|
|
+ def add_marker(self, offset_begin, offset_end, group):
|
|
|
+ if self.markers == None:
|
|
|
+ self.markers = [] # do not load in time of collection
|
|
|
+ self.markers.append(Marker(offset_begin, offset_end, group))
|
|
|
+ self.loader.db.create_marker(self.file_id, offset_begin, offset_end, group)
|
|
|
+
|
|
|
+ def iterate_markers(self, filter_group = Marker.T.COMMENT |
|
|
|
+ Marker.T.STRING | Marker.T.PREPROCESSOR,
|
|
|
+ region_id = None, exclude_children = True, merge = False):
|
|
|
+ self.load_markers()
|
|
|
+
|
|
|
+ # merged markers
|
|
|
+ if merge == True:
|
|
|
+ next_marker = None
|
|
|
+ for marker in self.iterate_markers(filter_group, region_id, exclude_children, merge = False):
|
|
|
+ if next_marker != None:
|
|
|
+ if next_marker.get_offset_end() == marker.get_offset_begin():
|
|
|
+ # sequential markers
|
|
|
+ next_marker = Marker(next_marker.get_offset_begin(),
|
|
|
+ marker.get_offset_end(),
|
|
|
+ next_marker.get_type() | marker.get_type())
|
|
|
+ else:
|
|
|
+ yield next_marker
|
|
|
+ next_marker = None
|
|
|
+ if next_marker == None:
|
|
|
+ next_marker = Marker(marker.get_offset_begin(),
|
|
|
+ marker.get_offset_end(),
|
|
|
+ marker.get_type())
|
|
|
+ if next_marker != None:
|
|
|
+ yield next_marker
|
|
|
+
|
|
|
+ # all markers per file
|
|
|
+ elif region_id == None:
|
|
|
+ next_code_marker_start = 0
|
|
|
+ for marker in self.markers:
|
|
|
+ if Marker.T.CODE & filter_group and next_code_marker_start < marker.get_offset_begin():
|
|
|
+ yield Marker(next_code_marker_start, marker.get_offset_begin(), Marker.T.CODE)
|
|
|
+ if marker.group & filter_group:
|
|
|
+ yield marker
|
|
|
+ next_code_marker_start = marker.get_offset_end()
|
|
|
+ if Marker.T.CODE & filter_group and next_code_marker_start < len(self.get_content()):
|
|
|
+ yield Marker(next_code_marker_start, len(self.get_content()), Marker.T.CODE)
|
|
|
+
|
|
|
+ # markers per region
|
|
|
+ else:
|
|
|
+ region = self.get_region(region_id)
|
|
|
+ if region != None:
|
|
|
+
|
|
|
+ # code parsers and database know about non-code markers
|
|
|
+ # clients want to iterate code as markers as well
|
|
|
+ # so, we embed code markers in run-time
|
|
|
+ class CodeMarker(Marker):
|
|
|
+ pass
|
|
|
+
|
|
|
+ # cache markers for all regions if it does not exist
|
|
|
+ if hasattr(region, '_markers_list') == False:
|
|
|
+ # subroutine to populate _markers_list attribute
|
|
|
+ # _markers_list does include code markers
|
|
|
+ def cache_markers_list_rec(data, region_id, marker_start_ind, next_code_marker_start):
|
|
|
+ region = data.get_region(region_id)
|
|
|
+ region._markers_list = []
|
|
|
+ region._first_marker_ind = marker_start_ind
|
|
|
+ #next_code_marker_start = region.get_offset_begin()
|
|
|
+
|
|
|
+ for sub_id in region.iterate_subregion_ids():
|
|
|
+ subregion = data.get_region(sub_id)
|
|
|
+ # cache all markers before the subregion
|
|
|
+ while len(data.markers) > marker_start_ind and \
|
|
|
+ subregion.get_offset_begin() > data.markers[marker_start_ind].get_offset_begin():
|
|
|
+ if next_code_marker_start < data.markers[marker_start_ind].get_offset_begin():
|
|
|
+ # append code markers coming before non-code marker
|
|
|
+ region._markers_list.append(CodeMarker(next_code_marker_start,
|
|
|
+ data.markers[marker_start_ind].get_offset_begin(),
|
|
|
+ Marker.T.CODE))
|
|
|
+ next_code_marker_start = data.markers[marker_start_ind].get_offset_end()
|
|
|
+ region._markers_list.append(marker_start_ind)
|
|
|
+ marker_start_ind += 1
|
|
|
+
|
|
|
+ # cache all code markers before the subregion but after the last marker
|
|
|
+ if next_code_marker_start < subregion.get_offset_begin():
|
|
|
+ region._markers_list.append(CodeMarker(next_code_marker_start,
|
|
|
+ subregion.get_offset_begin(),
|
|
|
+ Marker.T.CODE))
|
|
|
+ next_code_marker_start = subregion.get_offset_begin()
|
|
|
+
|
|
|
+ # here is the recursive call for all sub-regions
|
|
|
+ (marker_start_ind, next_code_marker_start) = cache_markers_list_rec(data,
|
|
|
+ sub_id,
|
|
|
+ marker_start_ind,
|
|
|
+ next_code_marker_start)
|
|
|
+
|
|
|
+ # cache all markers after the last subregion
|
|
|
+ while len(data.markers) > marker_start_ind and \
|
|
|
+ region.get_offset_end() > data.markers[marker_start_ind].get_offset_begin():
|
|
|
+ # append code markers coming before non-code marker
|
|
|
+ if next_code_marker_start < data.markers[marker_start_ind].get_offset_begin():
|
|
|
+ region._markers_list.append(CodeMarker(next_code_marker_start,
|
|
|
+ data.markers[marker_start_ind].get_offset_begin(),
|
|
|
+ Marker.T.CODE))
|
|
|
+ next_code_marker_start = data.markers[marker_start_ind].get_offset_end()
|
|
|
+ region._markers_list.append(marker_start_ind)
|
|
|
+ marker_start_ind += 1
|
|
|
+
|
|
|
+ # cache the last code segment after the last marker
|
|
|
+ if next_code_marker_start < region.get_offset_end():
|
|
|
+ region._markers_list.append(CodeMarker(next_code_marker_start,
|
|
|
+ region.get_offset_end(),
|
|
|
+ Marker.T.CODE))
|
|
|
+ next_code_marker_start = region.get_offset_end()
|
|
|
+
|
|
|
+ # return the starting point for the next call of this function
|
|
|
+ return (marker_start_ind, next_code_marker_start)
|
|
|
+
|
|
|
+ # append markers list to all regions recursively
|
|
|
+ (next_marker_pos, next_code_marker_start) = cache_markers_list_rec(self, 1, 0, 0)
|
|
|
+ assert(next_marker_pos == len(self.markers))
|
|
|
+
|
|
|
+ # excluding subregions
|
|
|
+ if exclude_children == True:
|
|
|
+ for marker_ind in region._markers_list:
|
|
|
+ if isinstance(marker_ind, int):
|
|
|
+ marker = self.markers[marker_ind]
|
|
|
+ else:
|
|
|
+ marker = marker_ind # CodeMarker
|
|
|
+ if marker.group & filter_group:
|
|
|
+ yield marker
|
|
|
+
|
|
|
+
|
|
|
+ # including subregions
|
|
|
+ else:
|
|
|
+ next_code_marker_start = region.get_offset_begin()
|
|
|
+ for marker in self.markers[region._first_marker_ind:]:
|
|
|
+ if marker.get_offset_begin() >= region.get_offset_end():
|
|
|
+ break
|
|
|
+ if region.get_offset_begin() > marker.get_offset_begin():
|
|
|
+ continue
|
|
|
+ if Marker.T.CODE & filter_group and next_code_marker_start < marker.get_offset_begin():
|
|
|
+ yield Marker(next_code_marker_start, marker.get_offset_begin(), Marker.T.CODE)
|
|
|
+ if marker.group & filter_group:
|
|
|
+ yield marker
|
|
|
+ next_code_marker_start = marker.get_offset_end()
|
|
|
+ if Marker.T.CODE & filter_group and next_code_marker_start < region.get_offset_end():
|
|
|
+ yield Marker(next_code_marker_start, region.get_offset_end(), Marker.T.CODE)
|
|
|
+
|
|
|
+ def get_marker_types(self):
|
|
|
+ return Marker.T
|
|
|
+
|
|
|
+ def are_markers_loaded(self):
|
|
|
+ return self.markers != None
|
|
|
+
|
|
|
+ def __repr__(self):
|
|
|
+ return Data.__repr__(self) + " and regions " + self.regions.__repr__()
|
|
|
+
|
|
|
|
|
|
class BasePlugin(object):
|
|
|
|
|
|
@@ -136,24 +536,32 @@ class Plugin(BasePlugin):
|
|
|
|
|
|
class MetricPlugin(Plugin):
|
|
|
|
|
|
- def add_field(self, is_active, field,
|
|
|
- pattern_to_search,
|
|
|
+ def declare_metric(self, is_active, field,
|
|
|
+ pattern_to_search_or_map_of_patterns,
|
|
|
marker_type_mask=Marker.T.ANY,
|
|
|
+ region_type_mask=FileRegionData.T.ANY,
|
|
|
exclude_subregions=True,
|
|
|
merge_markers=False):
|
|
|
if hasattr(self, '_fields') == False:
|
|
|
self._fields = {}
|
|
|
+
|
|
|
+ if isinstance(pattern_to_search_or_map_of_patterns, dict):
|
|
|
+ map_of_patterns = pattern_to_search_or_map_of_patterns
|
|
|
+ else:
|
|
|
+ map_of_patterns = {'default': pattern_to_search_or_map_of_patterns}
|
|
|
+
|
|
|
if is_active == True:
|
|
|
self._fields[field.name] = (field,
|
|
|
marker_type_mask,
|
|
|
exclude_subregions,
|
|
|
merge_markers,
|
|
|
- pattern_to_search)
|
|
|
+ map_of_patterns,
|
|
|
+ region_type_mask)
|
|
|
|
|
|
- def is_active(self, field_name = None):
|
|
|
- if field_name == None:
|
|
|
+ def is_active(self, metric_name = None):
|
|
|
+ if metric_name == None:
|
|
|
return (len(self._fields.keys()) > 0)
|
|
|
- return (field_name in self._fields.keys())
|
|
|
+ return (metric_name in self._fields.keys())
|
|
|
|
|
|
def get_fields(self):
|
|
|
result = []
|
|
|
@@ -161,24 +569,26 @@ class MetricPlugin(Plugin):
|
|
|
result.append(self._fields[key][0])
|
|
|
return result
|
|
|
|
|
|
- def count_if_active(self, field_name, data, namespace=None):
|
|
|
- if self.is_active(field_name) == False:
|
|
|
+ def count_if_active(self, metric_name, data, namespace=None, alias='default'):
|
|
|
+ if self.is_active(metric_name) == False:
|
|
|
return
|
|
|
|
|
|
if namespace == None:
|
|
|
namespace = self.get_name()
|
|
|
|
|
|
- field_data = self._fields[field_name]
|
|
|
+ field_data = self._fields[metric_name]
|
|
|
text = data.get_content()
|
|
|
- for region in data.iterate_regions():
|
|
|
+ # TODO raise exception if alias is bad
|
|
|
+ pattern_to_search = field_data[4][alias]
|
|
|
+ for region in data.iterate_regions(filter_group=field_data[5]):
|
|
|
count = 0
|
|
|
for marker in data.iterate_markers(
|
|
|
filter_group = field_data[1],
|
|
|
region_id = region.get_id(),
|
|
|
exclude_children = field_data[2],
|
|
|
merge=field_data[3]):
|
|
|
- count += len(field_data[4].findall(text, marker.get_offset_begin(), marker.get_offset_end()))
|
|
|
- region.set_data(namespace, field_name, count)
|
|
|
+ count += len(pattern_to_search.findall(text, marker.get_offset_begin(), marker.get_offset_end()))
|
|
|
+ region.set_data(namespace, metric_name, count)
|
|
|
|
|
|
class InterfaceNotImplemented(Exception):
|
|
|
def __init__(self, obj):
|
avkonst