# # Metrix++, Copyright 2009-2013, Metrix++ Project # Link: http://metrixplusplus.sourceforge.net # # This file is a part of Metrix++ Tool. # # Metrix++ is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, version 3 of the License. # # Metrix++ is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Metrix++. If not, see . # import logging class Data(object): def __init__(self): self.data = {} def get_data(self, namespace, field): if namespace not in self.data.keys(): return None if field not in self.data[namespace].keys(): return None return self.data[namespace][field] def set_data(self, namespace, field, value): if namespace not in self.data: self.data[namespace] = {} self.data[namespace][field] = value def iterate_namespaces(self): for namespace in self.data.keys(): yield namespace def iterate_fields(self, namespace): for field in self.data[namespace].keys(): yield (field, self.data[namespace][field]) def get_data_tree(self, namespaces=None): return self.data def __repr__(self): return object.__repr__(self) + " with data " + self.data.__repr__() class LoadableData(Data): def __init__(self, loader, file_id, region_id): Data.__init__(self) self.loader = loader self.file_id = file_id self.region_id = region_id self.loaded_namespaces = [] self.changed_namespaces = [] def load_namespace(self, namespace): try: row = self.loader.db.get_row(namespace, self.file_id, self.region_id) except Exception: logging.debug("No data in the database for namespace: " + namespace) return if row == None: return for column_name in row.keys(): packager = self.loader.get_namespace(namespace).get_field_packager(column_name) if packager == None: continue if row[column_name] == None: continue Data.set_data(self, namespace, column_name, packager.unpack(row[column_name])) def set_data(self, namespace, field, value): if namespace not in self.changed_namespaces: self.changed_namespaces.append(namespace) return Data.set_data(self, namespace, field, value) def get_data(self, namespace, field): if namespace not in self.loaded_namespaces: self.loaded_namespaces.append(namespace) self.load_namespace(namespace) return Data.get_data(self, namespace, field) def is_namespace_updated(self, namespace): return namespace in self.changed_namespaces def is_namespace_loaded(self, namespace): return namespace in self.loaded_namespaces def get_data_tree(self, namespaces=None): if namespaces == None: namespaces = self.loader.iterate_namespace_names() for each in namespaces: self.load_namespace(each) return Data.get_data_tree(self) class FileRegionData(LoadableData): class T(object): NONE = 0x00 GLOBAL = 0x01 CLASS = 0x02 STRUCT = 0x04 NAMESPACE = 0x08 FUNCTION = 0x10 INTERFACE = 0x20 ANY = 0xFF def to_str(self, group): if group == self.NONE: return "none" elif group == self.GLOBAL: return "global" elif group == self.CLASS: return "class" elif group == self.STRUCT: return "struct" elif group == self.NAMESPACE: return "namespace" elif group == self.FUNCTION: return "function" elif group == self.INTERFACE: return "interface" else: assert(False) def __init__(self, loader, file_id, region_id, region_name, offset_begin, offset_end, line_begin, line_end, cursor_line, group, checksum): LoadableData.__init__(self, loader, file_id, region_id) self.name = region_name self.begin = offset_begin self.end = offset_end self.line_begin = line_begin self.line_end = line_end self.cursor = cursor_line self.group = group self.checksum = checksum self.children = [] def get_id(self): return self.region_id def get_name(self): return self.name def get_offset_begin(self): return self.begin def get_offset_end(self): return self.end def get_line_begin(self): return self.line_begin def get_line_end(self): return self.line_end def get_cursor(self): return self.cursor def get_type(self): return self.group def get_checksum(self): return self.checksum def register_subregion_id(self, child_id): self.children.append(child_id) def iterate_subregion_ids(self): return self.children class Marker(object): class T(object): NONE = 0x00 COMMENT = 0x01 STRING = 0x02 PREPROCESSOR = 0x04 CODE = 0x08 ALL_EXCEPT_CODE = 0x07 ANY = 0xFF def to_str(self, group): if group == self.NONE: return "none" elif group == self.COMMENT: return "comment" elif group == self.STRING: return "string" elif group == self.PREPROCESSOR: return "preprocessor" elif group == self.CODE: return "code" else: assert(False) def __init__(self, offset_begin, offset_end, group): self.begin = offset_begin self.end = offset_end self.group = group def get_offset_begin(self): return self.begin def get_offset_end(self): return self.end def get_type(self): return self.group class FileData(LoadableData): def __init__(self, loader, path, file_id, checksum, content): LoadableData.__init__(self, loader, file_id, None) self.path = path self.checksum = checksum self.content = content self.regions = None self.markers = None self.loader = loader self.loading_tmp = [] def get_id(self): return self.file_id def get_path(self): return self.path def get_checksum(self): return self.checksum def get_content(self): return self.content def internal_append_region(self, region): # here we apply some magic - we rely on special ordering of coming regions, # which is supported by code parsers prev_id = None while True: if len(self.loading_tmp) == 0: break prev_id = self.loading_tmp.pop() if self.get_region(prev_id).get_offset_end() > region.get_offset_begin(): self.loading_tmp.append(prev_id) # return back break self.loading_tmp.append(region.get_id()) if prev_id != None: self.get_region(prev_id).register_subregion_id(region.get_id()) self.regions.append(region) def load_regions(self): if self.regions == None: self.regions = [] for each in self.loader.db.iterate_regions(self.get_id()): self.internal_append_region(FileRegionData(self.loader, self.get_id(), each.region_id, each.name, each.begin, each.end, each.line_begin, each.line_end, each.cursor, each.group, each.checksum)) assert(len(self.regions) == each.region_id) def add_region(self, region_name, offset_begin, offset_end, line_begin, line_end, cursor_line, group, checksum): if self.regions == None: self.regions = [] # do not load in time of collection new_id = len(self.regions) + 1 self.internal_append_region(FileRegionData(self.loader, self.get_id(), new_id, region_name, offset_begin, offset_end, line_begin, line_end, cursor_line, group, checksum)) self.loader.db.create_region(self.file_id, new_id, region_name, offset_begin, offset_end, line_begin, line_end, cursor_line, group, checksum) return new_id def get_region(self, region_id): self.load_regions() return self.regions[region_id - 1] def get_region_types(self): return FileRegionData.T def iterate_regions(self, filter_group = FileRegionData.T.ANY): self.load_regions() for each in self.regions: if each.group & filter_group: yield each def are_regions_loaded(self): return self.regions != None def load_markers(self): if self.markers == None: self.markers = [] for each in self.loader.db.iterate_markers(self.get_id()): self.markers.append(Marker(each.begin, each.end, each.group)) def add_marker(self, offset_begin, offset_end, group): if self.markers == None: self.markers = [] # do not load in time of collection self.markers.append(Marker(offset_begin, offset_end, group)) self.loader.db.create_marker(self.file_id, offset_begin, offset_end, group) def iterate_markers(self, filter_group = Marker.T.COMMENT | Marker.T.STRING | Marker.T.PREPROCESSOR, region_id = None, exclude_children = True, merge = False): self.load_markers() # merged markers if merge == True: next_marker = None for marker in self.iterate_markers(filter_group, region_id, exclude_children, merge = False): if next_marker != None: if next_marker.get_offset_end() == marker.get_offset_begin(): # sequential markers next_marker = Marker(next_marker.get_offset_begin(), marker.get_offset_end(), next_marker.get_type() | marker.get_type()) else: yield next_marker next_marker = None if next_marker == None: next_marker = Marker(marker.get_offset_begin(), marker.get_offset_end(), marker.get_type()) if next_marker != None: yield next_marker # all markers per file elif region_id == None: next_code_marker_start = 0 for marker in self.markers: if Marker.T.CODE & filter_group and next_code_marker_start < marker.get_offset_begin(): yield Marker(next_code_marker_start, marker.get_offset_begin(), Marker.T.CODE) if marker.group & filter_group: yield marker next_code_marker_start = marker.get_offset_end() if Marker.T.CODE & filter_group and next_code_marker_start < len(self.get_content()): yield Marker(next_code_marker_start, len(self.get_content()), Marker.T.CODE) # markers per region else: region = self.get_region(region_id) if region != None: # code parsers and database know about non-code markers # clients want to iterate code as markers as well # so, we embed code markers in run-time class CodeMarker(Marker): pass # cache markers for all regions if it does not exist if hasattr(region, '_markers_list') == False: # subroutine to populate _markers_list attribute # _markers_list does include code markers def cache_markers_list_rec(data, region_id, marker_start_ind, next_code_marker_start): region = data.get_region(region_id) region._markers_list = [] region._first_marker_ind = marker_start_ind #next_code_marker_start = region.get_offset_begin() for sub_id in region.iterate_subregion_ids(): subregion = data.get_region(sub_id) # cache all markers before the subregion while len(data.markers) > marker_start_ind and \ subregion.get_offset_begin() > data.markers[marker_start_ind].get_offset_begin(): if next_code_marker_start < data.markers[marker_start_ind].get_offset_begin(): # append code markers coming before non-code marker region._markers_list.append(CodeMarker(next_code_marker_start, data.markers[marker_start_ind].get_offset_begin(), Marker.T.CODE)) next_code_marker_start = data.markers[marker_start_ind].get_offset_end() region._markers_list.append(marker_start_ind) marker_start_ind += 1 # cache all code markers before the subregion but after the last marker if next_code_marker_start < subregion.get_offset_begin(): region._markers_list.append(CodeMarker(next_code_marker_start, subregion.get_offset_begin(), Marker.T.CODE)) next_code_marker_start = subregion.get_offset_begin() # here is the recursive call for all sub-regions (marker_start_ind, next_code_marker_start) = cache_markers_list_rec(data, sub_id, marker_start_ind, next_code_marker_start) # cache all markers after the last subregion while len(data.markers) > marker_start_ind and \ region.get_offset_end() > data.markers[marker_start_ind].get_offset_begin(): # append code markers coming before non-code marker if next_code_marker_start < data.markers[marker_start_ind].get_offset_begin(): region._markers_list.append(CodeMarker(next_code_marker_start, data.markers[marker_start_ind].get_offset_begin(), Marker.T.CODE)) next_code_marker_start = data.markers[marker_start_ind].get_offset_end() region._markers_list.append(marker_start_ind) marker_start_ind += 1 # cache the last code segment after the last marker if next_code_marker_start < region.get_offset_end(): region._markers_list.append(CodeMarker(next_code_marker_start, region.get_offset_end(), Marker.T.CODE)) next_code_marker_start = region.get_offset_end() # return the starting point for the next call of this function return (marker_start_ind, next_code_marker_start) # append markers list to all regions recursively (next_marker_pos, next_code_marker_start) = cache_markers_list_rec(self, 1, 0, 0) assert(next_marker_pos == len(self.markers)) # excluding subregions if exclude_children == True: for marker_ind in region._markers_list: if isinstance(marker_ind, int): marker = self.markers[marker_ind] else: marker = marker_ind # CodeMarker if marker.group & filter_group: yield marker # including subregions else: next_code_marker_start = region.get_offset_begin() for marker in self.markers[region._first_marker_ind:]: if marker.get_offset_begin() >= region.get_offset_end(): break if region.get_offset_begin() > marker.get_offset_begin(): continue if Marker.T.CODE & filter_group and next_code_marker_start < marker.get_offset_begin(): yield Marker(next_code_marker_start, marker.get_offset_begin(), Marker.T.CODE) if marker.group & filter_group: yield marker next_code_marker_start = marker.get_offset_end() if Marker.T.CODE & filter_group and next_code_marker_start < region.get_offset_end(): yield Marker(next_code_marker_start, region.get_offset_end(), Marker.T.CODE) def get_marker_types(self): return Marker.T def are_markers_loaded(self): return self.markers != None def __repr__(self): return Data.__repr__(self) + " and regions " + self.regions.__repr__() class BasePlugin(object): def initialize(self): pass def terminate(self): pass def set_name(self, name): self.name = name def get_name(self): if hasattr(self, 'name') == False: return None return self.name def set_version(self, version): self.version = version def get_version(self): if hasattr(self, 'version') == False: return None return self.version def set_plugin_loader(self, loader): self.plugin_loader = loader def get_plugin_loader(self): if hasattr(self, 'plugin_loader') == False: return None return self.plugin_loader class Plugin(BasePlugin): class Field(object): def __init__(self, name, ftype, non_zero=False): self.name = name self.type = ftype self.non_zero = non_zero class Property(object): def __init__(self, name, value): self.name = name self.value = value def initialize(self, namespace=None, support_regions=True, fields=[], properties=[]): super(Plugin, self).initialize() if hasattr(self, 'is_updated') == False: self.is_updated = False # original initialization db_loader = self.get_plugin_loader().get_database_loader() if namespace == None: namespace = self.get_name() if (len(fields) != 0 or len(properties) != 0): prev_version = db_loader.set_property(self.get_name() + ":version", self.get_version()) if prev_version != self.get_version(): self.is_updated = True for prop in properties: assert(prop.name != 'version') prev_prop = db_loader.set_property(self.get_name() + ":" + prop.name, prop.value) if prev_prop != prop.value: self.is_updated = True if len(fields) != 0: namespace_obj = db_loader.create_namespace(namespace, support_regions=support_regions, version=self.get_version()) for field in fields: is_created = namespace_obj.add_field(field.name, field.type, non_zero=field.non_zero) assert(is_created != None) # if field is created (not cloned from the previous db), # mark the plug-in as updated in order to trigger full rescan self.is_updated = self.is_updated or is_created class MetricPlugin(Plugin): def declare_metric(self, is_active, field, pattern_to_search_or_map_of_patterns, marker_type_mask=Marker.T.ANY, region_type_mask=FileRegionData.T.ANY, exclude_subregions=True, merge_markers=False): if hasattr(self, '_fields') == False: self._fields = {} if isinstance(pattern_to_search_or_map_of_patterns, dict): map_of_patterns = pattern_to_search_or_map_of_patterns else: map_of_patterns = {'default': pattern_to_search_or_map_of_patterns} if is_active == True: self._fields[field.name] = (field, marker_type_mask, exclude_subregions, merge_markers, map_of_patterns, region_type_mask) def is_active(self, metric_name = None): if metric_name == None: return (len(self._fields.keys()) > 0) return (metric_name in self._fields.keys()) def get_fields(self): result = [] for key in self._fields.keys(): result.append(self._fields[key][0]) return result def count_if_active(self, metric_name, data, namespace=None, alias='default'): if self.is_active(metric_name) == False: return if namespace == None: namespace = self.get_name() field_data = self._fields[metric_name] text = data.get_content() # TODO raise exception if alias is bad pattern_to_search = field_data[4][alias] for region in data.iterate_regions(filter_group=field_data[5]): count = 0 for marker in data.iterate_markers( filter_group = field_data[1], region_id = region.get_id(), exclude_children = field_data[2], merge=field_data[3]): count += len(pattern_to_search.findall(text, marker.get_offset_begin(), marker.get_offset_end())) region.set_data(namespace, metric_name, count) class InterfaceNotImplemented(Exception): def __init__(self, obj): import sys Exception.__init__(self, "Method '" + sys._getframe(1).f_code.co_name + "' has not been implemented for " + str(obj.__class__)) class IConfigurable(object): def configure(self, options): raise InterfaceNotImplemented(self) def declare_configuration(self, optparser): raise InterfaceNotImplemented(self) class IRunable(object): def run(self, args): raise InterfaceNotImplemented(self) class IParser(object): def process(self, parent, data, is_updated): raise InterfaceNotImplemented(self) class ICode(object): pass class ITool(object): def run(self, tool_args): raise InterfaceNotImplemented(self) class CallbackNotImplemented(Exception): def __init__(self, obj, callback_name): Exception.__init__(self, "Callback '" + callback_name + "' has not been implemented for " + str(obj.__class__)) class Child(object): def notify(self, parent, callback_name, *args): if hasattr(self, callback_name) == False: raise CallbackNotImplemented(self, callback_name) self.__getattribute__(callback_name)(parent, *args) def subscribe_by_parents_name(self, parent_name, callback_name='callback'): self.get_plugin_loader().get_plugin(parent_name).subscribe(self, callback_name) def subscribe_by_parents_interface(self, interface, callback_name='callback'): for plugin in self.get_plugin_loader().iterate_plugins(): if isinstance(plugin, interface): plugin.subscribe(self, callback_name) class Parent(object): def init_Parent(self): if hasattr(self, 'children') == False: self.children = [] def subscribe(self, obj, callback_name): self.init_Parent() if (isinstance(obj, Child) == False): raise TypeError() self.children.append((obj,callback_name)) def unsubscribe(self, obj, callback_name): self.init_Parent() self.children.remove((obj, callback_name)) def notify_children(self, *args): self.init_Parent() for child in self.children: child[0].notify(self, child[1], *args) def iterate_children(self): self.init_Parent() for child in self.children: yield child # TODO re-factor and remove this class ExitError(Exception): def __init__(self, plugin, reason): if plugin != None: Exception.__init__(self, "Plugin '" + plugin.get_name() + "' requested abnormal termination: " + reason) else: Exception.__init__(self, "'Abnormal termination requested: " + reason)