metrixpp/mpp/api.py

#
#    Metrix++, Copyright 2009-2019, Metrix++ Project
#    Link: https://github.com/metrixplusplus/metrixplusplus
#    
#    This file is a part of Metrix++ Tool.
#    

import os.path
import sys

import mpp.internal.dbwrap
import mpp.internal.api_impl

class InterfaceNotImplemented(Exception):
    def __init__(self, obj):
        Exception.__init__(self, "Method '"
                            + sys._getframe(1).f_code.co_name
                            + "' has not been implemented for "
                            + str(obj.__class__))

class IConfigurable(object):
    def configure(self, options):
        raise InterfaceNotImplemented(self)
    def declare_configuration(self, optparser):
        raise InterfaceNotImplemented(self)

class IRunable(object):
    def run(self, args):
        raise InterfaceNotImplemented(self)
    
class IParser(object):
    def process(self, parent, data, is_updated):
        raise InterfaceNotImplemented(self)

class ICode(object):
    pass

class CallbackNotImplemented(Exception):
    
    def __init__(self, obj, callback_name):
        Exception.__init__(self, "Callback '"
                           + callback_name
                           + "' has not been implemented for "
                           + str(obj.__class__))

class Child(object):
    
    def notify(self, parent, callback_name, *args):
        if hasattr(self, callback_name) == False:
            raise CallbackNotImplemented(self, callback_name)
        self.__getattribute__(callback_name)(parent, *args)

    def subscribe_by_parents_name(self, parent_name, callback_name='callback'):
        self.get_plugin(parent_name).subscribe(self, callback_name)
    
    def subscribe_by_parents_names(self, parent_names, callback_name='callback'):
        for parent_name in parent_names:
            self.get_plugin(parent_name).subscribe(self, callback_name)

    def subscribe_by_parents_interface(self, interface, callback_name='callback'):
        for plugin in self._get_plugin_loader().iterate_plugins():
            if isinstance(plugin, interface):
                plugin.subscribe(self, callback_name)


class Parent(object):
    
    def init_Parent(self):
        if hasattr(self, 'children') == False:
            self.children = []
            
    def subscribe(self, obj, callback_name):
        self.init_Parent()
        if (isinstance(obj, Child) == False):
            raise TypeError()
        self.children.append((obj,callback_name))

    def unsubscribe(self, obj, callback_name):
        self.init_Parent()
        self.children.remove((obj, callback_name))

    def notify_children(self, *args):
        self.init_Parent()
        for child in self.children:
            child[0].notify(self, child[1], *args)

    def iterate_children(self):
        self.init_Parent()
        for child in self.children:
            yield child

##############################################################################
#
# 
#
##############################################################################

class Data(object):

    def __init__(self):
        self.data = {}
        
    def get_data(self, namespace, field):
        if namespace not in list(self.data.keys()):
            return None
        if field not in list(self.data[namespace].keys()):
            return None
        return self.data[namespace][field]

    def set_data(self, namespace, field, value):
        if namespace not in self.data:
            self.data[namespace] = {}
        self.data[namespace][field] = value

    def iterate_namespaces(self):
        for namespace in list(self.data.keys()):
            yield namespace
            
    def iterate_fields(self, namespace):
        for field in list(self.data[namespace].keys()):
            yield (field, self.data[namespace][field])

    def get_data_tree(self, namespaces=None):
        return self.data

    def __repr__(self):
        return object.__repr__(self) + " with data " + self.data.__repr__()


class LoadableData(Data):
    
    def __init__(self, loader, file_id, region_id):
        Data.__init__(self)
        self.loader = loader
        self.file_id = file_id
        self.region_id = region_id
        self.loaded_namespaces = []
        self.changed_namespaces = []

    def load_namespace(self, namespace):
        namespace_obj = self.loader.get_namespace(namespace)
        if namespace_obj == None:
            return
        regions_supported = namespace_obj.are_regions_supported()
        if ((self.region_id == None and regions_supported == True) or 
            (self.region_id != None and regions_supported == False)):
            return
        row = self.loader.db.get_row(namespace, self.file_id, self.region_id)
        if row == None:
            return
        for column_name in list(row.keys()):
            try:
                packager = namespace_obj._get_field_packager(column_name)
            except mpp.internal.api_impl.PackagerError:
                continue
            if row[column_name] == None:
                continue
            Data.set_data(self, namespace, column_name, packager.unpack(row[column_name]))
        
    def set_data(self, namespace, field, value):
        if namespace not in self.changed_namespaces:
            self.changed_namespaces.append(namespace)
        return Data.set_data(self, namespace, field, value)

    def get_data(self, namespace, field):
        if namespace not in self.loaded_namespaces:
            self.loaded_namespaces.append(namespace)
            self.load_namespace(namespace)
        return Data.get_data(self, namespace, field)
    
    def is_namespace_updated(self, namespace):
        return namespace in self.changed_namespaces

    def is_namespace_loaded(self, namespace):
        return namespace in self.loaded_namespaces

    def get_data_tree(self, namespaces=None):
        if namespaces == None:
            namespaces = self.loader.iterate_namespace_names()
        for each in namespaces:
            self.load_namespace(each)
        return Data.get_data_tree(self)
    
class Region(LoadableData):
    
    class T(object):
        NONE      = 0x00
        GLOBAL    = 0x01
        CLASS     = 0x02
        STRUCT    = 0x04
        NAMESPACE = 0x08
        FUNCTION  = 0x10
        INTERFACE = 0x20
        ANY       = 0xFF
        
        def to_str(self, group):
            if group == self.NONE:
                return "none"
            if group == self.ANY:
                return "any"
            result = []
            if group & self.GLOBAL != 0:
                result.append("global")
            if group & self.CLASS != 0:
                result.append("class")
            if group & self.STRUCT != 0:
                result.append("struct")
            if group & self.NAMESPACE != 0:
                result.append("namespace")
            if group & self.FUNCTION != 0:
                result.append("function")
            if group & self.INTERFACE != 0:
                result.append("interface")
            assert(len(result) != 0)
            return ', '.join(result)

        def from_str(self, group):
            if group == "global":
                return self.GLOBAL
            elif group == "class":
                return self.CLASS
            elif group == "struct":
                return self.STRUCT
            elif group == "namespace":
                return self.NAMESPACE
            elif group == "function":
                return self.FUNCTION
            elif group == "interface":
                return self.INTERFACE
            elif group == "any":
                return self.ANY
            else:
                return None

    def __init__(self, loader, file_id, region_id, region_name, offset_begin, offset_end, line_begin, line_end, cursor_line, group, checksum):
        LoadableData.__init__(self, loader, file_id, region_id)
        self.name = region_name
        self.begin = offset_begin
        self.end = offset_end
        self.line_begin = line_begin
        self.line_end = line_end
        self.cursor = cursor_line
        self.group = group
        self.checksum = checksum
        
        self.children = []
    
    def get_id(self):
        return self.region_id

    def get_name(self):
        return self.name

    def get_offset_begin(self):
        return self.begin

    def get_offset_end(self):
        return self.end

    def get_line_begin(self):
        return self.line_begin

    def get_line_end(self):
        return self.line_end

    def get_cursor(self):
        return self.cursor

    def get_type(self):
        return self.group

    def get_checksum(self):
        return self.checksum
    
    def iterate_subregion_ids(self):
        return self.children

    def _register_subregion_id(self, child_id):
        self.children.append(child_id)
        
class Marker(object):
    class T(object):
        NONE            = 0x00
        COMMENT         = 0x01
        STRING          = 0x02
        PREPROCESSOR    = 0x04
        CODE            = 0x08
        ANY             = 0xFF

        def to_str(self, group):
            if group == self.NONE:
                return "none"
            elif group == self.COMMENT:
                return "comment"
            elif group == self.STRING:
                return "string"
            elif group == self.PREPROCESSOR:
                return "preprocessor"
            elif group == self.CODE:
                return "code"
            else:
                assert(False)
        
    def __init__(self, offset_begin, offset_end, group):
        self.begin = offset_begin
        self.end = offset_end
        self.group = group
        
    def get_offset_begin(self):
        return self.begin

    def get_offset_end(self):
        return self.end

    def get_type(self):
        return self.group

class FileData(LoadableData):
    
    def __init__(self, loader, path, file_id, checksum, content):
        LoadableData.__init__(self, loader, file_id, None)
        self.path = path
        self.checksum = checksum
        self.content = content
        self.regions = None
        self.markers = None
        self.loader = loader
        self.loading_tmp = []
        
    def get_id(self):
        return self.file_id

    def get_path(self):
        return self.path

    def get_checksum(self):
        return self.checksum
    
    def get_content(self):
        return self.content

    def _internal_append_region(self, region):
        # here we apply some magic - we rely on special ordering of coming regions,
        # which is supported by code parsers
        prev_id = None
        while True:
            if len(self.loading_tmp) == 0:
                break
            prev_id = self.loading_tmp.pop()
            if self.get_region(prev_id).get_offset_end() > region.get_offset_begin():
                self.loading_tmp.append(prev_id) # return back
                break
        self.loading_tmp.append(region.get_id())
        if prev_id != None:
            self.get_region(prev_id)._register_subregion_id(region.get_id())
        self.regions.append(region)

    def load_regions(self):
        if self.regions == None:
            self.regions = []
            for each in self.loader.db.iterate_regions(self.get_id()):
                self._internal_append_region(Region(self.loader,
                                                   self.get_id(),
                                                   each.region_id,
                                                   each.name,
                                                   each.begin,
                                                   each.end,
                                                   each.line_begin,
                                                   each.line_end,
                                                   each.cursor,
                                                   each.group,
                                                   each.checksum))
                assert(len(self.regions) == each.region_id)
        
    def add_region(self, region_name, offset_begin, offset_end, line_begin, line_end, cursor_line, group, checksum):
        if self.regions == None:
            # # do not load regions and markers in time of collection
            # if region is added first by parser, set markers to empty list as well
            # because if there are no markers in a file, it forces loading of markers
            # during iterate_markers call
            self.regions = []
            self.markers = []
        new_id = len(self.regions) + 1
        self._internal_append_region(Region(self.loader, self.get_id(), new_id, region_name, offset_begin, offset_end, line_begin, line_end, cursor_line, group, checksum))
        self.loader.db.create_region(self.file_id, new_id, region_name, offset_begin, offset_end, line_begin, line_end, cursor_line, group, checksum)
        return new_id
        
    def get_region(self, region_id):
        self.load_regions()
        return self.regions[region_id - 1]
    
    def iterate_regions(self, filter_group = Region.T.ANY, region_id = None):
        self.load_regions()
        if region_id == None:
            for each in self.regions:
                if each.group & filter_group:
                    yield each
        else:
            for sub_id in self.get_region(region_id).iterate_subregion_ids():
                each = self.get_region(sub_id)
                if each.group & filter_group:
                    yield each

    def are_regions_loaded(self):
        return self.regions != None

    def load_markers(self):
        if self.markers == None:
            # TODO add assert in case of an attempt to load data during collection
            assert(False) # TODO not used in post-processing tools for while, need to be fixed
            self.markers = []
            for each in self.loader.db.iterate_markers(self.get_id()):
                self.markers.append(Marker(each.begin, each.end, each.group))
        
    def add_marker(self, offset_begin, offset_end, group):
        if self.markers == None:
            # # do not load regions and markers in time of collection
            # if marker is added first by parser, set regions to empty list as well
            # because if there are no regions in a file, it forces loading of regions
            # during iterate_regions call
            self.regions = []
            self.markers = []
        self.markers.append(Marker(offset_begin, offset_end, group))
        # TODO drop collecting markers, it is faster to double parse
        # it is not the same with regions, it is faster to load regions
        # on iterative re-run
        #self.loader.db.create_marker(self.file_id, offset_begin, offset_end, group)
        
    def iterate_markers(self, filter_group = Marker.T.ANY,
                         region_id = None, exclude_children = True, merge = False):
        self.load_markers()
        
        # merged markers
        if merge == True:
            next_marker = None
            for marker in self.iterate_markers(filter_group, region_id, exclude_children, merge = False):
                if next_marker != None:
                    if next_marker.get_offset_end() == marker.get_offset_begin():
                        # sequential markers
                        next_marker = Marker(next_marker.get_offset_begin(),
                                             marker.get_offset_end(),
                                             next_marker.get_type() | marker.get_type())
                    else:
                        yield next_marker
                        next_marker = None
                if next_marker == None:
                    next_marker = Marker(marker.get_offset_begin(),
                                         marker.get_offset_end(),
                                         marker.get_type())
            if next_marker != None:
                yield next_marker
        
        # all markers per file
        elif region_id == None:
            next_code_marker_start = 0
            for marker in self.markers:
                if Marker.T.CODE & filter_group and next_code_marker_start < marker.get_offset_begin():
                    yield Marker(next_code_marker_start, marker.get_offset_begin(), Marker.T.CODE)
                if marker.group & filter_group:
                    yield marker
                next_code_marker_start = marker.get_offset_end()
            if Marker.T.CODE & filter_group and next_code_marker_start < len(self.get_content()):
                yield Marker(next_code_marker_start, len(self.get_content()), Marker.T.CODE)

        # markers per region
        else:
            region = self.get_region(region_id)
            if region != None:
                
                # code parsers and database know about non-code markers
                # clients want to iterate code as markers as well
                # so, we embed code markers in run-time
                class CodeMarker(Marker):
                    pass
                
                # cache markers for all regions if it does not exist
                if hasattr(region, '_markers_list') == False:
                    # subroutine to populate _markers_list attribute
                    # _markers_list does include code markers
                    def cache_markers_list_rec(data, region_id, marker_start_ind, next_code_marker_start):
                        region = data.get_region(region_id)
                        region._markers_list = []
                        region._first_marker_ind = marker_start_ind
                        #next_code_marker_start = region.get_offset_begin()
                        
                        for sub_id in region.iterate_subregion_ids():
                            subregion = data.get_region(sub_id)
                            # cache all markers before the subregion
                            while len(data.markers) > marker_start_ind and \
                                subregion.get_offset_begin() > data.markers[marker_start_ind].get_offset_begin():
                                    if next_code_marker_start < data.markers[marker_start_ind].get_offset_begin():
                                        # append code markers coming before non-code marker
                                        region._markers_list.append(CodeMarker(next_code_marker_start,
                                                                               data.markers[marker_start_ind].get_offset_begin(),
                                                                               Marker.T.CODE))
                                    next_code_marker_start = data.markers[marker_start_ind].get_offset_end()
                                    region._markers_list.append(marker_start_ind)
                                    marker_start_ind += 1
                                    
                            # cache all code markers before the subregion but after the last marker
                            if next_code_marker_start < subregion.get_offset_begin():
                                region._markers_list.append(CodeMarker(next_code_marker_start,
                                                                       subregion.get_offset_begin(),
                                                                       Marker.T.CODE))
                            next_code_marker_start = subregion.get_offset_begin()
                                
                            # here is the recursive call for all sub-regions
                            (marker_start_ind, next_code_marker_start) = cache_markers_list_rec(data,
                                                                      sub_id,
                                                                      marker_start_ind,
                                                                      next_code_marker_start)
                            
                        # cache all markers after the last subregion
                        while len(data.markers) > marker_start_ind and \
                            region.get_offset_end() > data.markers[marker_start_ind].get_offset_begin():
                                # append code markers coming before non-code marker
                                if next_code_marker_start < data.markers[marker_start_ind].get_offset_begin():
                                    region._markers_list.append(CodeMarker(next_code_marker_start,
                                                                           data.markers[marker_start_ind].get_offset_begin(),
                                                                           Marker.T.CODE))
                                next_code_marker_start = data.markers[marker_start_ind].get_offset_end()
                                region._markers_list.append(marker_start_ind)
                                marker_start_ind += 1
                        
                        # cache the last code segment after the last marker
                        if next_code_marker_start < region.get_offset_end():
                            region._markers_list.append(CodeMarker(next_code_marker_start,
                                                                   region.get_offset_end(),
                                                                   Marker.T.CODE))
                        next_code_marker_start = region.get_offset_end()
                        
                        # return the starting point for the next call of this function
                        return (marker_start_ind, next_code_marker_start)
                    
                    # append markers list to all regions recursively
                    (next_marker_pos, next_code_marker_start) = cache_markers_list_rec(self, 1, 0, 0)
                    assert(next_marker_pos == len(self.markers))
                
                # excluding subregions
                if exclude_children == True:
                    for marker_ind in region._markers_list:
                        if isinstance(marker_ind, int):
                            marker = self.markers[marker_ind]
                        else:
                            marker = marker_ind # CodeMarker
                        if marker.group & filter_group:
                            yield marker
                            
                            
                # including subregions
                else:
                    next_code_marker_start = region.get_offset_begin()
                    for marker in self.markers[region._first_marker_ind:]:
                        if marker.get_offset_begin() >= region.get_offset_end():
                            break
                        if region.get_offset_begin() > marker.get_offset_begin():
                            continue
                        if Marker.T.CODE & filter_group and next_code_marker_start < marker.get_offset_begin():
                            yield Marker(next_code_marker_start, marker.get_offset_begin(), Marker.T.CODE)
                        if marker.group & filter_group:
                            yield marker
                        next_code_marker_start = marker.get_offset_end()
                    if Marker.T.CODE & filter_group and next_code_marker_start < region.get_offset_end():
                        yield Marker(next_code_marker_start, region.get_offset_end(), Marker.T.CODE)

    def are_markers_loaded(self):
        return self.markers != None

    def __repr__(self):
        return Data.__repr__(self) + " and regions " + self.regions.__repr__()

class AggregatedData(Data):
    
    def __init__(self, loader, path):
        Data.__init__(self)
        self.path = path
        self.loader = loader
        self.subdirs = None
        self.subfiles = None
        
    def get_subdirs(self):
        if self.subdirs != None:
            return self.subdirs
        self.subdirs = []
        if self.path != None:
            for subdir in self.loader.db.iterate_dircontent(self.path, include_subdirs = True, include_subfiles = False):
                self.subdirs.append(subdir)
        return self.subdirs
    
    def get_subfiles(self):
        if self.subfiles != None:
            return self.subfiles
        self.subfiles = []
        if self.path != None:
            for subfile in self.loader.db.iterate_dircontent(self.path, include_subdirs = False, include_subfiles = True):
                self.subfiles.append(subfile)
        return self.subfiles


class SelectData(Data):

    def __init__(self, loader, path, file_id, region_id):
        Data.__init__(self)
        self.loader = loader
        self.path = path
        self.file_id = file_id
        self.region_id = region_id
        self.region = None
    
    def get_path(self):
        return self.path
    
    def get_region(self):
        if self.region == None and self.region_id != None:
            row = self.loader.db.get_region(self.file_id, self.region_id)
            if row != None:
                self.region = Region(self.loader,
                                             self.file_id,
                                             self.region_id,
                                             row.name,
                                             row.begin,
                                             row.end,
                                             row.line_begin,
                                             row.line_end,
                                             row.cursor,
                                             row.group,
                                             row.checksum)
        return self.region


class DiffData(Data):
    
    def __init__(self, new_data, old_data):
        Data.__init__(self)
        self.new_data = new_data
        self.old_data = old_data
    
    def get_data(self, namespace, field):
        new_data = self.new_data.get_data(namespace, field)
        old_data = self.old_data.get_data(namespace, field)
        if new_data == None:
            return None
        if old_data == None:
            # non_zero fields has got zero value by default if missed
            # the data can be also unavailable,
            # because previous collection does not include that
            # but external tools (like limit.py) should warn about this,
            # using list of registered database properties
            old_data = 0
        return new_data - old_data


            
####################################
# Loader
####################################

class Namespace(object):
    
    class NamespaceError(Exception):
        def __init__(self, namespace, reason):
            Exception.__init__(self, "Namespace '"
                            + namespace 
                            + "': '"
                            + reason
                            + "'")
    
    class FieldError(Exception):
        def __init__(self, field, reason):
            Exception.__init__(self, "Field '"
                        + field 
                        + "': '"
                        + reason
                        + "'")

    def __init__(self, db_handle, name, support_regions = False, version='1.0'):
        if not isinstance(name, str):
            raise Namespace.NamespaceError(name, "name not a string")
        self.name = name
        self.support_regions = support_regions
        self.fields = {}
        self.db = db_handle
        
        if self.db.check_table(name) == False:        
            self.db.create_table(name, support_regions, version)
        else:
            for column in self.db.iterate_columns(name):
                self.add_field(column.name,
                               mpp.internal.api_impl.PackagerFactory().get_python_type(column.sql_type),
                               non_zero=column.non_zero)
        
    def get_name(self):
        return self.name

    def are_regions_supported(self):
        return self.support_regions
    
    def add_field(self, field_name, python_type, non_zero=False):
        if not isinstance(field_name, str):
            raise Namespace.FieldError(field_name, "field_name not a string")
        packager = mpp.internal.api_impl.PackagerFactory().create(python_type, non_zero)
        if field_name in list(self.fields.keys()):
            raise Namespace.FieldError(field_name, "double used")
        self.fields[field_name] = packager
        
        if self.db.check_column(self.get_name(), field_name) == False:        
            # - False if cloned
            # - True if created
            return self.db.create_column(self.name, field_name, packager.get_sql_type(), non_zero=non_zero)
        return None # if double request
    
    def iterate_field_names(self):
        for name in list(self.fields.keys()):
            yield name
    
    def check_field(self, field_name):
        try:
            self._get_field_packager(field_name)
        except mpp.internal.api_impl.PackagerError:
            return False
        return True

    def get_field_sql_type(self, field_name):
        try:
            return self._get_field_packager(field_name).get_sql_type()
        except mpp.internal.api_impl.PackagerError:
            raise Namespace.FieldError(field_name, 'does not exist')

    def get_field_python_type(self, field_name):
        try:
            return self._get_field_packager(field_name).get_python_type()
        except mpp.internal.api_impl.PackagerError:
            raise Namespace.FieldError(field_name, 'does not exist')


    def is_field_non_zero(self, field_name):
        try:
            return self._get_field_packager(field_name).is_non_zero()
        except mpp.internal.api_impl.PackagerError:
            raise Namespace.FieldError(field_name, 'does not exist')

    def _get_field_packager(self, field_name):
        if field_name in list(self.fields.keys()):
            return self.fields[field_name]
        else:
            raise mpp.internal.api_impl.PackagerError("unknown field " + field_name + " requested")
    
class Loader(object):
    
    def __init__(self):
        self.namespaces = {}
        self.db = None
        self.last_file_data = None # for performance boost reasons
    
    def create_database(self, dbfile, previous_db = None):
        self.db = mpp.internal.dbwrap.Database()
        try:
            self.db.create(dbfile, clone_from=previous_db)
        except:
            return False
        return True
        
    def open_database(self, dbfile, read_only = True):
        self.db = mpp.internal.dbwrap.Database()
        if os.path.exists(dbfile) == False:
            return False
        try:
            self.db.connect(dbfile, read_only=read_only)
        except:
            return False
        
        for table in self.db.iterate_tables():
            self.create_namespace(table.name, table.support_regions)

        return True

    def set_property(self, property_name, value):
        if self.db == None:
            return None
        return self.db.set_property(property_name, str(value))
    
    def get_property(self, property_name):
        if self.db == None:
            return None
        return self.db.get_property(property_name)

    def iterate_properties(self):
        if self.db == None:
            return None
        return self.db.iterate_properties()
            
    def create_namespace(self, name, support_regions = False, version='1.0'):
        if self.db == None:
            return None
        
        if name in list(self.namespaces.keys()):
            raise Namespace.NamespaceError(name, "double used")
        new_namespace = Namespace(self.db, str(name), support_regions, version)
        self.namespaces[name] = new_namespace
        return new_namespace
    
    def iterate_namespace_names(self):
        for name in list(self.namespaces.keys()):
            yield name

    def get_namespace(self, name):
        if name in list(self.namespaces.keys()):
            return self.namespaces[name]
        else:
            return None

    def create_file_data(self, path, checksum, content):
        if self.db == None:
            return None

        (new_id, is_updated) = self.db.create_file(path, checksum)
        result = FileData(self, path, new_id, checksum, content) 
        self.last_file_data = result
        return (result, is_updated)

    def load_file_data(self, path):
        if self.db == None:
            return None

        if self.last_file_data != None and self.last_file_data.get_path() == path:
            return self.last_file_data
        
        data = self.db.get_file(path)
        if data == None:
            return None
        
        result = FileData(self, data.path, data.id, data.checksum, None)
        self.last_file_data = result
        return result

    class DataNotPackable(Exception):
        def __init__(self, namespace, field, value, packager, extra_message):
            Exception.__init__(self, "Data '"
                               + str(value)
                               + "' of type "
                               + str(value.__class__) 
                               + " referred by '"
                               + namespace
                               + "=>"
                               + field
                               + "' is not packable by registered packager '"
                               + str(packager.__class__)
                               + "': " + extra_message)

    def save_file_data(self, file_data):
        if self.db == None:
            return None

        class DataIterator(object):

            def iterate_packed_values(self, data, namespace, support_regions = False):
                for each in data.iterate_fields(namespace):
                    space = self.loader.get_namespace(namespace)
                    if space == None:
                        raise Loader.DataNotPackable(namespace, each[0], each[1], None, "The namespace has not been found")
                    
                    try:
                        packager = space._get_field_packager(each[0])
                    except mpp.internal.api_impl.PackagerError:
                        raise Loader.DataNotPackable(namespace, each[0], each[1], None, "The field has not been found")
        
                    if space.support_regions != support_regions:
                        raise Loader.DataNotPackable(namespace, each[0], each[1], packager, "Incompatible support for regions")
                    
                    try:
                        packed_data = packager.pack(each[1])
                        if packed_data == None:
                            continue
                    except mpp.internal.api_impl.PackagerError:
                        raise Loader.DataNotPackable(namespace, each[0], each[1], packager, "Packager raised exception")
                    
                    yield (each[0], packed_data)
            
            def __init__(self, loader, data, namespace, support_regions = False):
                self.loader = loader
                self.iterator = self.iterate_packed_values(data, namespace, support_regions)
    
            def __iter__(self):
                return self.iterator
        
        # TODO can construct to add multiple rows at one sql query
        # to improve the performance
        for namespace in file_data.iterate_namespaces():
            if file_data.is_namespace_updated(namespace) == False:
                continue
            self.db.add_row(namespace,
                            file_data.get_id(),
                            None,
                            DataIterator(self, file_data, namespace))
        
        if file_data.are_regions_loaded():
            for region in file_data.iterate_regions():
                for namespace in region.iterate_namespaces():
                    if region.is_namespace_updated(namespace) == False:
                        continue
                    self.db.add_row(namespace,
                                    file_data.get_id(),
                                    region.get_id(),
                                    DataIterator(self, region, namespace, support_regions = True))

    def iterate_file_data(self, path = None, path_like_filter = "%"):
        if self.db == None:
            return None
        
        final_path_like = path_like_filter
        if path != None:
            if self.db.check_dir(path) == False and self.db.check_file(path) == False:
                return None
            final_path_like = path + path_like_filter

        class FileDataIterator(object):
            def iterate_file_data(self, loader, final_path_like):
                for data in loader.db.iterate_files(path_like=final_path_like):
                    yield FileData(loader, data.path, data.id, data.checksum, None)
            
            def __init__(self, loader, final_path_like):
                self.iterator = self.iterate_file_data(loader, final_path_like)
    
            def __iter__(self):
                return self.iterator

        if self.db == None:
            return None
        return FileDataIterator(self, final_path_like)

    def load_aggregated_data(self, path = None, path_like_filter = "%", namespaces = None):
        if self.db == None:
            return None

        final_path_like = path_like_filter
        if path != None:
            if self.db.check_dir(path) == False and self.db.check_file(path) == False:
                return None
            final_path_like = path + path_like_filter
        
        if namespaces == None:
            namespaces = list(self.namespaces.keys())
        
        result = AggregatedData(self, path)
        for name in namespaces:
            namespace = self.get_namespace(name)
            data = self.db.aggregate_rows(name, path_like = final_path_like)
            for field in list(data.keys()):
                if namespace.get_field_python_type(field) == str:
                    continue
                data[field]['nonzero'] = namespace.is_field_non_zero(field)
                distribution = self.db.count_rows(name, path_like = final_path_like, group_by_column = field)
                data[field]['distribution-bars'] = []
                for each in distribution:
                    if each[0] == None:
                        continue
                    assert(float(data[field]['count'] != 0))
                    data[field]['distribution-bars'].append({'metric': each[0],
                                                             'count': each[1],
                                                             'ratio': (float(each[1]) / float(data[field]['count']))})
                result.set_data(name, field, data[field])
        return result
    
    def load_selected_data(self, namespace, fields = None, path = None, path_like_filter = "%", filters = [],
                           sort_by = None, limit_by = None):
        if self.db == None:
            return None
        
        final_path_like = path_like_filter
        if path != None:
            if self.db.check_dir(path) == False and self.db.check_file(path) == False:
                return None
            final_path_like = path + path_like_filter
        
        namespace_obj = self.get_namespace(namespace)
        if namespace_obj == None:
            return None
        
        class SelectDataIterator(object):
        
            def iterate_selected_values(self, loader, namespace_obj, final_path_like, fields, filters, sort_by, limit_by):
                for row in loader.db.select_rows(namespace_obj.get_name(), path_like=final_path_like, filters=filters,
                                                 order_by=sort_by, limit_by=limit_by):
                    region_id = None
                    if namespace_obj.are_regions_supported() == True:
                        region_id = row['region_id']
                    data = SelectData(loader, row['path'], row['id'], region_id)
                    field_names = fields
                    if fields == None:
                        field_names = namespace_obj.iterate_field_names()
                    for field in field_names:
                        data.set_data(namespace, field, row[field])
                    yield data
            
            def __init__(self, loader, namespace_obj, final_path_like, fields, filters, sort_by, limit_by):
                self.iterator = self.iterate_selected_values(loader, namespace_obj, final_path_like, fields, filters, sort_by, limit_by)
    
            def __iter__(self):
                return self.iterator

        return SelectDataIterator(self, namespace_obj, final_path_like, fields, filters, sort_by, limit_by)





class BasePlugin(object):
    
    def initialize(self):
        pass

    def terminate(self):
        pass
    
    def set_name(self, name):
        self.name = name

    def get_name(self):
        if hasattr(self, 'name') == False:
            return None
        return self.name

    def get_namespace(self):
        return self.get_name()

    def set_version(self, version):
        self.version = version

    def get_version(self):
        if hasattr(self, 'version') == False:
            return None
        return self.version

    def _set_plugin_loader(self, loader):
        self.plugin_loader = loader

    def _get_plugin_loader(self):
        if hasattr(self, 'plugin_loader') == False:
            return None
        return self.plugin_loader
    
    def get_plugin(self, plugin_name):
        return self._get_plugin_loader().get_plugin(plugin_name)

    def get_action(self):
        return self._get_plugin_loader().get_action()

class Plugin(BasePlugin):

    class Field(object):
        def __init__(self, name, ftype, non_zero=False):
            self.name = name
            self.type = ftype
            self.non_zero = non_zero
            self._regions_supported = True

    class Property(object):
        def __init__(self, name, value):
            self.name = name
            self.value = value
    
    def initialize(self, namespace=None, support_regions=True, fields=[], properties=[]):
        super(Plugin, self).initialize()
        
        if hasattr(self, 'is_updated') == False:
            self.is_updated = False # original initialization

        db_loader = self.get_plugin('mpp.dbf').get_loader()

        if namespace == None:
            namespace = self.get_name()

        if (len(fields) != 0 or len(properties) != 0):
            prev_version = db_loader.set_property(self.get_name() + ":version", self.get_version())
            if str(prev_version) != str(self.get_version()):
                self.is_updated = True

        for prop in properties:
            assert(prop.name != 'version')
            prev_prop = db_loader.set_property(self.get_name() + ":" + prop.name, prop.value)
            if str(prev_prop) != str(prop.value):
                self.is_updated = True

        if len(fields) != 0:
            namespace_obj = db_loader.create_namespace(namespace,
                                                       support_regions=support_regions,
                                                       version=self.get_version())
            for field in fields:
                is_created = namespace_obj.add_field(field.name, field.type, non_zero=field.non_zero)
                field._regions_supported = support_regions
                assert(is_created != None)
                # if field is created (not cloned from the previous db),
                # mark the plug-in as updated in order to trigger full rescan
                self.is_updated = self.is_updated or is_created

class MetricPluginMixin(Parent):

    class AliasError(Exception):
        def __init__(self, alias):
            Exception.__init__(self, "Unknown pattern alias: " + str(alias))
            
    class PlainCounter(object):
        
        def __init__(self, namespace, field, plugin, alias, data, region):
            self.namespace = namespace
            self.field = field
            self.plugin = plugin
            self.alias = alias
            self.data = data
            self.region = region
            self.result = 0
            
        def count(self, marker, pattern_to_search):
            self.result += len(pattern_to_search.findall(self.data.get_content(),
                                                       marker.get_offset_begin(),
                                                       marker.get_offset_end()))
        
        def get_result(self):
            return self.result

    class IterIncrementCounter(PlainCounter):
        
        def count(self, marker, pattern_to_search):
            self.marker = marker
            self.pattern_to_search = pattern_to_search
            for match in pattern_to_search.finditer(self.data.get_content(),
                                                    marker.get_offset_begin(),
                                                    marker.get_offset_end()):
                self.result += self.increment(match)
        
        def increment(self, match):
            return 1

    class IterAssignCounter(PlainCounter):
        
        def count(self, marker, pattern_to_search):
            self.marker = marker
            self.pattern_to_search = pattern_to_search
            for match in pattern_to_search.finditer(self.data.get_content(),
                                                    marker.get_offset_begin(),
                                                    marker.get_offset_end()):
                self.result = self.assign(match)
        
        def assign(self, match):
            return self.result

    class RankedCounter(PlainCounter):
        
        def __init__(self, *args, **kwargs):
            super(MetricPluginMixin.RankedCounter, self).__init__(*args, **kwargs)
            self.result = self.region.get_data(self.namespace, self.field)
            if self.result == None:
                self.result = 1
        
        def get_result(self):
            sourced_metric = self.region.get_data(self.rank_source[0], self.rank_source[1])
            for (ind, range_pair) in enumerate(self.rank_ranges):
                if ((range_pair[0] == None or sourced_metric >= range_pair[0])
                    and
                    (range_pair[1] == None or sourced_metric <= range_pair[1])):
                        self.result = self.result * (ind + 1)
                        break
            return self.result

    def declare_metric(self, is_active, field,
                       pattern_to_search_or_map_of_patterns,
                       marker_type_mask=Marker.T.ANY,
                       region_type_mask=Region.T.ANY,
                       exclude_subregions=True,
                       merge_markers=False):
        if hasattr(self, '_fields') == False:
            self._fields = {}
        
        if isinstance(pattern_to_search_or_map_of_patterns, dict):
            map_of_patterns = pattern_to_search_or_map_of_patterns
        else:
            map_of_patterns = {'*': pattern_to_search_or_map_of_patterns}
        # client may suply with pattern or pair of pattern + counter class
        for key in list(map_of_patterns.keys()):
            if isinstance(map_of_patterns[key], tuple) == False:
                # if it is not a pair, create a pair using default counter class
                map_of_patterns[key] = (map_of_patterns[key],
                                        MetricPluginMixin.PlainCounter)

        if is_active == True:
            self._fields[field.name] = (field,
                                        marker_type_mask,
                                        exclude_subregions,
                                        merge_markers,
                                        map_of_patterns,
                                        region_type_mask)
            
    def is_active(self, metric_name = None):
        if metric_name == None:
            return (len(list(self._fields.keys())) > 0)
        return (metric_name in list(self._fields.keys()))
    
    def get_fields(self):
        result = []
        for key in list(self._fields.keys()):
            result.append(self._fields[key][0])
        return result
    
    def callback(self, parent, data, is_updated):
        # count if metric is enabled, 
        # and (optimization for the case of iterative rescan:)
        # if file is updated or this plugin's settings are updated
        is_updated = is_updated or self.is_updated
        if is_updated == True:
            for field in self.get_fields():
                self.count_if_active(self.get_namespace(),
                                     field.name,
                                     data,
                                     alias=parent.get_name())
        # this mixin implements parent interface
        self.notify_children(data, is_updated)

    def count_if_active(self, namespace, field, data, alias='*'):
        if self.is_active(field) == False:
            return
        
        field_data = self._fields[field]
        if alias not in list(field_data[4].keys()):
            if '*' not in list(field_data[4].keys()):
                raise self.AliasError(alias)
            else:
                alias = '*'
        (pattern_to_search, counter_class) = field_data[4][alias]
        
        if field_data[0]._regions_supported == True:
            for region in data.iterate_regions(filter_group=field_data[5]):
                counter = counter_class(namespace, field, self, alias, data, region)
                if field_data[1] != Marker.T.NONE:
                    for marker in data.iterate_markers(
                                    filter_group = field_data[1],
                                    region_id = region.get_id(),
                                    exclude_children = field_data[2],
                                    merge=field_data[3]):
                        counter.count(marker, pattern_to_search)
                count = counter.get_result()
                if count != 0 or field_data[0].non_zero == False:
                    region.set_data(namespace, field, count)
        else:
            counter = counter_class(namespace, field, self, alias, data, None)
            if field_data[1] != Marker.T.NONE:
                for marker in data.iterate_markers(
                                filter_group = field_data[1],
                                region_id = None,
                                exclude_children = field_data[2],
                                merge=field_data[3]):
                    counter.count(marker, pattern_to_search)
            count = counter.get_result()
            if count != 0 or field_data[0].non_zero == False:
                data.set_data(namespace, field, count)