Major re-organization of metadata and identification of updates.

mainline/core/api.py

@@ -20,7 +20,11 @@
 class Plugin(object):
     
     def initialize(self):
-        pass
+        self.is_updated = False
+        db_loader = self.get_plugin_loader().get_database_loader()
+        prev_version = db_loader.set_property(self.get_name() + ":version", self.get_version())
+        if prev_version != self.get_version():
+            self.is_updated = True
     
     def terminate(self):
         pass
@@ -33,6 +37,14 @@ class Plugin(object):
             return None
         return self.name
 
+    def set_version(self, version):
+        self.version = version
+
+    def get_version(self):
+        if hasattr(self, 'version') == False:
+            return None
+        return self.version
+
     def set_plugin_loader(self, loader):
         self.plugin_loader = loader
 
@@ -62,6 +74,9 @@ class IRunable(object):
     def run(self, args):
         raise InterfaceNotImplemented(self)
     
+class IParser(object):
+    def process(self, parent, data, is_updated):
+        raise InterfaceNotImplemented(self)
 
 class CallbackNotImplemented(Exception):
     

mainline/core/db/loader.py

@@ -303,7 +303,7 @@ class FileData(LoadableData):
         if self.markers == None:
             self.markers = []
             for each in self.loader.db.iterate_markers(self.get_id()):
-                self.markers.append(self.Marker(each.begin, each.end, each.group))
+                self.markers.append(Marker(each.begin, each.end, each.group))
         
     def add_marker(self, offset_begin, offset_end, group):
         if self.markers == None:
@@ -399,8 +399,15 @@ class DiffData(Data):
     def get_data(self, namespace, field):
         new_data = self.new_data.get_data(namespace, field)
         old_data = self.old_data.get_data(namespace, field)
-        if new_data == None or old_data == None:
+        if new_data == None:
             return None
+        if old_data == None:
+            # non_zero fields has got zero value by default if missed
+            # the data can be also unavailable,
+            # because previous collection does not include that
+            # but external tools (like limit.py) should warn about this,
+            # using list of registered database properties
+            old_data = 0
         return new_data - old_data
 
 ####################################
@@ -648,6 +655,21 @@ class Loader(object):
         for table in self.db.iterate_tables():
             self.create_namespace(table.name, table.support_regions)
             
+    def set_property(self, property_name, value):
+        if self.db == None:
+            return None
+        return self.db.set_property(property_name, value)
+    
+    def get_property(self, property_name):
+        if self.db == None:
+            return None
+        return self.db.get_property(property_name)
+
+    def iterate_properties(self):
+        if self.db == None:
+            return None
+        return self.db.iterate_properties()
+            
     def create_namespace(self, name, support_regions = False):
         if self.db == None:
             return None
@@ -672,10 +694,10 @@ class Loader(object):
         if self.db == None:
             return None
 
-        new_id = self.db.create_file(path, checksum)
+        (new_id, is_updated) = self.db.create_file(path, checksum)
         result = FileData(self, path, new_id, checksum, content) 
         self.last_file_data = result
-        return result
+        return (result, is_updated)
 
     def load_file_data(self, path):
         if self.db == None:

mainline/core/db/sqlite.py

@@ -47,6 +47,12 @@ class Database(object):
             self.id = tag_id
             self.name = name
 
+    class PropertyData(object):
+        def __init__(self, property_id, name, value):
+            self.id = property_id
+            self.name = name
+            self.value = value
+
     class FileData(object):
         def __init__(self, file_id, path, checksum):
             self.id = file_id
@@ -93,6 +99,9 @@ class Database(object):
     class InternalCleanUpUtils(object):
         
         def clean_up_not_confirmed(self, db_loader):
+            sql = "DELETE FROM __info__ WHERE (confirmed = 0)"
+            db_loader.log(sql)
+            db_loader.conn.execute(sql)
             sql = "DELETE FROM __tags__ WHERE (confirmed = 0)"
             db_loader.log(sql)
             db_loader.conn.execute(sql)
@@ -213,10 +222,10 @@ class Database(object):
         self.read_only = read_only
         if self.read_only == False:
             try:
-                sql = "CREATE TABLE __info__ (id integer NOT NULL PRIMARY KEY AUTOINCREMENT, property text NOT NULL, value text, UNIQUE (property) ON CONFLICT REPLACE)"
+                sql = "CREATE TABLE __info__ (id integer NOT NULL PRIMARY KEY AUTOINCREMENT, property text NOT NULL, value text, confirmed integer NOT NULL, UNIQUE (property) ON CONFLICT REPLACE)"
                 self.log(sql)
                 self.conn.execute(sql)
-                sql = "INSERT INTO __info__ (property, value) VALUES ('version', '" + self.version + "')"
+                sql = "INSERT INTO __info__ (property, value, confirmed) VALUES ('version', '" + self.version + "', 1)"
                 self.log(sql)
                 self.conn.execute(sql)
                 sql = "CREATE TABLE __tables__ (id integer NOT NULL PRIMARY KEY, name text NOT NULL, support_regions integer NOT NULL, confirmed integer NOT NULL, UNIQUE (name))"
@@ -239,7 +248,35 @@ class Database(object):
                 self.conn.execute(sql)
             except sqlite3.OperationalError as e:
                 logging.debug("sqlite3.OperationalError: " + str(e))
+                
+    def set_property(self, property_name, value):
+        ret_val = None
+        sql = "SELECT * FROM __info__ WHERE (property = '" + property_name + "')"
+        self.log(sql)
+        result = self.conn.execute(sql).fetchall()
+        if len(result) != 0:
+            ret_val = result[0]['value']
+
+        sql = "INSERT INTO __info__ (property, value, confirmed) VALUES ('" + property_name + "', '" + value + "', 1)"
+        self.log(sql)
+        self.conn.execute(sql)
+        return ret_val
         
+    def get_property(self, property_name):
+        ret_val = None
+        sql = "SELECT * FROM __info__ WHERE (property = '" + property_name + "' AND confirmed = 1)"
+        self.log(sql)
+        result = self.conn.execute(sql).fetchall()
+        if len(result) != 0:
+            ret_val = result[0]['value']
+        return ret_val
+
+    def iterate_properties(self):
+        sql = "SELECT * FROM __info__ WHERE (confirmed = 1)"
+        self.log(sql)
+        for each in self.conn.execute(sql).fetchall():
+            yield self.PropertyData(each['id'], each['property'], each['value'])
+
     def create_table(self, table_name, support_regions = False):
         assert(self.read_only == False)
 
@@ -359,6 +396,7 @@ class Database(object):
             return False
         return True
 
+    # TODO activate usage of tags
     def create_file(self, path, checksum):
         assert(self.read_only == False)
         path = self.InternalPathUtils().normalize_path(path)
@@ -373,7 +411,7 @@ class Database(object):
                     sql = "UPDATE __files__ SET confirmed = 1 WHERE (id = " + str(old_id) +")"
                     self.log(sql)
                     self.conn.execute(sql)
-                    return old_id
+                    return (old_id, False)
                 else:
                     self.InternalCleanUpUtils().clean_up_file(self, result[0]['id'])
         
@@ -383,7 +421,7 @@ class Database(object):
         cur = self.conn.cursor()
         cur.execute(sql, column_data)
         self.InternalPathUtils().update_dirs(self, path=path)
-        return cur.lastrowid
+        return (cur.lastrowid, True)
     
     def iterate_dircontent(self, path, include_subdirs = True, include_subfiles = True):
         self.InternalPathUtils().update_dirs(self)
@@ -482,8 +520,8 @@ class Database(object):
     def iterate_markers(self, file_id):
         for each in self.select_rows("__markers__", filters = [("file_id", "=", file_id)]):
             yield self.MarkerData(each['file_id'],
-                                  each['name'],
                                   each['begin'],
+                                  each['end'],
                                   each['group_id'])
 
     def add_row(self, table_name, file_id, region_id, array_data):

mainline/core/dir.py

@@ -78,20 +78,23 @@ class DirectoryReader():
                         if plugin.non_recursively == False:
                             run_recursively(plugin, full_path)
                     else:
-                        logging.info("Processing: " + full_path)
-                        ts = time.time()
-                        
-                        f = open(full_path, 'r');
-                        text = f.read();
-                        f.close()
-                        checksum = binascii.crc32(text) & 0xffffffff # to match python 3
-
-                        data = plugin.get_plugin_loader().get_database_loader().create_file_data(full_path, checksum, text)
-                        plugin.notify_children(data)
-                        if plugin.is_proctime_enabled == True:
-                            data.set_data('general', 'proctime', time.time() - ts)
-                        plugin.get_plugin_loader().get_database_loader().save_file_data(data)
-                        logging.debug("-" * 60)
+                        parser = plugin.get_plugin_loader().get_parser(full_path)
+                        if parser == None:
+                            logging.info("Skipping: " + full_path)
+                        else:
+                            logging.info("Processing: " + full_path)
+                            ts = time.time()
+                            f = open(full_path, 'r');
+                            text = f.read();
+                            f.close()
+                            checksum = binascii.crc32(text) & 0xffffffff # to match python 3
+    
+                            (data, is_updated) = plugin.get_plugin_loader().get_database_loader().create_file_data(full_path, checksum, text)
+                            parser.process(plugin, data, is_updated)
+                            if plugin.is_proctime_enabled == True:
+                                data.set_data('general', 'proctime', time.time() - ts)
+                            plugin.get_plugin_loader().get_database_loader().save_file_data(data)
+                            logging.debug("-" * 60)
                 else:
                     logging.info("Excluding: " + full_path)
                     logging.debug("-" * 60)

mainline/core/loader.py

@@ -18,14 +18,16 @@
 #
 
 import core.api
+import core.db.loader
 
 import os
-import core.db.loader
+import fnmatch
 
 class Loader(object):
 
     def __init__(self):
         self.plugins = []
+        self.parsers = []
         self.hash    = {}
         self.exit_code = 0
         self.db = core.db.loader.Loader()
@@ -44,6 +46,15 @@ class Loader(object):
             for item in reversed(self.plugins):
                 yield item['instance']
             
+    def register_parser(self, fnmatch_exp_list, parser):
+        self.parsers.append((fnmatch_exp_list, parser))
+
+    def get_parser(self, file_path):
+        for parser in self.parsers:
+            for fnmatch_exp in parser[0]:
+                if fnmatch.fnmatch(file_path, fnmatch_exp):
+                    return parser[1]
+        return None
 
     def load(self, directory, optparser):
         import sys
@@ -83,6 +94,7 @@ class Loader(object):
             item['instance'] = class_attr.__new__(class_attr)
             item['instance'].__init__()
             item['instance'].set_name(item['package'] + "." + item['module'])
+            item['instance'].set_version(item['version'])
             item['instance'].set_plugin_loader(self)
 
         for item in self.iterate_plugins():

mainline/export.py

@@ -107,8 +107,8 @@ def main():
             regions = []
             for each in file_data.iterate_regions():
                 region_data_tree = each.get_data_tree(namespaces=namespaces)
-                if regions_matcher != None and regions_matcher.is_matched(each.id):
-                    region_data_prev = file_data_prev.get_region(regions_matcher.get_prev_id(each.id))
+                if regions_matcher != None and regions_matcher.is_matched(each.get_id()):
+                    region_data_prev = file_data_prev.get_region(regions_matcher.get_prev_id(each.get_id()))
                     region_data_tree = append_diff(region_data_tree,
                                                    region_data_prev.get_data_tree(namespaces=namespaces))
                 regions.append({"info": {"name" : each.name,

mainline/ext/std/code/brackets.ini

@@ -1,26 +0,0 @@
-;
-;    Metrix++, Copyright 2009-2013, Metrix++ Project
-;    Link: http://metrixplusplus.sourceforge.net
-;    
-;    This file is a part of Metrix++ Tool.
-;    
-;    Metrix++ is free software: you can redistribute it and/or modify
-;    it under the terms of the GNU General Public License as published by
-;    the Free Software Foundation, version 3 of the License.
-;    
-;    Metrix++ is distributed in the hope that it will be useful,
-;    but WITHOUT ANY WARRANTY; without even the implied warranty of
-;    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-;    GNU General Public License for more details.
-;    
-;    You should have received a copy of the GNU General Public License
-;    along with Metrix++.  If not, see <http://www.gnu.org/licenses/>.
-;
-
-[Plugin]
-version: 1.0
-package: std.code
-module:  brackets
-class:   Plugin
-depends: None
-enabled: True

mainline/ext/std/code/brackets.py

@@ -1,40 +0,0 @@
-#
-#    Metrix++, Copyright 2009-2013, Metrix++ Project
-#    Link: http://metrixplusplus.sourceforge.net
-#    
-#    This file is a part of Metrix++ Tool.
-#    
-#    Metrix++ is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU General Public License as published by
-#    the Free Software Foundation, version 3 of the License.
-#    
-#    Metrix++ is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-#    GNU General Public License for more details.
-#    
-#    You should have received a copy of the GNU General Public License
-#    along with Metrix++.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-import core.api
-
-class Plugin(core.api.Plugin, core.api.Child, core.api.IConfigurable):
-    
-    def declare_configuration(self, parser):
-        parser.add_option("--std.code.brackets.on", action="store_true", default=False,
-                         help="If enabled, counts number of mismatched brackets '{}' [default: %default]")
-    
-    def configure(self, options):
-        self.is_active = options.__dict__['std.code.brackets.on']
-        
-    def initialize(self):
-        if self.is_active == True:
-            namespace = self.get_plugin_loader().get_database_loader().create_namespace(self.get_name())
-            namespace.add_field('curly', int)
-            core.api.subscribe_by_parents_name('std.code.cpp', self, 'callback_cpp')
-
-    def callback_cpp(self, parent, data):
-        if data.get_data(self.get_name(), 'curly') != None:
-            return # data is available from cloned database, skip collection
-        data.set_data(self.get_name(), 'curly', data.get_data(parent.get_name(), 'mismatched_brackets'))

mainline/ext/std/code/complexity.py

@@ -35,26 +35,25 @@ class Plugin(core.api.Plugin, core.api.Child, core.api.IConfigurable):
             namespace = self.get_plugin_loader().get_database_loader().create_namespace(self.get_name(), support_regions = True)
             namespace.add_field('cyclomatic', int)
             core.api.subscribe_by_parents_name('std.code.cpp', self, 'callback_cpp')
+            # trigger version property set
+            core.api.Plugin.initialize(self)
 
     # cyclomatic complexity pattern
     pattern = re.compile(r'''([^0-9A-Za-z_]((if)|(case)|(for)|(while))[^0-9A-Za-z_])|[&][&]|[|][|]|[?]''')
 
-    def callback_cpp(self, parent, data):
+    def callback_cpp(self, parent, data, is_updated):
         
-        text = None
-        for (ind, region) in enumerate(data.iterate_regions(filter_group=data.get_region_types().FUNCTION)):
-            # cyclomatic complexity
-            if ind == 0 and region.get_data(self.get_name(), 'cyclomatic') != None:
-                return # data is available in first from cloned database, skip collection
-            if text == None: # lazy loading for performance benefits
-                text = data.get_content(exclude = data.get_marker_types().ALL_EXCEPT_CODE)
-            
-            count = 0
-            start_pos = region.get_offset_begin()
-            for sub_id in region.iterate_subregion_ids():
-                # exclude sub regions, like enclosed classes
-                count += len(self.pattern.findall(text, start_pos, data.get_region(sub_id).get_offset_begin()))
-                start_pos = data.get_region(sub_id).get_offset_end()
-            count += len(self.pattern.findall(text, start_pos, region.get_offset_end()))
-            region.set_data(self.get_name(), 'cyclomatic', count)
+        is_updated = is_updated or self.is_updated
+        if is_updated == True:
+            text = data.get_content(exclude = data.get_marker_types().ALL_EXCEPT_CODE)
+            for region in data.iterate_regions(filter_group=data.get_region_types().FUNCTION):
+                # cyclomatic complexity
+                count = 0
+                start_pos = region.get_offset_begin()
+                for sub_id in region.iterate_subregion_ids():
+                    # exclude sub regions, like enclosed classes
+                    count += len(self.pattern.findall(text, start_pos, data.get_region(sub_id).get_offset_begin()))
+                    start_pos = data.get_region(sub_id).get_offset_end()
+                count += len(self.pattern.findall(text, start_pos, region.get_offset_end()))
+                region.set_data(self.get_name(), 'cyclomatic', count)
 

mainline/ext/std/code/cpp.ini

@@ -18,7 +18,7 @@
 ;
 
 [Plugin]
-version: 1.0
+version: 1.1
 package: std.code
 module:  cpp
 class:   Plugin

mainline/ext/std/code/cpp.py

@@ -17,14 +17,14 @@
 #    along with Metrix++.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-import fnmatch
+
 import re
 import binascii
 import logging
 
 import core.api
 
-class Plugin(core.api.Plugin, core.api.Parent, core.api.Child, core.api.IConfigurable, core.api.ICode):
+class Plugin(core.api.Plugin, core.api.Parent, core.api.IParser, core.api.IConfigurable, core.api.ICode):
     
     def declare_configuration(self, parser):
         parser.add_option("--std.code.cpp.files", default="*.c,*.cpp,*.h,*.hpp",
@@ -32,26 +32,28 @@ class Plugin(core.api.Plugin, core.api.Parent, core.api.Child, core.api.IConfigu
     
     def configure(self, options):
         self.files = options.__dict__['std.code.cpp.files'].split(',')
+        self.files.sort() # sorted list goes to properties
         
     def initialize(self):
-        core.api.subscribe_by_parents_name('core.dir', self)
+        self.get_plugin_loader().register_parser(self.files, self)
+
+        # trigger version property set
+        core.api.Plugin.initialize(self)
+        db_loader = self.get_plugin_loader().get_database_loader()
+        prev_ext = db_loader.set_property(self.get_name() + ":files", ','.join(self.files))
+        if prev_ext != ','.join(self.files):
+            self.is_updated = True
         
         namespace = self.get_plugin_loader().get_database_loader().create_namespace(self.get_name())
-        namespace.add_field('files', int)
-        namespace.add_field('mismatched_brackets', None)
+        namespace.add_field('mismatched_brackets', int, non_zero=True)
     
-    def callback(self, parent, data):
-        for ext in self.files:
-            if fnmatch.fnmatch(data.get_path(), ext):
-                if data.get_data(self.get_name(), 'files') != None:
-                    self.notify_children(data)
-                    return
-
-                count_mismatched_brackets = CppCodeParser().run(data)
-                data.set_data(self.get_name(), 'files', 1)
+    def process(self, parent, data, is_updated):
+        is_updated = is_updated or self.is_updated
+        if is_updated == True:
+            count_mismatched_brackets = CppCodeParser().run(data)
+            if count_mismatched_brackets != 0:
                 data.set_data(self.get_name(), 'mismatched_brackets', count_mismatched_brackets)
-                self.notify_children(data)
-                break
+        self.notify_children(data, is_updated)
             
 class CppCodeParser(object):
     

mainline/ext/std/code/dumper.py

@@ -39,7 +39,7 @@ class Plugin(core.api.Plugin, core.api.Child, core.api.IConfigurable):
         # do not process files dumped by previous run of this module    
         self.get_plugin_loader().get_plugin('core.dir').add_exclude_rule(re.compile(r'.*' + Plugin.POST_NAME + r'$'))
         
-    def callback(self, parent, data):
+    def callback(self, parent, data, is_updated):
         file_name = data.get_path()
         text = data.get_content()
         

mainline/ext/std/code/test.py

@@ -32,7 +32,7 @@ class Plugin(core.api.Plugin, core.api.Child, core.api.IConfigurable):
         if self.is_active == True:
             core.api.subscribe_by_parents_interface(core.api.ICode, self)
 
-    def callback(self, parent, data):
+    def callback(self, parent, data, is_updated):
         
         def print_rec(data, indent, region_id):
             print ("   ." * indent) + str(data.get_region(region_id).get_type()) + " " + data.get_region(region_id).get_name() + " " + str(data.get_region(region_id).get_cursor())

mainline/info.py

@@ -0,0 +1,68 @@
+#
+#    Metrix++, Copyright 2009-2013, Metrix++ Project
+#    Link: http://metrixplusplus.sourceforge.net
+#    
+#    This file is a part of Metrix++ Tool.
+#    
+#    Metrix++ is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, version 3 of the License.
+#    
+#    Metrix++ is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+#    GNU General Public License for more details.
+#    
+#    You should have received a copy of the GNU General Public License
+#    along with Metrix++.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+import logging
+import time
+
+import core.db.loader
+import core.db.post
+import core.log
+import core.cmdparser
+
+
+def main():
+    exit_code = 0
+    log_plugin = core.log.Plugin()
+    db_plugin = core.db.post.Plugin()
+
+    parser = core.cmdparser.MultiOptionParser(usage="Usage: %prog [options]")
+    log_plugin.declare_configuration(parser)
+    db_plugin.declare_configuration(parser)
+
+    (options, args) = parser.parse_args()
+    log_plugin.configure(options)
+    db_plugin.configure(options)
+    
+    args = args # used
+
+    loader = core.db.loader.Loader()
+    loader.open_database(db_plugin.dbfile)
+    loader_prev = None
+    if db_plugin.dbfile != None:
+        loader_prev = core.db.loader.Loader()
+        loader_prev.open_database(db_plugin.dbfile_prev)
+    print "Info data:"
+    for each in loader.iterate_properties():
+        prev_value_str = ""
+        if loader_prev != None:
+            prev = loader_prev.get_property(each.name)
+            if prev != each.value:
+                prev_value_str = " [previous value: " + loader_prev.get_property(each.name) + "]"
+                print "(!)",
+        print "\t" + each.name + "\t=>\t" + each.value + prev_value_str
+        
+    return exit_code
+            
+if __name__ == '__main__':
+    ts = time.time()
+    core.log.set_default_format()
+    exit_code = main()
+    logging.warning("Exit code: " + str(exit_code) + ". Time spent: " + str(round((time.time() - ts), 2)) + " seconds. Done")
+    exit(exit_code) # number of reported messages, errors are reported as non-handled exceptions

mainline/limit.py

@@ -58,6 +58,16 @@ def main():
     for each in loader.iterate_namespace_names():
         warn_plugin.verify_fields(each, loader.get_namespace(each).iterate_field_names())
     
+    # Check for versions consistency
+    for each in loader.iterate_properties():
+        if db_plugin.dbfile_prev != None:
+            prev = loader_prev.get_property(each.name)
+            if prev != each.value:
+                logging.warn("Previous data has got different metadata:")
+                logging.warn(" - identification of change trends can be not reliable")
+                logging.warn(" - use 'info' tool to get more details")
+                break
+    
     paths = None
     if len(args) == 0:
         paths = [""]
@@ -105,8 +115,6 @@ def main():
                                 is_modified = False
                             diff = core.db.loader.DiffData(select_data,
                                                            file_data_prev.get_region(prev_id)).get_data(limit.namespace, limit.field)
-                            # TODO if diff is None, probably need to warn about this
-                            # a user may expect data available
 
                 if warn_plugin.is_mode_matched(limit.limit, select_data.get_data(limit.namespace, limit.field), diff, is_modified):
                     exit_code += 1