collect.py 7.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168
  1. #
  2. # Metrix++, Copyright 2009-2013, Metrix++ Project
  3. # Link: http://metrixplusplus.sourceforge.net
  4. #
  5. # This file is a part of Metrix++ Tool.
  6. #
  7. # Metrix++ is free software: you can redistribute it and/or modify
  8. # it under the terms of the GNU General Public License as published by
  9. # the Free Software Foundation, version 3 of the License.
  10. #
  11. # Metrix++ is distributed in the hope that it will be useful,
  12. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. # GNU General Public License for more details.
  15. #
  16. # You should have received a copy of the GNU General Public License
  17. # along with Metrix++. If not, see <http://www.gnu.org/licenses/>.
  18. #
  19. import mpp.api
  20. import re
  21. import os
  22. import logging
  23. import time
  24. import binascii
  25. import fnmatch
  26. class Plugin(mpp.api.Plugin, mpp.api.Parent, mpp.api.IConfigurable, mpp.api.IRunable):
  27. def __init__(self):
  28. self.reader = DirectoryReader()
  29. self.exclude_rules = []
  30. self.exclude_files = []
  31. self.parsers = []
  32. def register_parser(self, fnmatch_exp_list, parser):
  33. self.parsers.append((fnmatch_exp_list, parser))
  34. def get_parser(self, file_path):
  35. for parser in self.parsers:
  36. for fnmatch_exp in parser[0]:
  37. if fnmatch.fnmatch(file_path, fnmatch_exp):
  38. return parser[1]
  39. return None
  40. def declare_configuration(self, parser):
  41. parser.add_option("--std.general.proctime", "--sgpt", action="store_true", default=False,
  42. help="If the option is set (True), the tool measures processing time per file [default: %default]")
  43. parser.add_option("--std.general.procerrors", "--sgpe", action="store_true", default=False,
  44. help="If the option is set (True), the tool counts number of processing/parsing errors per file [default: %default]")
  45. parser.add_option("--std.general.size", "--sgs", action="store_true", default=False,
  46. help="If the option is set (True), the tool collects file size metric (in bytes) [default: %default]")
  47. parser.add_option("--exclude-files", "--ef", default=r'^[.]',
  48. help="Defines the pattern to exclude files from processing [default: %default]")
  49. parser.add_option("--non-recursively", "--nr", action="store_true", default=False,
  50. help="If the option is set (True), sub-directories are not processed [default: %default]")
  51. def configure(self, options):
  52. self.non_recursively = options.__dict__['non_recursively']
  53. self.add_exclude_rule(re.compile(options.__dict__['exclude_files']))
  54. self.is_proctime_enabled = options.__dict__['std.general.proctime']
  55. self.is_procerrors_enabled = options.__dict__['std.general.procerrors']
  56. self.is_size_enabled = options.__dict__['std.general.size']
  57. def initialize(self):
  58. fields = []
  59. if self.is_proctime_enabled == True:
  60. fields.append(self.Field('proctime', float))
  61. if self.is_procerrors_enabled == True:
  62. fields.append(self.Field('procerrors', int))
  63. if self.is_size_enabled == True:
  64. fields.append(self.Field('size', int))
  65. super(Plugin, self).initialize(namespace='std.general', support_regions=False, fields=fields)
  66. self.add_exclude_file(self.get_plugin_loader().get_plugin('mpp.dbf').get_dbfile_path())
  67. self.add_exclude_file(self.get_plugin_loader().get_plugin('mpp.dbf').get_dbfile_prev_path())
  68. def run(self, args):
  69. if len(args) == 0:
  70. return self.reader.run(self, "./")
  71. for directory in args:
  72. return self.reader.run(self, directory)
  73. def add_exclude_rule(self, re_compiled_pattern):
  74. # TODO file name may have special regexp symbols what causes an exception
  75. # For example try to run a collection with "--db-file=metrix++" option
  76. self.exclude_rules.append(re_compiled_pattern)
  77. def add_exclude_file(self, file_path):
  78. if file_path == None:
  79. return
  80. self.exclude_files.append(file_path)
  81. def is_file_excluded(self, file_name):
  82. for each in self.exclude_rules:
  83. if re.match(each, os.path.basename(file_name)) != None:
  84. return True
  85. for each in self.exclude_files:
  86. if os.path.basename(each) == os.path.basename(file_name):
  87. if os.stat(each) == os.stat(file_name):
  88. return True
  89. return False
  90. class DirectoryReader():
  91. def run(self, plugin, directory):
  92. IS_TEST_MODE = False
  93. if 'METRIXPLUSPLUS_TEST_MODE' in os.environ.keys():
  94. IS_TEST_MODE = True
  95. def run_per_file(plugin, fname, full_path):
  96. exit_code = 0
  97. norm_path = re.sub(r'''[\\]''', "/", full_path)
  98. if plugin.is_file_excluded(norm_path) == False:
  99. if os.path.isdir(full_path):
  100. if plugin.non_recursively == False:
  101. exit_code += run_recursively(plugin, full_path)
  102. else:
  103. parser = plugin.get_parser(full_path)
  104. if parser == None:
  105. logging.info("Skipping: " + norm_path)
  106. else:
  107. logging.info("Processing: " + norm_path)
  108. ts = time.time()
  109. f = open(full_path, 'r');
  110. text = f.read();
  111. f.close()
  112. checksum = binascii.crc32(text) & 0xffffffff # to match python 3
  113. db_loader = plugin.get_plugin_loader().get_plugin('mpp.dbf').get_loader()
  114. (data, is_updated) = db_loader.create_file_data(norm_path, checksum, text)
  115. procerrors = parser.process(plugin, data, is_updated)
  116. if plugin.is_proctime_enabled == True:
  117. data.set_data('std.general', 'proctime',
  118. (time.time() - ts) if IS_TEST_MODE == False else 0.01)
  119. if plugin.is_procerrors_enabled == True and procerrors != None and procerrors != 0:
  120. data.set_data('std.general', 'procerrors', procerrors)
  121. if plugin.is_size_enabled == True:
  122. data.set_data('std.general', 'size', len(text))
  123. db_loader.save_file_data(data)
  124. logging.debug("-" * 60)
  125. exit_code += procerrors
  126. else:
  127. logging.info("Excluding: " + norm_path)
  128. return exit_code
  129. def run_recursively(plugin, directory):
  130. exit_code = 0
  131. for fname in os.listdir(directory):
  132. full_path = os.path.join(directory, fname)
  133. exit_code += run_per_file(plugin, fname, full_path)
  134. return exit_code
  135. if os.path.exists(directory) == False:
  136. logging.error("Skipping (does not exist): " + directory)
  137. return 1
  138. if os.path.isdir(directory):
  139. total_errors = run_recursively(plugin, directory)
  140. else:
  141. total_errors = run_per_file(plugin, os.path.basename(directory), directory)
  142. total_errors = total_errors # used, warnings are per file if not zero
  143. return 0 # ignore errors, collection is successful anyway