collect.py 7.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171
  1. #
  2. # Metrix++, Copyright 2009-2013, Metrix++ Project
  3. # Link: http://metrixplusplus.sourceforge.net
  4. #
  5. # This file is a part of Metrix++ Tool.
  6. #
  7. # Metrix++ is free software: you can redistribute it and/or modify
  8. # it under the terms of the GNU General Public License as published by
  9. # the Free Software Foundation, version 3 of the License.
  10. #
  11. # Metrix++ is distributed in the hope that it will be useful,
  12. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. # GNU General Public License for more details.
  15. #
  16. # You should have received a copy of the GNU General Public License
  17. # along with Metrix++. If not, see <http://www.gnu.org/licenses/>.
  18. #
  19. import mpp.api
  20. import re
  21. import os
  22. import logging
  23. import time
  24. import binascii
  25. import fnmatch
  26. class Plugin(mpp.api.Plugin, mpp.api.Parent, mpp.api.IConfigurable, mpp.api.IRunable):
  27. def __init__(self):
  28. self.reader = DirectoryReader()
  29. self.exclude_rules = []
  30. self.exclude_files = []
  31. self.parsers = []
  32. super(Plugin, self).__init__()
  33. def declare_configuration(self, parser):
  34. parser.add_option("--std.general.proctime", "--sgpt", action="store_true", default=False,
  35. help="If the option is set (True), the tool measures processing time per file [default: %default]")
  36. parser.add_option("--std.general.procerrors", "--sgpe", action="store_true", default=False,
  37. help="If the option is set (True), the tool counts number of processing/parsing errors per file [default: %default]")
  38. parser.add_option("--std.general.size", "--sgs", action="store_true", default=False,
  39. help="If the option is set (True), the tool collects file size metric (in bytes) [default: %default]")
  40. parser.add_option("--exclude-files", "--ef", default=r'^[.]',
  41. help="Defines the pattern to exclude files from processing [default: %default]")
  42. parser.add_option("--non-recursively", "--nr", action="store_true", default=False,
  43. help="If the option is set (True), sub-directories are not processed [default: %default]")
  44. self.optparser = parser
  45. def configure(self, options):
  46. self.is_proctime_enabled = options.__dict__['std.general.proctime']
  47. self.is_procerrors_enabled = options.__dict__['std.general.procerrors']
  48. self.is_size_enabled = options.__dict__['std.general.size']
  49. try:
  50. self.add_exclude_rule(re.compile(options.__dict__['exclude_files']))
  51. except Exception as e:
  52. self.optparser.error("option --exclude-files: " + str(e))
  53. self.non_recursively = options.__dict__['non_recursively']
  54. def initialize(self):
  55. fields = []
  56. if self.is_proctime_enabled == True:
  57. fields.append(self.Field('proctime', float))
  58. if self.is_procerrors_enabled == True:
  59. fields.append(self.Field('procerrors', int))
  60. if self.is_size_enabled == True:
  61. fields.append(self.Field('size', int))
  62. super(Plugin, self).initialize(namespace='std.general', support_regions=False, fields=fields)
  63. self.add_exclude_file(self.get_plugin('mpp.dbf').get_dbfile_path())
  64. self.add_exclude_file(self.get_plugin('mpp.dbf').get_dbfile_prev_path())
  65. def run(self, args):
  66. if len(args) == 0:
  67. return self.reader.run(self, "./")
  68. for directory in args:
  69. return self.reader.run(self, directory)
  70. def register_parser(self, fnmatch_exp_list, parser):
  71. self.parsers.append((fnmatch_exp_list, parser))
  72. def get_parser(self, file_path):
  73. for parser in self.parsers:
  74. for fnmatch_exp in parser[0]:
  75. if fnmatch.fnmatch(file_path, fnmatch_exp):
  76. return parser[1]
  77. return None
  78. def add_exclude_rule(self, re_compiled_pattern):
  79. self.exclude_rules.append(re_compiled_pattern)
  80. def add_exclude_file(self, file_path):
  81. if file_path == None:
  82. return
  83. self.exclude_files.append(file_path)
  84. def is_file_excluded(self, file_name):
  85. for each in self.exclude_rules:
  86. if re.match(each, os.path.basename(file_name)) != None:
  87. return True
  88. for each in self.exclude_files:
  89. if os.path.basename(each) == os.path.basename(file_name):
  90. if os.stat(each) == os.stat(file_name):
  91. return True
  92. return False
  93. class DirectoryReader():
  94. def run(self, plugin, directory):
  95. IS_TEST_MODE = False
  96. if 'METRIXPLUSPLUS_TEST_MODE' in os.environ.keys():
  97. IS_TEST_MODE = True
  98. def run_per_file(plugin, fname, full_path):
  99. exit_code = 0
  100. norm_path = re.sub(r'''[\\]''', "/", full_path)
  101. if plugin.is_file_excluded(norm_path) == False:
  102. if os.path.isdir(full_path):
  103. if plugin.non_recursively == False:
  104. exit_code += run_recursively(plugin, full_path)
  105. else:
  106. parser = plugin.get_parser(full_path)
  107. if parser == None:
  108. logging.info("Skipping: " + norm_path)
  109. else:
  110. logging.info("Processing: " + norm_path)
  111. ts = time.time()
  112. f = open(full_path, 'r');
  113. text = f.read();
  114. f.close()
  115. checksum = binascii.crc32(text) & 0xffffffff # to match python 3
  116. db_loader = plugin.get_plugin('mpp.dbf').get_loader()
  117. (data, is_updated) = db_loader.create_file_data(norm_path, checksum, text)
  118. procerrors = parser.process(plugin, data, is_updated)
  119. if plugin.is_proctime_enabled == True:
  120. data.set_data('std.general', 'proctime',
  121. (time.time() - ts) if IS_TEST_MODE == False else 0.01)
  122. if plugin.is_procerrors_enabled == True and procerrors != None and procerrors != 0:
  123. data.set_data('std.general', 'procerrors', procerrors)
  124. if plugin.is_size_enabled == True:
  125. data.set_data('std.general', 'size', len(text))
  126. db_loader.save_file_data(data)
  127. logging.debug("-" * 60)
  128. exit_code += procerrors
  129. else:
  130. logging.info("Excluding: " + norm_path)
  131. return exit_code
  132. def run_recursively(plugin, directory):
  133. exit_code = 0
  134. for fname in os.listdir(directory):
  135. full_path = os.path.join(directory, fname)
  136. exit_code += run_per_file(plugin, fname, full_path)
  137. return exit_code
  138. if os.path.exists(directory) == False:
  139. logging.error("Skipping (does not exist): " + directory)
  140. return 1
  141. if os.path.isdir(directory):
  142. total_errors = run_recursively(plugin, directory)
  143. else:
  144. total_errors = run_per_file(plugin, os.path.basename(directory), directory)
  145. total_errors = total_errors # used, warnings are per file if not zero
  146. return 0 # ignore errors, collection is successful anyway