sqlite.py 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639
  1. #
  2. # Metrix++, Copyright 2009-2013, Metrix++ Project
  3. # Link: http://metrixplusplus.sourceforge.net
  4. #
  5. # This file is a part of Metrix++ Tool.
  6. #
  7. # Metrix++ is free software: you can redistribute it and/or modify
  8. # it under the terms of the GNU General Public License as published by
  9. # the Free Software Foundation, version 3 of the License.
  10. #
  11. # Metrix++ is distributed in the hope that it will be useful,
  12. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. # GNU General Public License for more details.
  15. #
  16. # You should have received a copy of the GNU General Public License
  17. # along with Metrix++. If not, see <http://www.gnu.org/licenses/>.
  18. #
  19. import sqlite3
  20. import re
  21. import os.path
  22. import logging
  23. import itertools
  24. import shutil
  25. class Database(object):
  26. last_used_id = 0
  27. version = "1.0"
  28. class TableData(object):
  29. def __init__(self, table_id, name, support_regions):
  30. self.id = table_id
  31. self.name = name
  32. self.support_regions = support_regions
  33. class ColumnData(object):
  34. def __init__(self, column_id, name, sql_type, non_zero):
  35. self.id = column_id
  36. self.name = name
  37. self.sql_type = sql_type
  38. self.non_zero = non_zero
  39. class TagData(object):
  40. def __init__(self, tag_id, name):
  41. self.id = tag_id
  42. self.name = name
  43. class PropertyData(object):
  44. def __init__(self, property_id, name, value):
  45. self.id = property_id
  46. self.name = name
  47. self.value = value
  48. class FileData(object):
  49. def __init__(self, file_id, path, checksum):
  50. self.id = file_id
  51. self.path = path
  52. self.checksum = checksum
  53. class RegionData(object):
  54. def __init__(self, file_id, region_id, name, begin, end, line_begin, line_end, cursor, group, checksum):
  55. self.file_id = file_id
  56. self.region_id = region_id
  57. self.name = name
  58. self.begin = begin
  59. self.end = end
  60. self.line_begin = line_begin
  61. self.line_end = line_end
  62. self.cursor = cursor
  63. self.group = group
  64. self.checksum = checksum
  65. class MarkerData(object):
  66. def __init__(self, file_id, begin, end, group):
  67. self.file_id = file_id
  68. self.begin = begin
  69. self.end = end
  70. self.group = group
  71. def __init__(self):
  72. self.read_only = False
  73. self.conn = None
  74. self.dirs = None
  75. self.is_cloned = False
  76. self.last_used_id += 1
  77. self.id = self.last_used_id
  78. def __del__(self):
  79. if self.conn != None:
  80. if self.is_cloned == True:
  81. logging.debug("Cleaning up database file")
  82. self.InternalCleanUpUtils().clean_up_not_confirmed(self)
  83. logging.debug("Committing database file")
  84. self.conn.commit()
  85. class InternalCleanUpUtils(object):
  86. def clean_up_not_confirmed(self, db_loader):
  87. sql = "DELETE FROM __info__ WHERE (confirmed = 0)"
  88. db_loader.log(sql)
  89. db_loader.conn.execute(sql)
  90. sql = "DELETE FROM __tags__ WHERE (confirmed = 0)"
  91. db_loader.log(sql)
  92. db_loader.conn.execute(sql)
  93. sql = "SELECT * FROM __tables__ WHERE (confirmed = 0)"
  94. db_loader.log(sql)
  95. for table in db_loader.conn.execute(sql).fetchall():
  96. sql = "DELETE FROM __columns__ WHERE table_id = '" + str(table['id']) + "'"
  97. db_loader.log(sql)
  98. db_loader.conn.execute(sql)
  99. sql = "DELETE FROM __tables__ WHERE id = '" + str(table['id']) + "'"
  100. db_loader.log(sql)
  101. db_loader.conn.execute(sql)
  102. sql = "DROP TABLE '" + table['name'] + "'"
  103. db_loader.log(sql)
  104. db_loader.conn.execute(sql)
  105. sql = "SELECT __columns__.name AS column_name, __tables__.name AS table_name, __columns__.id AS column_id FROM __columns__, __tables__ WHERE (__columns__.confirmed = 0 AND __columns__.table_id = __tables__.id)"
  106. db_loader.log(sql)
  107. for column in db_loader.conn.execute(sql).fetchall():
  108. logging.warn("New database file inherits useless column: '" + column['table_name'] + "'.'" + column['column_name'] + "'")
  109. sql = "DELETE FROM __columns__ WHERE id = '" + str(column['column_id']) + "'"
  110. db_loader.log(sql)
  111. db_loader.conn.execute(sql)
  112. sql = "UPDATE '" + column['table_name'] + "' SET '" + column['column_name'] + "' = NULL"
  113. db_loader.log(sql)
  114. db_loader.conn.execute(sql)
  115. self.clean_up_file(db_loader)
  116. def clean_up_file(self, db_loader, file_id = None):
  117. sql = "SELECT * FROM __tables__"
  118. db_loader.log(sql)
  119. for table in itertools.chain(db_loader.conn.execute(sql).fetchall(), [{'name':'__regions__'}, {'name':'__markers__'}]):
  120. sql = ""
  121. if file_id == None:
  122. sql = "DELETE FROM '" + table['name'] + "' WHERE file_id IN (SELECT __files__.id FROM __files__ WHERE __files__.confirmed = 0)"
  123. else:
  124. sql = "DELETE FROM '" + table['name'] + "' WHERE (file_id = " + str(file_id) + ")"
  125. db_loader.log(sql)
  126. db_loader.conn.execute(sql)
  127. class InternalPathUtils(object):
  128. def iterate_heads(self, path):
  129. dirs = []
  130. head = os.path.dirname(path)
  131. last_head = None # to process Windows drives
  132. while (head != "" and last_head != head):
  133. dirs.append(os.path.basename(head))
  134. last_head = head
  135. head = os.path.dirname(head)
  136. dirs.reverse()
  137. for each in dirs:
  138. yield each
  139. def normalize_path(self, path):
  140. if path == None:
  141. return None
  142. return re.sub(r'''[\\]''', "/", path)
  143. def update_dirs(self, db_loader, path = None):
  144. if db_loader.dirs == None:
  145. if path == None:
  146. db_loader.dirs = {} # initial construction
  147. else:
  148. return # avoid useless cache updates
  149. elif path == None:
  150. return # avoid multiple initial constructions
  151. path = self.normalize_path(path)
  152. rows = None
  153. if path == None:
  154. sql = "SELECT * FROM __files__"
  155. db_loader.log(sql)
  156. rows = db_loader.conn.execute(sql).fetchall()
  157. else:
  158. rows = [{"path": path}]
  159. for row in rows:
  160. cur_head = db_loader.dirs
  161. for dir_name in self.iterate_heads(row["path"]):
  162. if dir_name not in cur_head.keys():
  163. cur_head[dir_name] = {}
  164. cur_head = cur_head[dir_name]
  165. cur_head[os.path.basename(row["path"])] = None
  166. def create(self, file_name, clone_from = None):
  167. if clone_from != None:
  168. self.is_cloned = True
  169. logging.debug("Cloning database file: " + clone_from)
  170. shutil.copy2(clone_from, file_name)
  171. logging.debug("Connecting database file: " + file_name)
  172. self.conn = sqlite3.connect(file_name)
  173. self.conn.row_factory = sqlite3.Row
  174. self.read_only = False
  175. sql = "UPDATE __tables__ SET confirmed = 0"
  176. self.log(sql)
  177. self.conn.execute(sql)
  178. sql = "UPDATE __columns__ SET confirmed = 0"
  179. self.log(sql)
  180. self.conn.execute(sql)
  181. sql = "UPDATE __tags__ SET confirmed = 0"
  182. self.log(sql)
  183. self.conn.execute(sql)
  184. sql = "UPDATE __files__ SET confirmed = 0"
  185. self.log(sql)
  186. self.conn.execute(sql)
  187. else:
  188. self.connect(file_name)
  189. def connect(self, file_name, read_only = False):
  190. logging.debug("Connecting database file: " + file_name)
  191. self.conn = sqlite3.connect(file_name)
  192. self.conn.row_factory = sqlite3.Row
  193. self.read_only = read_only
  194. if self.read_only == False:
  195. try:
  196. sql = "CREATE TABLE __info__ (id integer NOT NULL PRIMARY KEY AUTOINCREMENT, property text NOT NULL, value text, confirmed integer NOT NULL, UNIQUE (property) ON CONFLICT REPLACE)"
  197. self.log(sql)
  198. self.conn.execute(sql)
  199. sql = "INSERT INTO __info__ (property, value, confirmed) VALUES ('version', '" + self.version + "', 1)"
  200. self.log(sql)
  201. self.conn.execute(sql)
  202. sql = "CREATE TABLE __tables__ (id integer NOT NULL PRIMARY KEY, name text NOT NULL, support_regions integer NOT NULL, confirmed integer NOT NULL, UNIQUE (name))"
  203. self.log(sql)
  204. self.conn.execute(sql)
  205. sql = "CREATE TABLE __columns__ (id integer NOT NULL PRIMARY KEY, name text NOT NULL, type text NOT NULL, table_id integer NOT_NULL, non_zero integer NOT NULL, confirmed integer NOT NULL, UNIQUE (name, table_id))"
  206. self.log(sql)
  207. self.conn.execute(sql)
  208. sql = "CREATE TABLE __tags__ (id integer NOT NULL PRIMARY KEY, name text NOT NULL UNIQUE, confirmed integer NOT NULL)"
  209. self.log(sql)
  210. self.conn.execute(sql)
  211. sql = "CREATE TABLE __files__ (id integer NOT NULL PRIMARY KEY AUTOINCREMENT, path text NOT NULL, checksum integer NOT NULL, tag1 integer, tag2 integer, tag3 integer, confirmed integer NOT NULL, UNIQUE(path))"
  212. self.log(sql)
  213. self.conn.execute(sql)
  214. sql = "CREATE TABLE __regions__ (file_id integer NOT NULL, region_id integer NOT NULL, name text NOT NULL, begin integer NOT NULL, end integer NOT NULL, line_begin integer NOT NULL, line_end integer NOT NULL, cursor integer NOT NULL, group_id integer NOT NULL, checksum integer NOT NULL, PRIMARY KEY (file_id, region_id))"
  215. self.log(sql)
  216. self.conn.execute(sql)
  217. sql = "CREATE TABLE __markers__ (id integer NOT NULL PRIMARY KEY, file_id integer NOT NULL, begin integer NOT NULL, end integer NOT NULL, group_id integer NOT NULL)"
  218. self.log(sql)
  219. self.conn.execute(sql)
  220. except sqlite3.OperationalError as e:
  221. logging.debug("sqlite3.OperationalError: " + str(e))
  222. def set_property(self, property_name, value):
  223. ret_val = None
  224. sql = "SELECT * FROM __info__ WHERE (property = '" + property_name + "')"
  225. self.log(sql)
  226. result = self.conn.execute(sql).fetchall()
  227. if len(result) != 0:
  228. ret_val = result[0]['value']
  229. sql = "INSERT INTO __info__ (property, value, confirmed) VALUES ('" + property_name + "', '" + value + "', 1)"
  230. self.log(sql)
  231. self.conn.execute(sql)
  232. return ret_val
  233. def get_property(self, property_name):
  234. ret_val = None
  235. sql = "SELECT * FROM __info__ WHERE (property = '" + property_name + "' AND confirmed = 1)"
  236. self.log(sql)
  237. result = self.conn.execute(sql).fetchall()
  238. if len(result) != 0:
  239. ret_val = result[0]['value']
  240. return ret_val
  241. def iterate_properties(self):
  242. sql = "SELECT * FROM __info__ WHERE (confirmed = 1)"
  243. self.log(sql)
  244. for each in self.conn.execute(sql).fetchall():
  245. yield self.PropertyData(each['id'], each['property'], each['value'])
  246. def create_table(self, table_name, support_regions = False):
  247. assert(self.read_only == False)
  248. sql = "SELECT * FROM __tables__ WHERE (name = '" + table_name + "'AND confirmed == 0)"
  249. self.log(sql)
  250. result = self.conn.execute(sql).fetchall()
  251. if len(result) != 0:
  252. sql = "UPDATE __tables__ SET confirmed = 1 WHERE (name = '" + table_name + "')"
  253. self.log(sql)
  254. self.conn.execute(sql)
  255. return
  256. sql = "CREATE TABLE '" + table_name + "' (file_id integer NOT NULL PRIMARY KEY)"
  257. if support_regions == True:
  258. sql = str("CREATE TABLE '" + table_name + "' (file_id integer NOT NULL, region_id integer NOT NULL, "
  259. + "PRIMARY KEY (file_id, region_id))")
  260. self.log(sql)
  261. self.conn.execute(sql)
  262. sql = "INSERT INTO __tables__ (name, support_regions, confirmed) VALUES ('" + table_name + "', '" + str(int(support_regions)) + "', 1)"
  263. self.log(sql)
  264. self.conn.execute(sql)
  265. def iterate_tables(self):
  266. sql = "SELECT * FROM __tables__ WHERE (confirmed = 1)"
  267. self.log(sql)
  268. result = self.conn.execute(sql).fetchall()
  269. for row in result:
  270. yield self.TableData(int(row["id"]), str(row["name"]), bool(row["support_regions"]))
  271. def check_table(self, table_name):
  272. sql = "SELECT * FROM __tables__ WHERE (name = '" + table_name + "' AND confirmed = 1)"
  273. self.log(sql)
  274. result = self.conn.execute(sql).fetchall()
  275. if len(result) == 0:
  276. return False
  277. return True
  278. def create_column(self, table_name, column_name, column_type, non_zero=False):
  279. assert(self.read_only == False)
  280. if column_type == None:
  281. logging.debug("Skipping column '" + column_name + "' creation for table '" + table_name + "'")
  282. return
  283. sql = "SELECT id FROM __tables__ WHERE (name = '" + table_name + "')"
  284. self.log(sql)
  285. table_id = self.conn.execute(sql).next()['id']
  286. sql = "SELECT * FROM __columns__ WHERE (table_id = '" + str(table_id) + "' AND name = '" + column_name + "' AND confirmed == 0)"
  287. self.log(sql)
  288. result = self.conn.execute(sql).fetchall()
  289. if len(result) != 0:
  290. sql = "UPDATE __columns__ SET confirmed = 1 WHERE (table_id = '" + str(table_id) + "' AND name = '" + column_name + "')"
  291. self.log(sql)
  292. self.conn.execute(sql)
  293. return
  294. sql = "ALTER TABLE '" + table_name + "' ADD COLUMN '" + column_name + "' " + column_type
  295. self.log(sql)
  296. self.conn.execute(sql)
  297. sql = "SELECT id FROM __tables__ WHERE (name = '" + table_name + "')"
  298. self.log(sql)
  299. table_id = self.conn.execute(sql).next()['id']
  300. sql = "INSERT INTO __columns__ (name, type, table_id, non_zero, confirmed) VALUES ('" + column_name + "', '" + column_type + "', '" + str(table_id) + "', '" + str(int(non_zero)) + "', 1)"
  301. self.log(sql)
  302. self.conn.execute(sql)
  303. def iterate_columns(self, table_name):
  304. sql = "SELECT id FROM __tables__ WHERE (name = '" + table_name + "')"
  305. self.log(sql)
  306. table_id = self.conn.execute(sql).next()['id']
  307. sql = "SELECT * FROM __columns__ WHERE (table_id = '" + str(table_id) + "' AND confirmed = 1)"
  308. self.log(sql)
  309. result = self.conn.execute(sql).fetchall()
  310. for row in result:
  311. yield self.ColumnData(int(row["id"]), str(row["name"]), str(row["type"]), bool(row["non_zero"]))
  312. def check_column(self, table_name, column_name):
  313. sql = "SELECT id FROM __tables__ WHERE (name = '" + table_name + "')"
  314. self.log(sql)
  315. table_id = self.conn.execute(sql).next()['id']
  316. sql = "SELECT * FROM __columns__ WHERE (table_id = '" + str(table_id) + "' AND name = '" + column_name + "' AND confirmed = 1)"
  317. self.log(sql)
  318. result = self.conn.execute(sql).fetchall()
  319. if len(result) == 0:
  320. return False
  321. return True
  322. def create_tag(self, tag_name):
  323. assert(self.read_only == False)
  324. sql = "SELECT * FROM __tags__ WHERE (name = '" + tag_name + "' AND confirmed == 0)"
  325. self.log(sql)
  326. result = self.conn.execute(sql).fetchall()
  327. if len(result) != 0:
  328. sql = "UPDATE __tags__ SET confirmed = 1 WHERE (name = '" + tag_name + "')"
  329. self.log(sql)
  330. self.conn.execute(sql)
  331. return
  332. sql = "INSERT INTO __tags__ (name, confirmed) VALUES ('" + tag_name + "', 1)"
  333. self.log(sql)
  334. self.conn.execute(sql)
  335. def iterate_tags(self):
  336. sql = "SELECT * FROM __tags__ WHERE (confirmed = 1)"
  337. self.log(sql)
  338. result = self.conn.execute(sql).fetchall()
  339. for row in result:
  340. yield self.TagData(int(row["id"]), str(row["name"]))
  341. def check_tag(self, tag_name):
  342. sql = "SELECT * FROM __tags__ WHERE (name = '" + tag_name + "' AND confirmed = 1)"
  343. self.log(sql)
  344. result = self.conn.execute(sql).fetchall()
  345. if len(result) == 0:
  346. return False
  347. return True
  348. # TODO activate usage of tags
  349. def create_file(self, path, checksum):
  350. assert(self.read_only == False)
  351. path = self.InternalPathUtils().normalize_path(path)
  352. if self.is_cloned == True:
  353. sql = "SELECT * FROM __files__ WHERE (path = '" + path + "')"
  354. self.log(sql)
  355. result = self.conn.execute(sql).fetchall()
  356. if len(result) != 0:
  357. if result[0]['checksum'] == checksum:
  358. old_id = result[0]['id']
  359. sql = "UPDATE __files__ SET confirmed = 1 WHERE (id = " + str(old_id) +")"
  360. self.log(sql)
  361. self.conn.execute(sql)
  362. return (old_id, False)
  363. else:
  364. self.InternalCleanUpUtils().clean_up_file(self, result[0]['id'])
  365. sql = "INSERT OR REPLACE INTO __files__ (path, checksum, confirmed) VALUES (?, ?, 1)"
  366. column_data = [path, checksum]
  367. self.log(sql + " /with arguments: " + str(column_data))
  368. cur = self.conn.cursor()
  369. cur.execute(sql, column_data)
  370. self.InternalPathUtils().update_dirs(self, path=path)
  371. return (cur.lastrowid, True)
  372. def iterate_dircontent(self, path, include_subdirs = True, include_subfiles = True):
  373. self.InternalPathUtils().update_dirs(self)
  374. path = self.InternalPathUtils().normalize_path(path)
  375. cur_head = self.dirs
  376. valid = True
  377. if path != "":
  378. for head in self.InternalPathUtils().iterate_heads(path):
  379. if head not in cur_head.keys():
  380. # non existing directory
  381. valid = False
  382. else:
  383. cur_head = cur_head[head]
  384. basename = os.path.basename(path)
  385. if basename not in cur_head.keys() or cur_head[basename] == None:
  386. # do not exist or points to the file
  387. valid = False
  388. else:
  389. cur_head = cur_head[basename]
  390. if valid == True:
  391. for elem in cur_head.keys():
  392. if include_subdirs == True and cur_head[elem] != None:
  393. yield elem
  394. if include_subfiles == True and cur_head[elem] == None:
  395. yield elem
  396. def check_file(self, path):
  397. return self.get_file(path) != None
  398. def check_dir(self, path):
  399. for each in self.iterate_dircontent(path):
  400. each = each # used
  401. return True # there is at least one item
  402. return False
  403. def get_file(self, path):
  404. path = self.InternalPathUtils().normalize_path(path)
  405. result = self.select_rows("__files__", filters = [("path", "=", path), ("confirmed", "=", 1)])
  406. if len(result) == 0:
  407. return None
  408. assert(len(result) == 1)
  409. return self.FileData(result[0]['id'], result[0]['path'], result[0]['checksum'])
  410. def iterate_files(self, path_like = None):
  411. for row in self.select_rows('__files__', path_like=path_like, filters=[('confirmed','=','1')]):
  412. yield self.FileData(row['id'], row['path'], row['checksum'])
  413. def create_region(self, file_id, region_id, name, begin, end, line_begin, line_end, cursor, group, checksum):
  414. assert(self.read_only == False)
  415. sql = "INSERT OR REPLACE INTO __regions__ (file_id, region_id, name, begin, end, line_begin, line_end, cursor, group_id, checksum) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
  416. column_data = [file_id, region_id, name, begin, end, line_begin, line_end, cursor, group, checksum]
  417. self.log(sql + " /with arguments: " + str(column_data))
  418. cur = self.conn.cursor()
  419. cur.execute(sql, column_data)
  420. return cur.lastrowid
  421. def get_region(self, file_id, region_id):
  422. result = self.select_rows("__regions__", filters = [("file_id", "=", file_id), ("region_id", "=", region_id)])
  423. if len(result) == 0:
  424. return None
  425. return self.RegionData(result[0]['file_id'],
  426. result[0]['region_id'],
  427. result[0]['name'],
  428. result[0]['begin'],
  429. result[0]['end'],
  430. result[0]['line_begin'],
  431. result[0]['line_end'],
  432. result[0]['cursor'],
  433. result[0]['group_id'],
  434. result[0]['checksum'])
  435. def iterate_regions(self, file_id):
  436. for each in self.select_rows("__regions__", filters = [("file_id", "=", file_id)]):
  437. yield self.RegionData(each['file_id'],
  438. each['region_id'],
  439. each['name'],
  440. each['begin'],
  441. each['end'],
  442. each['line_begin'],
  443. each['line_end'],
  444. each['cursor'],
  445. each['group_id'],
  446. each['checksum'])
  447. def create_marker(self, file_id, begin, end, group):
  448. assert(self.read_only == False)
  449. sql = "INSERT OR REPLACE INTO __markers__ (file_id, begin, end, group_id) VALUES (?, ?, ?, ?)"
  450. column_data = [file_id, begin, end, group]
  451. self.log(sql + " /with arguments: " + str(column_data))
  452. cur = self.conn.cursor()
  453. cur.execute(sql, column_data)
  454. return cur.lastrowid
  455. def iterate_markers(self, file_id):
  456. for each in self.select_rows("__markers__", filters = [("file_id", "=", file_id)]):
  457. yield self.MarkerData(each['file_id'],
  458. each['begin'],
  459. each['end'],
  460. each['group_id'])
  461. def add_row(self, table_name, file_id, region_id, array_data):
  462. assert(self.read_only == False)
  463. column_names = "'file_id'"
  464. column_values = "?"
  465. column_data = [file_id]
  466. if region_id != None:
  467. column_names += ", 'region_id'"
  468. column_values += ", ?"
  469. column_data.append(region_id)
  470. useful_data = 0
  471. for each in array_data:
  472. column_names += ", '" + each[0] + "'"
  473. column_values += ", ?"
  474. column_data.append(each[1])
  475. useful_data += 1
  476. if useful_data == 0:
  477. return
  478. sql = "INSERT OR REPLACE INTO '" + table_name + "' (" + column_names + ") VALUES (" + column_values + ")"
  479. self.log(sql + " /with arguments: " + str(column_data))
  480. cur = self.conn.cursor()
  481. cur.execute(sql, column_data)
  482. return cur.lastrowid
  483. def select_rows(self, table_name, path_like = None, column_names = [], filters = []):
  484. safe_column_names = []
  485. for each in column_names:
  486. safe_column_names.append("'" + each + "'")
  487. return self.select_rows_unsafe(table_name, path_like = path_like, column_names = safe_column_names, filters = filters)
  488. def select_rows_unsafe(self, table_name, path_like = None, column_names = [], filters = []):
  489. path_like = self.InternalPathUtils().normalize_path(path_like)
  490. if self.conn == None:
  491. return []
  492. table_stmt = "'" + table_name + "'"
  493. what_stmt = ", ".join(column_names)
  494. if len(what_stmt) == 0:
  495. what_stmt = "*"
  496. elif path_like != None and table_name != '__files__':
  497. what_stmt += ", '__files__'.'path', '__files__'.'id'"
  498. inner_stmt = ""
  499. if path_like != None and table_name != '__files__':
  500. inner_stmt = " INNER JOIN '__files__' ON '__files__'.'id' = '" + table_name + "'.'file_id' "
  501. where_stmt = " "
  502. values = ()
  503. if len(filters) != 0:
  504. if filters[0][1] == 'IN':
  505. where_stmt = " WHERE (`" + filters[0][0] + "` " + filters[0][1] + " " + filters[0][2]
  506. else:
  507. where_stmt = " WHERE (`" + filters[0][0] + "` " + filters[0][1] + " ?"
  508. values = (filters[0][2],)
  509. for each in filters[1:]:
  510. if each[1] == 'IN':
  511. where_stmt += " AND `" + each[0] + "` " + each[1] + " " + each[2]
  512. else:
  513. where_stmt += " AND `" + each[0] + "` " + each[1] + " ?"
  514. values += (each[2], )
  515. if path_like != None:
  516. where_stmt += " AND '__files__'.'path' LIKE ?"
  517. values += (path_like, )
  518. where_stmt += ")"
  519. else:
  520. where_stmt = " WHERE '__files__'.'path' LIKE ?"
  521. values += (path_like, )
  522. sql = "SELECT " + what_stmt + " FROM " + table_stmt + inner_stmt + where_stmt
  523. self.log(sql + " /with arguments: " + str(values))
  524. return self.conn.execute(sql, values).fetchall()
  525. def get_row(self, table_name, file_id, region_id):
  526. selected = self.get_rows(table_name, file_id, region_id)
  527. # assures that only one row in database
  528. # if assertion happens, caller's intention is not right, use get_rows instead
  529. assert(len(selected) == 0 or len(selected) == 1)
  530. if len(selected) == 0:
  531. return None
  532. return selected[0]
  533. def get_rows(self, table_name, file_id, region_id):
  534. filters = [("file_id", '=', file_id)]
  535. if region_id != None:
  536. filters.append(("region_id", '=', region_id))
  537. return self.select_rows(table_name, filters=filters)
  538. def aggregate_rows(self, table_name, path_like = None, column_names = None, filters = []):
  539. if column_names == None:
  540. column_names = []
  541. for column in self.iterate_columns(table_name):
  542. column_names.append(column.name)
  543. if len(column_names) == 0:
  544. # it is possible that a table does not have meanfull columns
  545. return {}
  546. total_column_names = []
  547. for column_name in column_names:
  548. for func in ['max', 'min', 'avg', 'total']:
  549. total_column_names.append(func + "('" + table_name + "'.'" + column_name + "') AS " + "'" + column_name + "_" + func + "'")
  550. data = self.select_rows_unsafe(table_name, path_like = path_like, column_names = total_column_names, filters = filters)
  551. assert(len(data) == 1)
  552. result = {}
  553. for column_name in column_names:
  554. result[column_name] = {}
  555. for func in ['max', 'min', 'avg', 'total']:
  556. result[column_name][func] = data[0][column_name + "_" + func]
  557. return result
  558. def log(self, sql):
  559. #import traceback
  560. #traceback.print_stack()
  561. logging.debug("[" + str(self.id) + "] Executing query: " + sql)