add rebuild indexes option to Database menu (can we put tooltips over these menu options?)
This commit is contained in:
sqlcoder
parent
8221e86757
commit
00c85aa62d
|
|
@ -21,9 +21,9 @@ Create and manage the database objects.
|
|||
|
||||
########################################################################
|
||||
|
||||
# ToDo: - vacuum for mysql
|
||||
# - rebuild indexes / vacuum option
|
||||
# - check speed of get_stats_from_hand()
|
||||
# ToDo: - rebuild indexes / vacuum option
|
||||
# - check speed of get_stats_from_hand() - add log info
|
||||
# - check size of db, seems big? (mysql)
|
||||
# - investigate size of mysql db (200K for just 7K hands? 2GB for 140K hands?)
|
||||
|
||||
# postmaster -D /var/lib/pgsql/data
|
||||
|
|
@ -74,14 +74,9 @@ class Database:
|
|||
indexes = [
|
||||
[ ] # no db with index 0
|
||||
, [ ] # no db with index 1
|
||||
, [ # indexes for mysql (list index 2)
|
||||
, [ # indexes for mysql (list index 2) (foreign keys not here, in next data structure)
|
||||
# {'tab':'Players', 'col':'name', 'drop':0} unique indexes not dropped
|
||||
# {'tab':'Hands', 'col':'siteHandNo', 'drop':0} unique indexes not dropped
|
||||
{'tab':'Hands', 'col':'gametypeId', 'drop':0} # mct 22/3/09
|
||||
, {'tab':'HandsPlayers', 'col':'handId', 'drop':0} # not needed, handled by fk
|
||||
, {'tab':'HandsPlayers', 'col':'playerId', 'drop':0} # not needed, handled by fk
|
||||
, {'tab':'HandsPlayers', 'col':'tourneyTypeId', 'drop':0}
|
||||
, {'tab':'HandsPlayers', 'col':'tourneysPlayersId', 'drop':0}
|
||||
#, {'tab':'Tourneys', 'col':'siteTourneyNo', 'drop':0} unique indexes not dropped
|
||||
]
|
||||
, [ # indexes for postgres (list index 3)
|
||||
|
|
@ -122,6 +117,8 @@ class Database:
|
|||
{'fktab':'Hands', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1}
|
||||
, {'fktab':'HandsPlayers', 'fkcol':'handId', 'rtab':'Hands', 'rcol':'id', 'drop':1}
|
||||
, {'fktab':'HandsPlayers', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':1}
|
||||
, {'fktab':'HandsPlayers', 'fkcol':'tourneyTypeId', 'rtab':'TourneyTypes', 'rcol':'id', 'drop':1}
|
||||
, {'fktab':'HandsPlayers', 'fkcol':'tourneysPlayersId','rtab':'TourneysPlayers','rcol':'id', 'drop':1}
|
||||
, {'fktab':'HandsActions', 'fkcol':'handsPlayerId', 'rtab':'HandsPlayers', 'rcol':'id', 'drop':1}
|
||||
, {'fktab':'HudCache', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1}
|
||||
, {'fktab':'HudCache', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':0}
|
||||
|
|
@ -1171,9 +1168,9 @@ class Database:
|
|||
print "dropping mysql index ", idx['tab'], idx['col']
|
||||
try:
|
||||
self.get_cursor().execute( "alter table %s drop index %s"
|
||||
, (idx['tab'],idx['col']) )
|
||||
, (idx['tab'], idx['col']) )
|
||||
except:
|
||||
pass
|
||||
print " drop idx failed: " + str(sys.exc_info())
|
||||
elif self.backend == self.PGSQL:
|
||||
print "dropping pg index ", idx['tab'], idx['col']
|
||||
# mod to use tab_col for index name?
|
||||
|
|
@ -1181,13 +1178,119 @@ class Database:
|
|||
self.get_cursor().execute( "drop index %s_%s_idx"
|
||||
% (idx['tab'],idx['col']) )
|
||||
except:
|
||||
pass
|
||||
print " drop idx failed: " + str(sys.exc_info())
|
||||
else:
|
||||
print "Only MySQL and Postgres supported so far"
|
||||
return -1
|
||||
if self.backend == self.PGSQL:
|
||||
self.connection.set_isolation_level(1) # go back to normal isolation level
|
||||
#end def dropAllIndexes
|
||||
|
||||
def createAllForeignKeys(self):
|
||||
"""Create foreign keys"""
|
||||
|
||||
try:
|
||||
if self.backend == self.PGSQL:
|
||||
self.connection.set_isolation_level(0) # allow table/index operations to work
|
||||
c = self.get_cursor()
|
||||
except:
|
||||
print " set_isolation_level failed: " + str(sys.exc_info())
|
||||
|
||||
for fk in self.foreignKeys[self.backend]:
|
||||
if self.backend == self.MYSQL_INNODB:
|
||||
c.execute("SELECT constraint_name " +
|
||||
"FROM information_schema.KEY_COLUMN_USAGE " +
|
||||
#"WHERE REFERENCED_TABLE_SCHEMA = 'fpdb'
|
||||
"WHERE 1=1 " +
|
||||
"AND table_name = %s AND column_name = %s " +
|
||||
"AND referenced_table_name = %s " +
|
||||
"AND referenced_column_name = %s ",
|
||||
(fk['fktab'], fk['fkcol'], fk['rtab'], fk['rcol']) )
|
||||
cons = c.fetchone()
|
||||
#print "afterbulk: cons=", cons
|
||||
if cons:
|
||||
pass
|
||||
else:
|
||||
print "creating fk ", fk['fktab'], fk['fkcol'], "->", fk['rtab'], fk['rcol']
|
||||
try:
|
||||
c.execute("alter table " + fk['fktab'] + " add foreign key ("
|
||||
+ fk['fkcol'] + ") references " + fk['rtab'] + "("
|
||||
+ fk['rcol'] + ")")
|
||||
except:
|
||||
print " create fk failed: " + str(sys.exc_info())
|
||||
elif self.backend == self.PGSQL:
|
||||
print "creating fk ", fk['fktab'], fk['fkcol'], "->", fk['rtab'], fk['rcol']
|
||||
try:
|
||||
c.execute("alter table " + fk['fktab'] + " add constraint "
|
||||
+ fk['fktab'] + '_' + fk['fkcol'] + '_fkey'
|
||||
+ " foreign key (" + fk['fkcol']
|
||||
+ ") references " + fk['rtab'] + "(" + fk['rcol'] + ")")
|
||||
except:
|
||||
print " create fk failed: " + str(sys.exc_info())
|
||||
else:
|
||||
print "Only MySQL and Postgres supported so far"
|
||||
|
||||
try:
|
||||
if self.backend == self.PGSQL:
|
||||
self.connection.set_isolation_level(1) # go back to normal isolation level
|
||||
except:
|
||||
print " set_isolation_level failed: " + str(sys.exc_info())
|
||||
#end def createAllForeignKeys
|
||||
|
||||
def dropAllForeignKeys(self):
|
||||
"""Drop all standalone indexes (i.e. not including primary keys or foreign keys)
|
||||
using list of indexes in indexes data structure"""
|
||||
# maybe upgrade to use data dictionary?? (but take care to exclude PK and FK)
|
||||
if self.backend == self.PGSQL:
|
||||
self.connection.set_isolation_level(0) # allow table/index operations to work
|
||||
c = self.get_cursor()
|
||||
|
||||
for fk in self.foreignKeys[self.backend]:
|
||||
if self.backend == self.MYSQL_INNODB:
|
||||
c.execute("SELECT constraint_name " +
|
||||
"FROM information_schema.KEY_COLUMN_USAGE " +
|
||||
#"WHERE REFERENCED_TABLE_SCHEMA = 'fpdb'
|
||||
"WHERE 1=1 " +
|
||||
"AND table_name = %s AND column_name = %s " +
|
||||
"AND referenced_table_name = %s " +
|
||||
"AND referenced_column_name = %s ",
|
||||
(fk['fktab'], fk['fkcol'], fk['rtab'], fk['rcol']) )
|
||||
cons = c.fetchone()
|
||||
#print "preparebulk find fk: cons=", cons
|
||||
if cons:
|
||||
print "dropping mysql fk", cons[0], fk['fktab'], fk['fkcol']
|
||||
try:
|
||||
c.execute("alter table " + fk['fktab'] + " drop foreign key " + cons[0])
|
||||
except:
|
||||
print " drop failed: " + str(sys.exc_info())
|
||||
elif self.backend == self.PGSQL:
|
||||
# DON'T FORGET TO RECREATE THEM!!
|
||||
print "dropping pg fk", fk['fktab'], fk['fkcol']
|
||||
try:
|
||||
# try to lock table to see if index drop will work:
|
||||
# hmmm, tested by commenting out rollback in grapher. lock seems to work but
|
||||
# then drop still hangs :-( does work in some tests though??
|
||||
# will leave code here for now pending further tests/enhancement ...
|
||||
c.execute( "lock table %s in exclusive mode nowait" % (fk['fktab'],) )
|
||||
#print "after lock, status:", c.statusmessage
|
||||
#print "alter table %s drop constraint %s_%s_fkey" % (fk['fktab'], fk['fktab'], fk['fkcol'])
|
||||
try:
|
||||
c.execute("alter table %s drop constraint %s_%s_fkey" % (fk['fktab'], fk['fktab'], fk['fkcol']))
|
||||
print "dropped pg fk pg fk %s_%s_fkey, continuing ..." % (fk['fktab'], fk['fkcol'])
|
||||
except:
|
||||
if "does not exist" not in str(sys.exc_value):
|
||||
print "warning: drop pg fk %s_%s_fkey failed: %s, continuing ..." \
|
||||
% (fk['fktab'], fk['fkcol'], str(sys.exc_value).rstrip('\n') )
|
||||
except:
|
||||
print "warning: constraint %s_%s_fkey not dropped: %s, continuing ..." \
|
||||
% (fk['fktab'],fk['fkcol'], str(sys.exc_value).rstrip('\n'))
|
||||
else:
|
||||
print "Only MySQL and Postgres supported so far"
|
||||
|
||||
if self.backend == self.PGSQL:
|
||||
self.connection.set_isolation_level(1) # go back to normal isolation level
|
||||
#end def dropAllForeignKeys
|
||||
|
||||
|
||||
def fillDefaultData(self):
|
||||
c = self.get_cursor()
|
||||
|
|
@ -1215,6 +1318,12 @@ class Database:
|
|||
|
||||
#end def fillDefaultData
|
||||
|
||||
def rebuild_indexes(self, start=None):
|
||||
self.dropAllIndexes()
|
||||
self.createAllIndexes()
|
||||
self.dropAllForeignKeys()
|
||||
self.createAllForeignKeys()
|
||||
|
||||
def rebuild_hudcache(self, start=None):
|
||||
"""clears hudcache and rebuilds from the individual handsplayers records"""
|
||||
|
||||
|
|
@ -1297,7 +1406,7 @@ class Database:
|
|||
except:
|
||||
print "Error during analyze:", str(sys.exc_value)
|
||||
elif self.backend == self.PGSQL:
|
||||
self.connection.set_isolation_level(0) # allow vacuum to work
|
||||
self.connection.set_isolation_level(0) # allow analyze to work
|
||||
try:
|
||||
self.get_cursor().execute(self.sql.query['analyze'])
|
||||
except:
|
||||
|
|
@ -1308,6 +1417,25 @@ class Database:
|
|||
print "Analyze took %.1f seconds" % (atime,)
|
||||
#end def analyzeDB
|
||||
|
||||
def vacuumDB(self):
|
||||
"""Do whatever the DB can offer to update index/table statistics"""
|
||||
stime = time()
|
||||
if self.backend == self.MYSQL_INNODB:
|
||||
try:
|
||||
self.get_cursor().execute(self.sql.query['vacuum'])
|
||||
except:
|
||||
print "Error during vacuum:", str(sys.exc_value)
|
||||
elif self.backend == self.PGSQL:
|
||||
self.connection.set_isolation_level(0) # allow vacuum to work
|
||||
try:
|
||||
self.get_cursor().execute(self.sql.query['vacuum'])
|
||||
except:
|
||||
print "Error during vacuum:", str(sys.exc_value)
|
||||
self.connection.set_isolation_level(1) # go back to normal isolation level
|
||||
self.commit()
|
||||
atime = time() - stime
|
||||
print "Vacuum took %.1f seconds" % (atime,)
|
||||
#end def analyzeDB
|
||||
|
||||
# Start of Hand Writing routines. Idea is to provide a mixture of routines to store Hand data
|
||||
# however the calling prog requires. Main aims:
|
||||
|
|
@ -1389,7 +1517,7 @@ class Database:
|
|||
|
||||
q = q.replace('%s', self.sql.query['placeholder'])
|
||||
|
||||
c = self.connection.cursor()
|
||||
c = self.get_cursor()
|
||||
|
||||
c.execute(q, (
|
||||
p['tableName'],
|
||||
|
|
@ -1575,7 +1703,7 @@ class Database:
|
|||
|
||||
#print "DEBUG: inserts: %s" %inserts
|
||||
#print "DEBUG: q: %s" % q
|
||||
c = self.connection.cursor()
|
||||
c = self.get_cursor()
|
||||
c.executemany(q, inserts)
|
||||
|
||||
def storeHudCacheNew(self, gid, pid, hc):
|
||||
|
|
|
|||
|
|
@ -461,7 +461,7 @@ class Sql:
|
|||
totalProfit INT,
|
||||
comment text,
|
||||
commentTs DATETIME,
|
||||
tourneysPlayersId BIGINT UNSIGNED,
|
||||
tourneysPlayersId BIGINT UNSIGNED, FOREIGN KEY (tourneysPlayersId) REFERENCES TourneysPlayers(id),
|
||||
tourneyTypeId SMALLINT UNSIGNED NOT NULL DEFAULT 1, FOREIGN KEY (tourneyTypeId) REFERENCES TourneyTypes(id),
|
||||
|
||||
wonWhenSeenStreet1 FLOAT,
|
||||
|
|
@ -551,9 +551,7 @@ class Sql:
|
|||
street3Raises TINYINT,
|
||||
street4Raises TINYINT,
|
||||
|
||||
actionString VARCHAR(15),
|
||||
|
||||
FOREIGN KEY (tourneysPlayersId) REFERENCES TourneysPlayers(id))
|
||||
actionString VARCHAR(15))
|
||||
ENGINE=INNODB"""
|
||||
elif db_server == 'postgresql':
|
||||
self.query['createHandsPlayersTable'] = """CREATE TABLE HandsPlayers (
|
||||
|
|
@ -3100,8 +3098,10 @@ class Sql:
|
|||
analyze table Autorates, GameTypes, Hands, HandsPlayers, HudCache, Players
|
||||
, Settings, Sites, Tourneys, TourneysPlayers, TourneyTypes
|
||||
"""
|
||||
else: # assume postgres
|
||||
self.query['analyze'] = "vacuum analyze"
|
||||
elif db_server == 'postgresql':
|
||||
self.query['analyze'] = "analyze"
|
||||
elif db_server == 'sqlite':
|
||||
self.query['analyze'] = "analyze"
|
||||
|
||||
if db_server == 'mysql':
|
||||
self.query['lockForInsert'] = """
|
||||
|
|
@ -3109,8 +3109,20 @@ class Sql:
|
|||
, HudCache write, GameTypes write, Sites write, Tourneys write
|
||||
, TourneysPlayers write, TourneyTypes write, Autorates write
|
||||
"""
|
||||
else: # assume postgres
|
||||
elif db_server == 'postgresql':
|
||||
self.query['lockForInsert'] = ""
|
||||
elif db_server == 'sqlite':
|
||||
self.query['lockForInsert'] = ""
|
||||
|
||||
if db_server == 'mysql':
|
||||
self.query['vacuum'] = """optimize table Hands, HandsPlayers, HandsActions, Players
|
||||
, HudCache, GameTypes, Sites, Tourneys
|
||||
, TourneysPlayers, TourneyTypes, Autorates
|
||||
"""
|
||||
elif db_server == 'postgresql':
|
||||
self.query['vacuum'] = """ vacuum """
|
||||
elif db_server == 'sqlite':
|
||||
self.query['vacuum'] = """ vacuum """
|
||||
|
||||
self.query['getGametypeFL'] = """SELECT id
|
||||
FROM Gametypes
|
||||
|
|
|
|||
|
|
@ -357,6 +357,27 @@ class fpdb:
|
|||
|
||||
self.release_global_lock()
|
||||
|
||||
def dia_rebuild_indexes(self, widget, data=None):
|
||||
if self.obtain_global_lock():
|
||||
self.dia_confirm = gtk.MessageDialog(parent=None
|
||||
,flags=0
|
||||
,type=gtk.MESSAGE_WARNING
|
||||
,buttons=(gtk.BUTTONS_YES_NO)
|
||||
,message_format="Confirm rebuilding database indexes")
|
||||
diastring = "Please confirm that you want to rebuild the database indexes."
|
||||
self.dia_confirm.format_secondary_text(diastring)
|
||||
|
||||
response = self.dia_confirm.run()
|
||||
self.dia_confirm.destroy()
|
||||
if response == gtk.RESPONSE_YES:
|
||||
self.db.rebuild_indexes()
|
||||
self.db.vacuumDB()
|
||||
self.db.analyzeDB()
|
||||
elif response == gtk.RESPONSE_NO:
|
||||
print 'User cancelled rebuilding db indexes'
|
||||
|
||||
self.release_global_lock()
|
||||
|
||||
def __calendar_dialog(self, widget, entry):
|
||||
self.dia_confirm.set_modal(False)
|
||||
d = gtk.Window(gtk.WINDOW_TOPLEVEL)
|
||||
|
|
@ -451,6 +472,7 @@ class fpdb:
|
|||
<menuitem action="createuser"/>
|
||||
<menuitem action="createtabs"/>
|
||||
<menuitem action="rebuildhudcache"/>
|
||||
<menuitem action="rebuildindexes"/>
|
||||
<menuitem action="stats"/>
|
||||
</menu>
|
||||
<menu action="help">
|
||||
|
|
@ -492,6 +514,7 @@ class fpdb:
|
|||
('createuser', None, 'Create or Delete _User (todo)', None, 'Create or Delete User', self.dia_create_del_user),
|
||||
('createtabs', None, 'Create or Recreate _Tables', None, 'Create or Recreate Tables ', self.dia_recreate_tables),
|
||||
('rebuildhudcache', None, 'Rebuild HUD Cache', None, 'Rebuild HUD Cache', self.dia_recreate_hudcache),
|
||||
('rebuildindexes', None, 'Rebuild DB Indexes', None, 'Rebuild DB Indexes', self.dia_rebuild_indexes),
|
||||
('stats', None, '_Statistics (todo)', None, 'View Database Statistics', self.dia_database_stats),
|
||||
('help', None, '_Help'),
|
||||
('Abbrev', None, '_Abbrevations (todo)', None, 'List of Abbrevations', self.tab_abbreviations),
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user