Merge branch 'master' of git://git.assembla.com/fpdb-sql.git

This commit is contained in:
Eric Blade 2009-11-29 22:02:49 -05:00
commit 3439523de8
4 changed files with 192 additions and 23 deletions

View File

@ -688,7 +688,7 @@ class Config:
except: hui['agg_bb_mult'] = 1 except: hui['agg_bb_mult'] = 1
try: hui['seats_style'] = self.ui.seats_style try: hui['seats_style'] = self.ui.seats_style
except: hui['seats_style'] = 'C' # A / C / E, use A(ll) / C(ustom) / E(xact) seat numbers except: hui['seats_style'] = 'A' # A / C / E, use A(ll) / C(ustom) / E(xact) seat numbers
try: hui['seats_cust_nums'] = self.ui.seats_cust_nums try: hui['seats_cust_nums'] = self.ui.seats_cust_nums
except: hui['seats_cust_nums'] = ['n/a', 'n/a', (2,2), (3,4), (3,5), (4,6), (5,7), (6,8), (7,9), (8,10), (8,10)] except: hui['seats_cust_nums'] = ['n/a', 'n/a', (2,2), (3,4), (3,5), (4,6), (5,7), (6,8), (7,9), (8,10), (8,10)]
@ -711,7 +711,7 @@ class Config:
except: hui['h_agg_bb_mult'] = 1 except: hui['h_agg_bb_mult'] = 1
try: hui['h_seats_style'] = self.ui.h_seats_style try: hui['h_seats_style'] = self.ui.h_seats_style
except: hui['h_seats_style'] = 'E' # A / C / E, use A(ll) / C(ustom) / E(xact) seat numbers except: hui['h_seats_style'] = 'A' # A / C / E, use A(ll) / C(ustom) / E(xact) seat numbers
try: hui['h_seats_cust_nums'] = self.ui.h_seats_cust_nums try: hui['h_seats_cust_nums'] = self.ui.h_seats_cust_nums
except: hui['h_seats_cust_nums'] = ['n/a', 'n/a', (2,2), (3,4), (3,5), (4,6), (5,7), (6,8), (7,9), (8,10), (8,10)] except: hui['h_seats_cust_nums'] = ['n/a', 'n/a', (2,2), (3,4), (3,5), (4,6), (5,7), (6,8), (7,9), (8,10), (8,10)]

View File

@ -21,6 +21,11 @@ Create and manage the database objects.
######################################################################## ########################################################################
# ToDo: - rebuild indexes / vacuum option
# - check speed of get_stats_from_hand() - add log info
# - check size of db, seems big? (mysql)
# - investigate size of mysql db (200K for just 7K hands? 2GB for 140K hands?)
# postmaster -D /var/lib/pgsql/data # postmaster -D /var/lib/pgsql/data
# Standard Library modules # Standard Library modules
@ -69,14 +74,9 @@ class Database:
indexes = [ indexes = [
[ ] # no db with index 0 [ ] # no db with index 0
, [ ] # no db with index 1 , [ ] # no db with index 1
, [ # indexes for mysql (list index 2) , [ # indexes for mysql (list index 2) (foreign keys not here, in next data structure)
# {'tab':'Players', 'col':'name', 'drop':0} unique indexes not dropped # {'tab':'Players', 'col':'name', 'drop':0} unique indexes not dropped
# {'tab':'Hands', 'col':'siteHandNo', 'drop':0} unique indexes not dropped # {'tab':'Hands', 'col':'siteHandNo', 'drop':0} unique indexes not dropped
{'tab':'Hands', 'col':'gametypeId', 'drop':0} # mct 22/3/09
, {'tab':'HandsPlayers', 'col':'handId', 'drop':0} # not needed, handled by fk
, {'tab':'HandsPlayers', 'col':'playerId', 'drop':0} # not needed, handled by fk
, {'tab':'HandsPlayers', 'col':'tourneyTypeId', 'drop':0}
, {'tab':'HandsPlayers', 'col':'tourneysPlayersId', 'drop':0}
#, {'tab':'Tourneys', 'col':'siteTourneyNo', 'drop':0} unique indexes not dropped #, {'tab':'Tourneys', 'col':'siteTourneyNo', 'drop':0} unique indexes not dropped
] ]
, [ # indexes for postgres (list index 3) , [ # indexes for postgres (list index 3)
@ -117,6 +117,8 @@ class Database:
{'fktab':'Hands', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} {'fktab':'Hands', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1}
, {'fktab':'HandsPlayers', 'fkcol':'handId', 'rtab':'Hands', 'rcol':'id', 'drop':1} , {'fktab':'HandsPlayers', 'fkcol':'handId', 'rtab':'Hands', 'rcol':'id', 'drop':1}
, {'fktab':'HandsPlayers', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':1} , {'fktab':'HandsPlayers', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':1}
, {'fktab':'HandsPlayers', 'fkcol':'tourneyTypeId', 'rtab':'TourneyTypes', 'rcol':'id', 'drop':1}
, {'fktab':'HandsPlayers', 'fkcol':'tourneysPlayersId','rtab':'TourneysPlayers','rcol':'id', 'drop':1}
, {'fktab':'HandsActions', 'fkcol':'handsPlayerId', 'rtab':'HandsPlayers', 'rcol':'id', 'drop':1} , {'fktab':'HandsActions', 'fkcol':'handsPlayerId', 'rtab':'HandsPlayers', 'rcol':'id', 'drop':1}
, {'fktab':'HudCache', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} , {'fktab':'HudCache', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1}
, {'fktab':'HudCache', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':0} , {'fktab':'HudCache', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':0}
@ -431,6 +433,7 @@ class Database:
err = traceback.extract_tb(sys.exc_info()[2])[-1] err = traceback.extract_tb(sys.exc_info()[2])[-1]
print "*** Database Error: "+err[2]+"("+str(err[1])+"): "+str(sys.exc_info()[1]) print "*** Database Error: "+err[2]+"("+str(err[1])+"): "+str(sys.exc_info()[1])
# is get_stats_from_hand slow?
def get_stats_from_hand( self, hand, type # type is "ring" or "tour" def get_stats_from_hand( self, hand, type # type is "ring" or "tour"
, hud_params = {'hud_style':'A', 'agg_bb_mult':1000 , hud_params = {'hud_style':'A', 'agg_bb_mult':1000
,'seats_style':'A', 'seats_cust_nums':['n/a', 'n/a', (2,2), (3,4), (3,5), (4,6), (5,7), (6,8), (7,9), (8,10), (8,10)] ,'seats_style':'A', 'seats_cust_nums':['n/a', 'n/a', (2,2), (3,4), (3,5), (4,6), (5,7), (6,8), (7,9), (8,10), (8,10)]
@ -1165,9 +1168,9 @@ class Database:
print "dropping mysql index ", idx['tab'], idx['col'] print "dropping mysql index ", idx['tab'], idx['col']
try: try:
self.get_cursor().execute( "alter table %s drop index %s" self.get_cursor().execute( "alter table %s drop index %s"
, (idx['tab'],idx['col']) ) , (idx['tab'], idx['col']) )
except: except:
pass print " drop idx failed: " + str(sys.exc_info())
elif self.backend == self.PGSQL: elif self.backend == self.PGSQL:
print "dropping pg index ", idx['tab'], idx['col'] print "dropping pg index ", idx['tab'], idx['col']
# mod to use tab_col for index name? # mod to use tab_col for index name?
@ -1175,13 +1178,119 @@ class Database:
self.get_cursor().execute( "drop index %s_%s_idx" self.get_cursor().execute( "drop index %s_%s_idx"
% (idx['tab'],idx['col']) ) % (idx['tab'],idx['col']) )
except: except:
pass print " drop idx failed: " + str(sys.exc_info())
else: else:
print "Only MySQL and Postgres supported so far" print "Only MySQL and Postgres supported so far"
return -1 return -1
if self.backend == self.PGSQL: if self.backend == self.PGSQL:
self.connection.set_isolation_level(1) # go back to normal isolation level self.connection.set_isolation_level(1) # go back to normal isolation level
#end def dropAllIndexes #end def dropAllIndexes
def createAllForeignKeys(self):
"""Create foreign keys"""
try:
if self.backend == self.PGSQL:
self.connection.set_isolation_level(0) # allow table/index operations to work
c = self.get_cursor()
except:
print " set_isolation_level failed: " + str(sys.exc_info())
for fk in self.foreignKeys[self.backend]:
if self.backend == self.MYSQL_INNODB:
c.execute("SELECT constraint_name " +
"FROM information_schema.KEY_COLUMN_USAGE " +
#"WHERE REFERENCED_TABLE_SCHEMA = 'fpdb'
"WHERE 1=1 " +
"AND table_name = %s AND column_name = %s " +
"AND referenced_table_name = %s " +
"AND referenced_column_name = %s ",
(fk['fktab'], fk['fkcol'], fk['rtab'], fk['rcol']) )
cons = c.fetchone()
#print "afterbulk: cons=", cons
if cons:
pass
else:
print "creating fk ", fk['fktab'], fk['fkcol'], "->", fk['rtab'], fk['rcol']
try:
c.execute("alter table " + fk['fktab'] + " add foreign key ("
+ fk['fkcol'] + ") references " + fk['rtab'] + "("
+ fk['rcol'] + ")")
except:
print " create fk failed: " + str(sys.exc_info())
elif self.backend == self.PGSQL:
print "creating fk ", fk['fktab'], fk['fkcol'], "->", fk['rtab'], fk['rcol']
try:
c.execute("alter table " + fk['fktab'] + " add constraint "
+ fk['fktab'] + '_' + fk['fkcol'] + '_fkey'
+ " foreign key (" + fk['fkcol']
+ ") references " + fk['rtab'] + "(" + fk['rcol'] + ")")
except:
print " create fk failed: " + str(sys.exc_info())
else:
print "Only MySQL and Postgres supported so far"
try:
if self.backend == self.PGSQL:
self.connection.set_isolation_level(1) # go back to normal isolation level
except:
print " set_isolation_level failed: " + str(sys.exc_info())
#end def createAllForeignKeys
def dropAllForeignKeys(self):
"""Drop all standalone indexes (i.e. not including primary keys or foreign keys)
using list of indexes in indexes data structure"""
# maybe upgrade to use data dictionary?? (but take care to exclude PK and FK)
if self.backend == self.PGSQL:
self.connection.set_isolation_level(0) # allow table/index operations to work
c = self.get_cursor()
for fk in self.foreignKeys[self.backend]:
if self.backend == self.MYSQL_INNODB:
c.execute("SELECT constraint_name " +
"FROM information_schema.KEY_COLUMN_USAGE " +
#"WHERE REFERENCED_TABLE_SCHEMA = 'fpdb'
"WHERE 1=1 " +
"AND table_name = %s AND column_name = %s " +
"AND referenced_table_name = %s " +
"AND referenced_column_name = %s ",
(fk['fktab'], fk['fkcol'], fk['rtab'], fk['rcol']) )
cons = c.fetchone()
#print "preparebulk find fk: cons=", cons
if cons:
print "dropping mysql fk", cons[0], fk['fktab'], fk['fkcol']
try:
c.execute("alter table " + fk['fktab'] + " drop foreign key " + cons[0])
except:
print " drop failed: " + str(sys.exc_info())
elif self.backend == self.PGSQL:
# DON'T FORGET TO RECREATE THEM!!
print "dropping pg fk", fk['fktab'], fk['fkcol']
try:
# try to lock table to see if index drop will work:
# hmmm, tested by commenting out rollback in grapher. lock seems to work but
# then drop still hangs :-( does work in some tests though??
# will leave code here for now pending further tests/enhancement ...
c.execute( "lock table %s in exclusive mode nowait" % (fk['fktab'],) )
#print "after lock, status:", c.statusmessage
#print "alter table %s drop constraint %s_%s_fkey" % (fk['fktab'], fk['fktab'], fk['fkcol'])
try:
c.execute("alter table %s drop constraint %s_%s_fkey" % (fk['fktab'], fk['fktab'], fk['fkcol']))
print "dropped pg fk pg fk %s_%s_fkey, continuing ..." % (fk['fktab'], fk['fkcol'])
except:
if "does not exist" not in str(sys.exc_value):
print "warning: drop pg fk %s_%s_fkey failed: %s, continuing ..." \
% (fk['fktab'], fk['fkcol'], str(sys.exc_value).rstrip('\n') )
except:
print "warning: constraint %s_%s_fkey not dropped: %s, continuing ..." \
% (fk['fktab'],fk['fkcol'], str(sys.exc_value).rstrip('\n'))
else:
print "Only MySQL and Postgres supported so far"
if self.backend == self.PGSQL:
self.connection.set_isolation_level(1) # go back to normal isolation level
#end def dropAllForeignKeys
def fillDefaultData(self): def fillDefaultData(self):
c = self.get_cursor() c = self.get_cursor()
@ -1209,6 +1318,12 @@ class Database:
#end def fillDefaultData #end def fillDefaultData
def rebuild_indexes(self, start=None):
self.dropAllIndexes()
self.createAllIndexes()
self.dropAllForeignKeys()
self.createAllForeignKeys()
def rebuild_hudcache(self, start=None): def rebuild_hudcache(self, start=None):
"""clears hudcache and rebuilds from the individual handsplayers records""" """clears hudcache and rebuilds from the individual handsplayers records"""
@ -1291,7 +1406,7 @@ class Database:
except: except:
print "Error during analyze:", str(sys.exc_value) print "Error during analyze:", str(sys.exc_value)
elif self.backend == self.PGSQL: elif self.backend == self.PGSQL:
self.connection.set_isolation_level(0) # allow vacuum to work self.connection.set_isolation_level(0) # allow analyze to work
try: try:
self.get_cursor().execute(self.sql.query['analyze']) self.get_cursor().execute(self.sql.query['analyze'])
except: except:
@ -1302,6 +1417,25 @@ class Database:
print "Analyze took %.1f seconds" % (atime,) print "Analyze took %.1f seconds" % (atime,)
#end def analyzeDB #end def analyzeDB
def vacuumDB(self):
"""Do whatever the DB can offer to update index/table statistics"""
stime = time()
if self.backend == self.MYSQL_INNODB:
try:
self.get_cursor().execute(self.sql.query['vacuum'])
except:
print "Error during vacuum:", str(sys.exc_value)
elif self.backend == self.PGSQL:
self.connection.set_isolation_level(0) # allow vacuum to work
try:
self.get_cursor().execute(self.sql.query['vacuum'])
except:
print "Error during vacuum:", str(sys.exc_value)
self.connection.set_isolation_level(1) # go back to normal isolation level
self.commit()
atime = time() - stime
print "Vacuum took %.1f seconds" % (atime,)
#end def analyzeDB
# Start of Hand Writing routines. Idea is to provide a mixture of routines to store Hand data # Start of Hand Writing routines. Idea is to provide a mixture of routines to store Hand data
# however the calling prog requires. Main aims: # however the calling prog requires. Main aims:
@ -1383,7 +1517,7 @@ class Database:
q = q.replace('%s', self.sql.query['placeholder']) q = q.replace('%s', self.sql.query['placeholder'])
c = self.connection.cursor() c = self.get_cursor()
c.execute(q, ( c.execute(q, (
p['tableName'], p['tableName'],
@ -1569,7 +1703,7 @@ class Database:
#print "DEBUG: inserts: %s" %inserts #print "DEBUG: inserts: %s" %inserts
#print "DEBUG: q: %s" % q #print "DEBUG: q: %s" % q
c = self.connection.cursor() c = self.get_cursor()
c.executemany(q, inserts) c.executemany(q, inserts)
def storeHudCacheNew(self, gid, pid, hc): def storeHudCacheNew(self, gid, pid, hc):

View File

@ -461,7 +461,7 @@ class Sql:
totalProfit INT, totalProfit INT,
comment text, comment text,
commentTs DATETIME, commentTs DATETIME,
tourneysPlayersId BIGINT UNSIGNED, tourneysPlayersId BIGINT UNSIGNED, FOREIGN KEY (tourneysPlayersId) REFERENCES TourneysPlayers(id),
tourneyTypeId SMALLINT UNSIGNED NOT NULL DEFAULT 1, FOREIGN KEY (tourneyTypeId) REFERENCES TourneyTypes(id), tourneyTypeId SMALLINT UNSIGNED NOT NULL DEFAULT 1, FOREIGN KEY (tourneyTypeId) REFERENCES TourneyTypes(id),
wonWhenSeenStreet1 FLOAT, wonWhenSeenStreet1 FLOAT,
@ -551,9 +551,7 @@ class Sql:
street3Raises TINYINT, street3Raises TINYINT,
street4Raises TINYINT, street4Raises TINYINT,
actionString VARCHAR(15), actionString VARCHAR(15))
FOREIGN KEY (tourneysPlayersId) REFERENCES TourneysPlayers(id))
ENGINE=INNODB""" ENGINE=INNODB"""
elif db_server == 'postgresql': elif db_server == 'postgresql':
self.query['createHandsPlayersTable'] = """CREATE TABLE HandsPlayers ( self.query['createHandsPlayersTable'] = """CREATE TABLE HandsPlayers (
@ -3100,8 +3098,10 @@ class Sql:
analyze table Autorates, GameTypes, Hands, HandsPlayers, HudCache, Players analyze table Autorates, GameTypes, Hands, HandsPlayers, HudCache, Players
, Settings, Sites, Tourneys, TourneysPlayers, TourneyTypes , Settings, Sites, Tourneys, TourneysPlayers, TourneyTypes
""" """
else: # assume postgres elif db_server == 'postgresql':
self.query['analyze'] = "vacuum analyze" self.query['analyze'] = "analyze"
elif db_server == 'sqlite':
self.query['analyze'] = "analyze"
if db_server == 'mysql': if db_server == 'mysql':
self.query['lockForInsert'] = """ self.query['lockForInsert'] = """
@ -3109,8 +3109,20 @@ class Sql:
, HudCache write, GameTypes write, Sites write, Tourneys write , HudCache write, GameTypes write, Sites write, Tourneys write
, TourneysPlayers write, TourneyTypes write, Autorates write , TourneysPlayers write, TourneyTypes write, Autorates write
""" """
else: # assume postgres elif db_server == 'postgresql':
self.query['lockForInsert'] = "" self.query['lockForInsert'] = ""
elif db_server == 'sqlite':
self.query['lockForInsert'] = ""
if db_server == 'mysql':
self.query['vacuum'] = """optimize table Hands, HandsPlayers, HandsActions, Players
, HudCache, GameTypes, Sites, Tourneys
, TourneysPlayers, TourneyTypes, Autorates
"""
elif db_server == 'postgresql':
self.query['vacuum'] = """ vacuum """
elif db_server == 'sqlite':
self.query['vacuum'] = """ vacuum """
self.query['getGametypeFL'] = """SELECT id self.query['getGametypeFL'] = """SELECT id
FROM Gametypes FROM Gametypes

View File

@ -159,10 +159,10 @@ class fpdb:
def add_icon_to_button(self, button): def add_icon_to_button(self, button):
iconBox = gtk.HBox(False, 0) iconBox = gtk.HBox(False, 0)
image = gtk.Image() image = gtk.Image()
image.set_from_stock(gtk.STOCK_CLOSE, gtk.ICON_SIZE_MENU) image.set_from_stock(gtk.STOCK_CLOSE, gtk.ICON_SIZE_SMALL_TOOLBAR)
gtk.Button.set_relief(button, gtk.RELIEF_NONE) gtk.Button.set_relief(button, gtk.RELIEF_NONE)
settings = gtk.Widget.get_settings(button); settings = gtk.Widget.get_settings(button);
(w,h) = gtk.icon_size_lookup_for_settings(settings, gtk.ICON_SIZE_MENU); (w,h) = gtk.icon_size_lookup_for_settings(settings, gtk.ICON_SIZE_SMALL_TOOLBAR);
gtk.Widget.set_size_request (button, w + 4, h + 4); gtk.Widget.set_size_request (button, w + 4, h + 4);
image.show() image.show()
iconBox.pack_start(image, True, False, 0) iconBox.pack_start(image, True, False, 0)
@ -357,6 +357,27 @@ class fpdb:
self.release_global_lock() self.release_global_lock()
def dia_rebuild_indexes(self, widget, data=None):
if self.obtain_global_lock():
self.dia_confirm = gtk.MessageDialog(parent=None
,flags=0
,type=gtk.MESSAGE_WARNING
,buttons=(gtk.BUTTONS_YES_NO)
,message_format="Confirm rebuilding database indexes")
diastring = "Please confirm that you want to rebuild the database indexes."
self.dia_confirm.format_secondary_text(diastring)
response = self.dia_confirm.run()
self.dia_confirm.destroy()
if response == gtk.RESPONSE_YES:
self.db.rebuild_indexes()
self.db.vacuumDB()
self.db.analyzeDB()
elif response == gtk.RESPONSE_NO:
print 'User cancelled rebuilding db indexes'
self.release_global_lock()
def __calendar_dialog(self, widget, entry): def __calendar_dialog(self, widget, entry):
self.dia_confirm.set_modal(False) self.dia_confirm.set_modal(False)
d = gtk.Window(gtk.WINDOW_TOPLEVEL) d = gtk.Window(gtk.WINDOW_TOPLEVEL)
@ -451,6 +472,7 @@ class fpdb:
<menuitem action="createuser"/> <menuitem action="createuser"/>
<menuitem action="createtabs"/> <menuitem action="createtabs"/>
<menuitem action="rebuildhudcache"/> <menuitem action="rebuildhudcache"/>
<menuitem action="rebuildindexes"/>
<menuitem action="stats"/> <menuitem action="stats"/>
</menu> </menu>
<menu action="help"> <menu action="help">
@ -492,6 +514,7 @@ class fpdb:
('createuser', None, 'Create or Delete _User (todo)', None, 'Create or Delete User', self.dia_create_del_user), ('createuser', None, 'Create or Delete _User (todo)', None, 'Create or Delete User', self.dia_create_del_user),
('createtabs', None, 'Create or Recreate _Tables', None, 'Create or Recreate Tables ', self.dia_recreate_tables), ('createtabs', None, 'Create or Recreate _Tables', None, 'Create or Recreate Tables ', self.dia_recreate_tables),
('rebuildhudcache', None, 'Rebuild HUD Cache', None, 'Rebuild HUD Cache', self.dia_recreate_hudcache), ('rebuildhudcache', None, 'Rebuild HUD Cache', None, 'Rebuild HUD Cache', self.dia_recreate_hudcache),
('rebuildindexes', None, 'Rebuild DB Indexes', None, 'Rebuild DB Indexes', self.dia_rebuild_indexes),
('stats', None, '_Statistics (todo)', None, 'View Database Statistics', self.dia_database_stats), ('stats', None, '_Statistics (todo)', None, 'View Database Statistics', self.dia_database_stats),
('help', None, '_Help'), ('help', None, '_Help'),
('Abbrev', None, '_Abbrevations (todo)', None, 'List of Abbrevations', self.tab_abbreviations), ('Abbrev', None, '_Abbrevations (todo)', None, 'List of Abbrevations', self.tab_abbreviations),