add rebuild indexes option to Database menu (can we put tooltips over these menu options?)

pyfpdb/Database.py

@@ -21,9 +21,9 @@ Create and manage the database objects.
 
 ########################################################################
 
-# ToDo:  - vacuum for mysql
+# ToDo:  - rebuild indexes / vacuum option
-#        - rebuild indexes / vacuum option
+#        - check speed of get_stats_from_hand() - add log info
-#        - check speed of get_stats_from_hand()
+#        - check size of db, seems big? (mysql)
 #        - investigate size of mysql db (200K for just 7K hands? 2GB for 140K hands?)
 
 # postmaster -D /var/lib/pgsql/data
@@ -74,14 +74,9 @@ class Database:
     indexes = [
                 [ ] # no db with index 0
               , [ ] # no db with index 1
-              , [ # indexes for mysql (list index 2)
+              , [ # indexes for mysql (list index 2) (foreign keys not here, in next data structure)
                 #  {'tab':'Players',         'col':'name',              'drop':0}  unique indexes not dropped
                 #  {'tab':'Hands',           'col':'siteHandNo',        'drop':0}  unique indexes not dropped
-                  {'tab':'Hands',           'col':'gametypeId',        'drop':0} # mct 22/3/09
-                , {'tab':'HandsPlayers',    'col':'handId',            'drop':0} # not needed, handled by fk
-                , {'tab':'HandsPlayers',    'col':'playerId',          'drop':0} # not needed, handled by fk
-                , {'tab':'HandsPlayers',    'col':'tourneyTypeId',     'drop':0}
-                , {'tab':'HandsPlayers',    'col':'tourneysPlayersId', 'drop':0}
                 #, {'tab':'Tourneys',        'col':'siteTourneyNo',     'drop':0}  unique indexes not dropped
                 ]
               , [ # indexes for postgres (list index 3)
@@ -122,6 +117,8 @@ class Database:
                       {'fktab':'Hands',        'fkcol':'gametypeId',    'rtab':'Gametypes',     'rcol':'id', 'drop':1}
                     , {'fktab':'HandsPlayers', 'fkcol':'handId',        'rtab':'Hands',         'rcol':'id', 'drop':1}
                     , {'fktab':'HandsPlayers', 'fkcol':'playerId',      'rtab':'Players',       'rcol':'id', 'drop':1}
+                    , {'fktab':'HandsPlayers', 'fkcol':'tourneyTypeId', 'rtab':'TourneyTypes',  'rcol':'id', 'drop':1}
+                    , {'fktab':'HandsPlayers', 'fkcol':'tourneysPlayersId','rtab':'TourneysPlayers','rcol':'id', 'drop':1}
                     , {'fktab':'HandsActions', 'fkcol':'handsPlayerId', 'rtab':'HandsPlayers',  'rcol':'id', 'drop':1}
                     , {'fktab':'HudCache',     'fkcol':'gametypeId',    'rtab':'Gametypes',     'rcol':'id', 'drop':1}
                     , {'fktab':'HudCache',     'fkcol':'playerId',      'rtab':'Players',       'rcol':'id', 'drop':0}
@@ -1171,9 +1168,9 @@ class Database:
                 print "dropping mysql index ", idx['tab'], idx['col']
                 try:
                     self.get_cursor().execute( "alter table %s drop index %s"
-                                             , (idx['tab'],idx['col']) )
+                                             , (idx['tab'], idx['col']) )
                 except:
-                    pass
+                    print "    drop idx failed: " + str(sys.exc_info())
             elif self.backend == self.PGSQL:
                 print "dropping pg index ", idx['tab'], idx['col']
                 # mod to use tab_col for index name?
@@ -1181,13 +1178,119 @@ class Database:
                     self.get_cursor().execute( "drop index %s_%s_idx"
                                                % (idx['tab'],idx['col']) )
                 except:
-                    pass
+                    print "    drop idx failed: " + str(sys.exc_info())
             else:
                 print "Only MySQL and Postgres supported so far"
                 return -1
         if self.backend == self.PGSQL:
             self.connection.set_isolation_level(1)   # go back to normal isolation level
     #end def dropAllIndexes
+
+    def createAllForeignKeys(self):
+        """Create foreign keys"""
+
+        try:
+            if self.backend == self.PGSQL:
+                self.connection.set_isolation_level(0)   # allow table/index operations to work
+            c = self.get_cursor()
+        except:
+            print "    set_isolation_level failed: " + str(sys.exc_info())
+
+        for fk in self.foreignKeys[self.backend]:
+            if self.backend == self.MYSQL_INNODB:
+                c.execute("SELECT constraint_name " +
+                          "FROM information_schema.KEY_COLUMN_USAGE " +
+                          #"WHERE REFERENCED_TABLE_SCHEMA = 'fpdb'
+                          "WHERE 1=1 " +
+                          "AND table_name = %s AND column_name = %s " + 
+                          "AND referenced_table_name = %s " +
+                          "AND referenced_column_name = %s ",
+                          (fk['fktab'], fk['fkcol'], fk['rtab'], fk['rcol']) )
+                cons = c.fetchone()
+                #print "afterbulk: cons=", cons
+                if cons:
+                    pass
+                else:
+                    print "creating fk ", fk['fktab'], fk['fkcol'], "->", fk['rtab'], fk['rcol']
+                    try:
+                        c.execute("alter table " + fk['fktab'] + " add foreign key (" 
+                                  + fk['fkcol'] + ") references " + fk['rtab'] + "(" 
+                                  + fk['rcol'] + ")")
+                    except:
+                        print "    create fk failed: " + str(sys.exc_info())
+            elif self.backend == self.PGSQL:
+                print "creating fk ", fk['fktab'], fk['fkcol'], "->", fk['rtab'], fk['rcol']
+                try:
+                    c.execute("alter table " + fk['fktab'] + " add constraint "
+                              + fk['fktab'] + '_' + fk['fkcol'] + '_fkey'
+                              + " foreign key (" + fk['fkcol']
+                              + ") references " + fk['rtab'] + "(" + fk['rcol'] + ")")
+                except:
+                    print "   create fk failed: " + str(sys.exc_info())
+            else:
+                print "Only MySQL and Postgres supported so far"
+
+        try:
+            if self.backend == self.PGSQL:
+                self.connection.set_isolation_level(1)   # go back to normal isolation level
+        except:
+            print "    set_isolation_level failed: " + str(sys.exc_info())
+    #end def createAllForeignKeys
+
+    def dropAllForeignKeys(self):
+        """Drop all standalone indexes (i.e. not including primary keys or foreign keys)
+           using list of indexes in indexes data structure"""
+        # maybe upgrade to use data dictionary?? (but take care to exclude PK and FK)
+        if self.backend == self.PGSQL:
+            self.connection.set_isolation_level(0)   # allow table/index operations to work
+        c = self.get_cursor()
+
+        for fk in self.foreignKeys[self.backend]:
+            if self.backend == self.MYSQL_INNODB:
+                c.execute("SELECT constraint_name " +
+                          "FROM information_schema.KEY_COLUMN_USAGE " +
+                          #"WHERE REFERENCED_TABLE_SCHEMA = 'fpdb'
+                          "WHERE 1=1 " +
+                          "AND table_name = %s AND column_name = %s " + 
+                          "AND referenced_table_name = %s " +
+                          "AND referenced_column_name = %s ",
+                          (fk['fktab'], fk['fkcol'], fk['rtab'], fk['rcol']) )
+                cons = c.fetchone()
+                #print "preparebulk find fk: cons=", cons
+                if cons:
+                    print "dropping mysql fk", cons[0], fk['fktab'], fk['fkcol']
+                    try:
+                        c.execute("alter table " + fk['fktab'] + " drop foreign key " + cons[0])
+                    except:
+                        print "    drop failed: " + str(sys.exc_info())
+            elif self.backend == self.PGSQL:
+#    DON'T FORGET TO RECREATE THEM!!
+                print "dropping pg fk", fk['fktab'], fk['fkcol']
+                try:
+                    # try to lock table to see if index drop will work:
+                    # hmmm, tested by commenting out rollback in grapher. lock seems to work but 
+                    # then drop still hangs :-(  does work in some tests though??
+                    # will leave code here for now pending further tests/enhancement ...
+                    c.execute( "lock table %s in exclusive mode nowait" % (fk['fktab'],) )
+                    #print "after lock, status:", c.statusmessage
+                    #print "alter table %s drop constraint %s_%s_fkey" % (fk['fktab'], fk['fktab'], fk['fkcol'])
+                    try:
+                        c.execute("alter table %s drop constraint %s_%s_fkey" % (fk['fktab'], fk['fktab'], fk['fkcol']))
+                        print "dropped pg fk pg fk %s_%s_fkey, continuing ..." % (fk['fktab'], fk['fkcol'])
+                    except:
+                        if "does not exist" not in str(sys.exc_value):
+                            print "warning: drop pg fk %s_%s_fkey failed: %s, continuing ..." \
+                                  % (fk['fktab'], fk['fkcol'], str(sys.exc_value).rstrip('\n') )
+                except:
+                    print "warning: constraint %s_%s_fkey not dropped: %s, continuing ..." \
+                          % (fk['fktab'],fk['fkcol'], str(sys.exc_value).rstrip('\n'))
+            else:
+                print "Only MySQL and Postgres supported so far"
+        
+        if self.backend == self.PGSQL:
+            self.connection.set_isolation_level(1)   # go back to normal isolation level
+    #end def dropAllForeignKeys
+
     
     def fillDefaultData(self):
         c = self.get_cursor() 
@@ -1215,6 +1318,12 @@ class Database:
 
     #end def fillDefaultData
 
+    def rebuild_indexes(self, start=None):
+        self.dropAllIndexes()
+        self.createAllIndexes()
+        self.dropAllForeignKeys()
+        self.createAllForeignKeys()
+
     def rebuild_hudcache(self, start=None):
         """clears hudcache and rebuilds from the individual handsplayers records"""
 
@@ -1297,7 +1406,7 @@ class Database:
             except:
                 print "Error during analyze:", str(sys.exc_value)
         elif self.backend == self.PGSQL:
-            self.connection.set_isolation_level(0)   # allow vacuum to work
+            self.connection.set_isolation_level(0)   # allow analyze to work
             try:
                 self.get_cursor().execute(self.sql.query['analyze'])
             except:
@@ -1308,6 +1417,25 @@ class Database:
         print "Analyze took %.1f seconds" % (atime,)
     #end def analyzeDB
 
+    def vacuumDB(self):
+        """Do whatever the DB can offer to update index/table statistics"""
+        stime = time()
+        if self.backend == self.MYSQL_INNODB:
+            try:
+                self.get_cursor().execute(self.sql.query['vacuum'])
+            except:
+                print "Error during vacuum:", str(sys.exc_value)
+        elif self.backend == self.PGSQL:
+            self.connection.set_isolation_level(0)   # allow vacuum to work
+            try:
+                self.get_cursor().execute(self.sql.query['vacuum'])
+            except:
+                print "Error during vacuum:", str(sys.exc_value)
+            self.connection.set_isolation_level(1)   # go back to normal isolation level
+        self.commit()
+        atime = time() - stime
+        print "Vacuum took %.1f seconds" % (atime,)
+    #end def analyzeDB
 
 # Start of Hand Writing routines. Idea is to provide a mixture of routines to store Hand data
 # however the calling prog requires. Main aims:
@@ -1389,7 +1517,7 @@ class Database:
 
         q = q.replace('%s', self.sql.query['placeholder'])
 
-        c = self.connection.cursor()
+        c = self.get_cursor()
 
         c.execute(q, (
                 p['tableName'], 
@@ -1575,7 +1703,7 @@ class Database:
 
         #print "DEBUG: inserts: %s" %inserts
         #print "DEBUG: q: %s" % q
-        c = self.connection.cursor()
+        c = self.get_cursor()
         c.executemany(q, inserts)
 
     def storeHudCacheNew(self, gid, pid, hc):

pyfpdb/SQL.py

@@ -461,7 +461,7 @@ class Sql:
                         totalProfit INT,
                         comment text,
                         commentTs DATETIME,
-                        tourneysPlayersId BIGINT UNSIGNED,
+                        tourneysPlayersId BIGINT UNSIGNED, FOREIGN KEY (tourneysPlayersId) REFERENCES TourneysPlayers(id),
                         tourneyTypeId SMALLINT UNSIGNED NOT NULL DEFAULT 1, FOREIGN KEY (tourneyTypeId) REFERENCES TourneyTypes(id),
 
                         wonWhenSeenStreet1 FLOAT,
@@ -551,9 +551,7 @@ class Sql:
                         street3Raises TINYINT,
                         street4Raises TINYINT,
 
-                        actionString VARCHAR(15),
+                        actionString VARCHAR(15))
-
-                        FOREIGN KEY (tourneysPlayersId) REFERENCES TourneysPlayers(id))
                         ENGINE=INNODB"""
         elif db_server == 'postgresql':
             self.query['createHandsPlayersTable'] = """CREATE TABLE HandsPlayers (
@@ -3100,8 +3098,10 @@ class Sql:
             analyze table Autorates, GameTypes, Hands, HandsPlayers, HudCache, Players
                         , Settings, Sites, Tourneys, TourneysPlayers, TourneyTypes
             """
-        else:  # assume postgres
+        elif db_server == 'postgresql':
-            self.query['analyze'] = "vacuum analyze"
+            self.query['analyze'] = "analyze"
+        elif db_server == 'sqlite':
+            self.query['analyze'] = "analyze"
 
         if db_server == 'mysql':
             self.query['lockForInsert'] = """
@@ -3109,8 +3109,20 @@ class Sql:
                           , HudCache write, GameTypes write, Sites write, Tourneys write
                           , TourneysPlayers write, TourneyTypes write, Autorates write
                 """
-        else:  # assume postgres
+        elif db_server == 'postgresql':
             self.query['lockForInsert'] = ""
+        elif db_server == 'sqlite':
+            self.query['lockForInsert'] = ""
+
+        if db_server == 'mysql':
+            self.query['vacuum'] = """optimize table Hands, HandsPlayers, HandsActions, Players
+                                                   , HudCache, GameTypes, Sites, Tourneys
+                                                   , TourneysPlayers, TourneyTypes, Autorates
+                                      """
+        elif db_server == 'postgresql':
+            self.query['vacuum'] = """ vacuum """
+        elif db_server == 'sqlite':
+            self.query['vacuum'] = """ vacuum """
 
         self.query['getGametypeFL'] = """SELECT id
                                            FROM Gametypes

pyfpdb/fpdb.py

@@ -357,6 +357,27 @@ class fpdb:
 
         self.release_global_lock()
 
+    def dia_rebuild_indexes(self, widget, data=None):
+        if self.obtain_global_lock():
+            self.dia_confirm = gtk.MessageDialog(parent=None
+                                                ,flags=0
+                                                ,type=gtk.MESSAGE_WARNING
+                                                ,buttons=(gtk.BUTTONS_YES_NO)
+                                                ,message_format="Confirm rebuilding database indexes")
+            diastring = "Please confirm that you want to rebuild the database indexes."
+            self.dia_confirm.format_secondary_text(diastring)
+
+            response = self.dia_confirm.run()
+            self.dia_confirm.destroy()
+            if response == gtk.RESPONSE_YES:
+                self.db.rebuild_indexes()
+                self.db.vacuumDB()
+                self.db.analyzeDB()
+            elif response == gtk.RESPONSE_NO:
+                print 'User cancelled rebuilding db indexes'
+
+        self.release_global_lock()
+
     def __calendar_dialog(self, widget, entry):
         self.dia_confirm.set_modal(False)
         d = gtk.Window(gtk.WINDOW_TOPLEVEL)
@@ -451,6 +472,7 @@ class fpdb:
                   <menuitem action="createuser"/>
                   <menuitem action="createtabs"/>
                   <menuitem action="rebuildhudcache"/>
+                  <menuitem action="rebuildindexes"/>
                   <menuitem action="stats"/>
                 </menu>
                 <menu action="help">
@@ -492,6 +514,7 @@ class fpdb:
                                  ('createuser', None, 'Create or Delete _User (todo)', None, 'Create or Delete User', self.dia_create_del_user),
                                  ('createtabs', None, 'Create or Recreate _Tables', None, 'Create or Recreate Tables ', self.dia_recreate_tables),
                                  ('rebuildhudcache', None, 'Rebuild HUD Cache', None, 'Rebuild HUD Cache', self.dia_recreate_hudcache),
+                                 ('rebuildindexes', None, 'Rebuild DB Indexes', None, 'Rebuild DB Indexes', self.dia_rebuild_indexes),
                                  ('stats', None, '_Statistics (todo)', None, 'View Database Statistics', self.dia_database_stats),
                                  ('help', None, '_Help'),
                                  ('Abbrev', None, '_Abbrevations (todo)', None, 'List of Abbrevations', self.tab_abbreviations),