From 50f775e7935549c38e89951e01ea44600f89a42f Mon Sep 17 00:00:00 2001 From: Matt Turnbull Date: Sat, 21 Mar 2009 17:46:16 +0000 Subject: [PATCH 1/4] More sensible prints when dropping psql indexes --- pyfpdb/fpdb_simple.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pyfpdb/fpdb_simple.py b/pyfpdb/fpdb_simple.py index 2677d11b..51df87a1 100644 --- a/pyfpdb/fpdb_simple.py +++ b/pyfpdb/fpdb_simple.py @@ -157,12 +157,13 @@ def prepareBulkImport(fdb): pass elif fdb.backend == PGSQL: # DON'T FORGET TO RECREATE THEM!! - print "dropping pg fk", fk['fktab'], fk['fkcol'] + #print "dropping pg fk", fk['fktab'], fk['fkcol'] try: fdb.cursor.execute("alter table " + fk['fktab'] + " drop constraint " + fk['fktab'] + '_' + fk['fkcol'] + '_fkey') + print "dropped pg fk pg fk %s_%s_fkey" % (fk['fktab'], fk['fkcol']) except: - pass + print "! failed drop pg fk %s_%s_fkey" % (fk['fktab'], fk['fkcol']) else: print "Only MySQL and Postgres supported so far" return -1 @@ -177,15 +178,15 @@ def prepareBulkImport(fdb): pass elif fdb.backend == PGSQL: # DON'T FORGET TO RECREATE THEM!! - print "Index dropping disabled for postgresql." - print "dropping pg index ", idx['tab'], idx['col'] + #print "Index dropping disabled for postgresql." + #print "dropping pg index ", idx['tab'], idx['col'] # mod to use tab_col for index name? try: - print "drop index %s_%s_idx" % (idx['tab'],idx['col']) fdb.cursor.execute( "drop index %s_%s_idx" % (idx['tab'],idx['col']) ) - print "dropped pg index ", idx['tab'], idx['col'] + print "drop index %s_%s_idx" % (idx['tab'],idx['col']) + #print "dropped pg index ", idx['tab'], idx['col'] except: - pass + print "! failed drop index %s_%s_idx" % (idx['tab'],idx['col']) else: print "Only MySQL and Postgres supported so far" return -1 From 3f879ddb3d56122d08c4314ed0e280b6fe225827 Mon Sep 17 00:00:00 2001 From: Matt Turnbull Date: Sat, 21 Mar 2009 21:38:32 +0000 Subject: [PATCH 2/4] psql drop fk hang: seemed to be caused by having more than one connection to the db open. Made GuiBulkImport use the importer's connection instead of its own. It still has the one given to it by fpdb.py. I really don't know. Perhaps it needed a commit. --- pyfpdb/GuiBulkImport.py | 8 ++++---- pyfpdb/fpdb_import.py | 1 - pyfpdb/fpdb_simple.py | 6 +++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/pyfpdb/GuiBulkImport.py b/pyfpdb/GuiBulkImport.py index 84ef4645..2b1fe500 100755 --- a/pyfpdb/GuiBulkImport.py +++ b/pyfpdb/GuiBulkImport.py @@ -197,8 +197,8 @@ class GuiBulkImport(): self.load_button.show() # see how many hands are in the db and adjust accordingly - tcursor = db.db.cursor() - tcursor.execute("Select count(1) from Hands;") + tcursor = self.importer.fdb.db.cursor() + tcursor.execute("Select count(1) from Hands") row = tcursor.fetchone() tcursor.close() self.n_hands_in_db = row[0] @@ -233,8 +233,8 @@ def main(argv=None): (options, sys.argv) = parser.parse_args(args = argv) config = Configuration.Config() - db = fpdb_db.fpdb_db() - + db = None + settings = {} settings['minPrint'] = options.minPrint if os.name == 'nt': settings['os'] = 'windows' diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 5d04a1f9..2c33a841 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -178,7 +178,6 @@ class Importer: tmpcursor = self.fdb.db.cursor() tmpcursor.execute("Select count(1) from Hands;") self.settings['handsInDB'] = tmpcursor.fetchone()[0] - tmpcursor.close() except: pass # if this fails we're probably doomed anyway if self.settings['handsInDB'] < 5000: return "drop" diff --git a/pyfpdb/fpdb_simple.py b/pyfpdb/fpdb_simple.py index e39fabd5..aaf2d1d2 100644 --- a/pyfpdb/fpdb_simple.py +++ b/pyfpdb/fpdb_simple.py @@ -53,7 +53,7 @@ indexes = [ , [ # indexes for postgres (list index 3) {'tab':'Boardcards', 'col':'handId', 'drop':0} , {'tab':'Gametypes', 'col':'siteId', 'drop':0} - , {'tab':'Hands', 'col':'gametypeId', 'drop':1} + , {'tab':'Hands', 'col':'gametypeId', 'drop':1} # todo :1 by sqlcoder , but this is needed for all duplicate checks. test 0 , {'tab':'Hands', 'col':'siteHandNo', 'drop':0} , {'tab':'HandsActions', 'col':'handplayerId', 'drop':0} , {'tab':'HandsPlayers', 'col':'handId', 'drop':1} @@ -164,8 +164,8 @@ def prepareBulkImport(fdb): # DON'T FORGET TO RECREATE THEM!! #print "dropping pg fk", fk['fktab'], fk['fkcol'] try: - fdb.cursor.execute("alter table " + fk['fktab'] + " drop constraint " - + fk['fktab'] + '_' + fk['fkcol'] + '_fkey') + #print "alter table %s drop constraint %s_%s_fkey" % (fk['fktab'], fk['fktab'], fk['fkcol']) + fdb.cursor.execute("alter table %s drop constraint %s_%s_fkey" % (fk['fktab'], fk['fktab'], fk['fkcol'])) print "dropped pg fk pg fk %s_%s_fkey" % (fk['fktab'], fk['fkcol']) except: print "! failed drop pg fk %s_%s_fkey" % (fk['fktab'], fk['fkcol']) From 285c7d882493b065b8a2b5695b52db6bf022118c Mon Sep 17 00:00:00 2001 From: Matt Turnbull Date: Sun, 22 Mar 2009 14:53:10 +0000 Subject: [PATCH 3/4] have to supply argv to main() as a list, splitting on whitespace won't work if you supply an argument like a path with a space in it. --- pyfpdb/GuiBulkImport.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyfpdb/GuiBulkImport.py b/pyfpdb/GuiBulkImport.py index 2b1fe500..1a62a37d 100755 --- a/pyfpdb/GuiBulkImport.py +++ b/pyfpdb/GuiBulkImport.py @@ -210,11 +210,9 @@ class GuiBulkImport(): def main(argv=None): """main can also be called in the python interpreter, by supplying the command line as the argument. >>>import GuiBulkImport ->>>GuiBulkImport.main("-f ~/data/hands")""" +>>>GuiBulkImport.main(['-f'.'~/data/hands'])""" if argv is None: argv = sys.argv[1:] - else: - argv = argv.split(" ") def destroy(*args): # call back for terminating the main eventloop gtk.main_quit() From 56fe94219df2fbcb8b0d85f8c08f72825c4980f0 Mon Sep 17 00:00:00 2001 From: Matt Turnbull Date: Sun, 22 Mar 2009 14:56:13 +0000 Subject: [PATCH 4/4] This index is used for every insert to check if it is a dupe. Test confirms it's useful : see the last two graphs of http://fpdb.wiki.sourceforge.net/postgresql+tuning --- pyfpdb/fpdb_simple.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyfpdb/fpdb_simple.py b/pyfpdb/fpdb_simple.py index aaf2d1d2..58fdff05 100644 --- a/pyfpdb/fpdb_simple.py +++ b/pyfpdb/fpdb_simple.py @@ -53,7 +53,7 @@ indexes = [ , [ # indexes for postgres (list index 3) {'tab':'Boardcards', 'col':'handId', 'drop':0} , {'tab':'Gametypes', 'col':'siteId', 'drop':0} - , {'tab':'Hands', 'col':'gametypeId', 'drop':1} # todo :1 by sqlcoder , but this is needed for all duplicate checks. test 0 + , {'tab':'Hands', 'col':'gametypeId', 'drop':0} # mct 22/3/09 , {'tab':'Hands', 'col':'siteHandNo', 'drop':0} , {'tab':'HandsActions', 'col':'handplayerId', 'drop':0} , {'tab':'HandsPlayers', 'col':'handId', 'drop':1}