From 85c635d1da2677067f1b0c21a2b93198e40fab87 Mon Sep 17 00:00:00 2001 From: eblade Date: Fri, 31 Jul 2009 00:13:51 -0400 Subject: [PATCH 01/42] remove error handler on Load Profile menu opt (menu opt doesn't work anyway, so it doesn't error) --- pyfpdb/fpdb.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyfpdb/fpdb.py b/pyfpdb/fpdb.py index b3162239..48f0202c 100755 --- a/pyfpdb/fpdb.py +++ b/pyfpdb/fpdb.py @@ -204,10 +204,10 @@ class fpdb: # print 'User cancelled loading profile' #except: # pass - try: - self.load_profile() - except: - pass + #try: + self.load_profile() + #except: + # pass self.release_global_lock() #end def dia_load_profile From 21b859f244931770385b62bc9fb204f61ee2a7bc Mon Sep 17 00:00:00 2001 From: eblade Date: Fri, 31 Jul 2009 00:15:25 -0400 Subject: [PATCH 02/42] remove error handler on hudcache rebuild menu, if it errors we should know about it --- pyfpdb/fpdb.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/pyfpdb/fpdb.py b/pyfpdb/fpdb.py index 48f0202c..570143cf 100755 --- a/pyfpdb/fpdb.py +++ b/pyfpdb/fpdb.py @@ -246,19 +246,16 @@ class fpdb: def dia_recreate_hudcache(self, widget, data=None): if self.obtain_global_lock(): - try: - dia_confirm = gtk.MessageDialog(parent=None, flags=0, type=gtk.MESSAGE_WARNING, buttons=(gtk.BUTTONS_YES_NO), message_format="Confirm recreating HUD cache") - diastring = "Please confirm that you want to re-create the HUD cache." - dia_confirm.format_secondary_text(diastring) - - response = dia_confirm.run() - dia_confirm.destroy() - if response == gtk.RESPONSE_YES: - self.db.rebuild_hudcache() - elif response == gtk.REPSONSE_NO: - print 'User cancelled rebuilding hud cache' - except: - pass + dia_confirm = gtk.MessageDialog(parent=None, flags=0, type=gtk.MESSAGE_WARNING, buttons=(gtk.BUTTONS_YES_NO), message_format="Confirm recreating HUD cache") + diastring = "Please confirm that you want to re-create the HUD cache." + dia_confirm.format_secondary_text(diastring) + + response = dia_confirm.run() + dia_confirm.destroy() + if response == gtk.RESPONSE_YES: + self.db.rebuild_hudcache() + elif response == gtk.REPSONSE_NO: + print 'User cancelled rebuilding hud cache' self.release_global_lock() From 047b5d94d98ddb51f4fbf0e87da9f081c02876b1 Mon Sep 17 00:00:00 2001 From: eblade Date: Fri, 31 Jul 2009 00:40:31 -0400 Subject: [PATCH 03/42] move constant dicts/lists from functions to global vars, so they aren't loaded every time the function is run --- pyfpdb/Card.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/pyfpdb/Card.py b/pyfpdb/Card.py index 6ab9b62d..8e99599b 100755 --- a/pyfpdb/Card.py +++ b/pyfpdb/Card.py @@ -94,29 +94,32 @@ def cardFromValueSuit(value, suit): elif suit == 's': return(value+38) else: return(0) -def valueSuitFromCard(card): - """ Function to convert a card stored in the database (int 0-52) into value - and suit like 9s, 4c etc """ - if card < 0 or card > 52 or not card: - return('') - else: - return( ['', '2h', '3h', '4h', '5h', '6h', '7h', '8h', '9h', 'Th', 'Jh', 'Qh', 'Kh', 'Ah' +suitFromCardList = ['', '2h', '3h', '4h', '5h', '6h', '7h', '8h', '9h', 'Th', 'Jh', 'Qh', 'Kh', 'Ah' , '2d', '3d', '4d', '5d', '6d', '7d', '8d', '9d', 'Td', 'Jd', 'Qd', 'Kd', 'Ad' , '2c', '3c', '4c', '5c', '6c', '7c', '8c', '9c', 'Tc', 'Jc', 'Qc', 'Kc', 'Ac' , '2s', '3s', '4s', '5s', '6s', '7s', '8s', '9s', 'Ts', 'Js', 'Qs', 'Ks', 'As' - ][card] ) + ] +def valueSuitFromCard(card): + """ Function to convert a card stored in the database (int 0-52) into value + and suit like 9s, 4c etc """ + global suitFromCardList + if card < 0 or card > 52 or not card: + return('') + else: + return suitFromCardList[card] -def encodeCard(cardString): - """Take a card string (Ah) and convert it to the db card code (1).""" - try: - return {'2h': 1, '3h': 2, '4h': 3, '5h': 4, '6h': 5, '7h': 6, '8h': 7, '9h': 8, 'Th': 9, 'Jh': 10, 'Qh': 11, 'Kh': 12, 'Ah': 13, +encodeCardList = {'2h': 1, '3h': 2, '4h': 3, '5h': 4, '6h': 5, '7h': 6, '8h': 7, '9h': 8, 'Th': 9, 'Jh': 10, 'Qh': 11, 'Kh': 12, 'Ah': 13, '2d': 14, '3d': 15, '4d': 16, '5d': 17, '6d': 18, '7d': 19, '8d': 20, '9d': 21, 'Td': 22, 'Jd': 23, 'Qd': 24, 'Kd': 25, 'Ad': 26, '2c': 27, '3c': 28, '4c': 29, '5c': 30, '6c': 31, '7c': 32, '8c': 33, '9c': 34, 'Tc': 35, 'Jc': 36, 'Qc': 27, 'Kc': 38, 'Ac': 39, '2s': 40, '3s': 41, '4s': 42, '5s': 43, '6s': 44, '7s': 45, '8s': 46, '9s': 47, 'Ts': 48, 'Js': 49, 'Qs': 50, 'Ks': 51, 'As': 52, ' ': 0 - }[cardString] - except: - return 0 # everthing that isn't known is a unknown! + } + +def encodeCard(cardString): + """Take a card string (Ah) and convert it to the db card code (1).""" + global encodeCardList + if cardString not in encodeCardList: return 0 + return encodeCardList[cardString] if __name__ == '__main__': print "fpdb card encoding(same as pokersource)" From 6df03cb2343ec2abc1932c855c84e2441afb016c Mon Sep 17 00:00:00 2001 From: eblade Date: Fri, 31 Jul 2009 01:15:28 -0400 Subject: [PATCH 04/42] HUD_main: if new hand id int is not available, we can't print it, so that generates a second error, double fault fpdb_import: ok, we're keeping two lists now, one with updated st_size and one with m_time. grrr. --- pyfpdb/HUD_main.py | 3 ++- pyfpdb/fpdb_import.py | 16 ++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/pyfpdb/HUD_main.py b/pyfpdb/HUD_main.py index 218707bd..7378d6f7 100755 --- a/pyfpdb/HUD_main.py +++ b/pyfpdb/HUD_main.py @@ -163,7 +163,8 @@ class HUD_main(object): cards['common'] = comm_cards['common'] except Exception, err: print "db error: skipping ", new_hand_id, err - sys.stderr.write("Database error %s in hand %d. Skipping.\n" % (err, int(new_hand_id))) + if new_hand_id: # new_hand_id is none if we had an error prior to the store + sys.stderr.write("Database error %s in hand %d. Skipping.\n" % (err, int(new_hand_id))) continue if type == "tour": # hand is from a tournament diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 76b483ba..85526d2c 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -68,7 +68,8 @@ class Importer: self.addToDirList = {} self.removeFromFileList = {} # to remove deleted files self.monitor = False - self.updated = {} #Time last import was run {file:mtime} + self.updatedsize = {} + self.updatedtime = {} self.lines = None self.faobs = None # File as one big string self.pos_in_file = {} # dict to remember how far we have read in the file @@ -268,15 +269,18 @@ class Importer: if os.path.exists(file): stat_info = os.stat(file) #rulog.writelines("path exists ") - if file in self.updated: - if stat_info.st_size > self.updated[file]: + if file in self.updatedsize: # we should be able to assume that if we're in size, we're in time as well + if stat_info.st_size > self.updatedsize[file] or stat_info.st_mtime > self.updatedtime[file]: self.import_file_dict(file, self.filelist[file][0], self.filelist[file][1]) - self.updated[file] = stat_info.st_size + self.updatedsize[file] = stat_info.st_size + self.updatedtime[file] = time() else: if os.path.isdir(file) or (time() - stat_info.st_mtime) < 60: - self.updated[file] = 0 + self.updatedsize[file] = 0 + self.updatedtime[file] = 0 else: - self.updated[file] = stat_info.st_size + self.updatedsize[file] = stat_info.st_size + self.updatedtime[file] = time() else: self.removeFromFileList[file] = True self.addToDirList = filter(lambda x: self.addImportDirectory(x, True, self.addToDirList[x][0], self.addToDirList[x][1]), self.addToDirList) From 36de79140c15c136e9e038e49aeb984fa2022660 Mon Sep 17 00:00:00 2001 From: sqlcoder Date: Fri, 31 Jul 2009 20:12:29 +0100 Subject: [PATCH 05/42] round hands/sec and show some traceback info in exception handler (v useful) --- pyfpdb/GuiBulkImport.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pyfpdb/GuiBulkImport.py b/pyfpdb/GuiBulkImport.py index f363be6c..72ec362b 100755 --- a/pyfpdb/GuiBulkImport.py +++ b/pyfpdb/GuiBulkImport.py @@ -21,6 +21,7 @@ import os import sys from time import time from optparse import OptionParser +import traceback # pyGTK modules import pygtk @@ -81,14 +82,14 @@ class GuiBulkImport(): ttime = time() - starttime if ttime == 0: ttime = 1 - print 'GuiBulkImport.load done: Stored: %d \tDuplicates: %d \tPartial: %d \tErrors: %d in %s seconds - %d/sec'\ - % (stored, dups, partial, errs, ttime, stored / ttime) + print 'GuiBulkImport.load done: Stored: %d \tDuplicates: %d \tPartial: %d \tErrors: %d in %s seconds - %.0f/sec'\ + % (stored, dups, partial, errs, ttime, (stored+0.0) / ttime) self.importer.clearFileList() self.lab_info.set_text("Import finished") except: - print "bulkimport.loadclicked error: "+str(sys.exc_value) - pass + err = traceback.extract_tb(sys.exc_info()[2])[-1] + print "***Error: "+err[2]+"("+str(err[1])+"): "+str(sys.exc_info()[1]) self.settings['global_lock'].release() else: print "bulk-import aborted - global lock not available" From b503626c2b96e7407f102ecef3bf42f353176ed2 Mon Sep 17 00:00:00 2001 From: sqlcoder Date: Fri, 31 Jul 2009 21:24:21 +0100 Subject: [PATCH 06/42] add db writer threading --- pyfpdb/Database.py | 259 ++++++++++++++++++++----------------- pyfpdb/fpdb_import.py | 238 +++++++++++++++++++++++----------- pyfpdb/fpdb_parse_logic.py | 14 +- 3 files changed, 310 insertions(+), 201 deletions(-) diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index 8d58da50..e72c2882 100755 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -31,6 +31,7 @@ from time import time, strftime import string import re import logging +import Queue # pyGTK modules @@ -469,23 +470,19 @@ class Database: ,action_types, allIns, action_amounts, actionNos, hudImportData, maxSeats, tableName ,seatNos): - try: - fpdb_simple.fillCardArrays(len(names), base, category, card_values, card_suits) + fpdb_simple.fillCardArrays(len(names), base, category, card_values, card_suits) - hands_id = self.storeHands(self.backend, site_hand_no, gametype_id - ,hand_start_time, names, tableName, maxSeats, hudImportData - ,(None, None, None, None, None), (None, None, None, None, None)) + hands_id = self.storeHands(self.backend, site_hand_no, gametype_id + ,hand_start_time, names, tableName, maxSeats, hudImportData + ,(None, None, None, None, None), (None, None, None, None, None)) - #print "before calling store_hands_players_stud, antes:", antes - hands_players_ids = self.store_hands_players_stud(self.backend, hands_id, player_ids - ,start_cashes, antes, card_values - ,card_suits, winnings, rakes, seatNos) + #print "before calling store_hands_players_stud, antes:", antes + hands_players_ids = self.store_hands_players_stud(self.backend, hands_id, player_ids + ,start_cashes, antes, card_values + ,card_suits, winnings, rakes, seatNos) - if 'dropHudCache' not in settings or settings['dropHudCache'] != 'drop': - self.storeHudCache(self.backend, base, category, gametype_id, hand_start_time, player_ids, hudImportData) - except: - print "ring_stud error: " + str(sys.exc_value) # in case exception doesn't get printed - raise fpdb_simple.FpdbError("ring_stud error: " + str(sys.exc_value)) + if 'dropHudCache' not in settings or settings['dropHudCache'] != 'drop': + self.storeHudCache(self.backend, base, category, gametype_id, hand_start_time, player_ids, hudImportData) return hands_id #end def ring_stud @@ -496,30 +493,26 @@ class Database: ,action_amounts, actionNos, hudImportData, maxSeats, tableName, seatNos): """stores a holdem/omaha hand into the database""" - try: - t0 = time() - #print "in ring_holdem_omaha" - fpdb_simple.fillCardArrays(len(names), base, category, card_values, card_suits) - t1 = time() - fpdb_simple.fill_board_cards(board_values, board_suits) - t2 = time() + t0 = time() + #print "in ring_holdem_omaha" + fpdb_simple.fillCardArrays(len(names), base, category, card_values, card_suits) + t1 = time() + fpdb_simple.fill_board_cards(board_values, board_suits) + t2 = time() - hands_id = self.storeHands(self.backend, site_hand_no, gametype_id - ,hand_start_time, names, tableName, maxSeats - ,hudImportData, board_values, board_suits) - #TEMPORARY CALL! - Just until all functions are migrated - t3 = time() - hands_players_ids = self.store_hands_players_holdem_omaha( - self.backend, category, hands_id, player_ids, start_cashes - , positions, card_values, card_suits, winnings, rakes, seatNos, hudImportData) - t4 = time() - if 'dropHudCache' not in settings or settings['dropHudCache'] != 'drop': - self.storeHudCache(self.backend, base, category, gametype_id, hand_start_time, player_ids, hudImportData) - t5 = time() - #print "fills=(%4.3f) saves=(%4.3f,%4.3f,%4.3f)" % (t2-t0, t3-t2, t4-t3, t5-t4) - except: - print "ring_holdem_omaha error: " + str(sys.exc_value) # in case exception doesn't get printed - raise fpdb_simple.FpdbError("ring_holdem_omaha error: " + str(sys.exc_value)) + hands_id = self.storeHands(self.backend, site_hand_no, gametype_id + ,hand_start_time, names, tableName, maxSeats + ,hudImportData, board_values, board_suits) + #TEMPORARY CALL! - Just until all functions are migrated + t3 = time() + hands_players_ids = self.store_hands_players_holdem_omaha( + self.backend, category, hands_id, player_ids, start_cashes + , positions, card_values, card_suits, winnings, rakes, seatNos, hudImportData) + t4 = time() + if 'dropHudCache' not in settings or settings['dropHudCache'] != 'drop': + self.storeHudCache(self.backend, base, category, gametype_id, hand_start_time, player_ids, hudImportData) + t5 = time() + #print "fills=(%4.3f) saves=(%4.3f,%4.3f,%4.3f)" % (t2-t0, t3-t2, t4-t3, t5-t4) return hands_id #end def ring_holdem_omaha @@ -532,28 +525,24 @@ class Database: ,actionNos, hudImportData, maxSeats, tableName, seatNos): """stores a tourney holdem/omaha hand into the database""" - try: - fpdb_simple.fillCardArrays(len(names), base, category, card_values, card_suits) - fpdb_simple.fill_board_cards(board_values, board_suits) + fpdb_simple.fillCardArrays(len(names), base, category, card_values, card_suits) + fpdb_simple.fill_board_cards(board_values, board_suits) - tourney_id = self.store_tourneys(tourneyTypeId, siteTourneyNo, entries, prizepool, tourney_start) - tourneys_players_ids = self.store_tourneys_players(tourney_id, player_ids, payin_amounts, ranks, winnings) + tourney_id = self.store_tourneys(tourneyTypeId, siteTourneyNo, entries, prizepool, tourney_start) + tourneys_players_ids = self.store_tourneys_players(tourney_id, player_ids, payin_amounts, ranks, winnings) - hands_id = self.storeHands(self.backend, site_hand_no, gametype_id - ,hand_start_time, names, tableName, maxSeats - ,hudImportData, board_values, board_suits) + hands_id = self.storeHands(self.backend, site_hand_no, gametype_id + ,hand_start_time, names, tableName, maxSeats + ,hudImportData, board_values, board_suits) - hands_players_ids = self.store_hands_players_holdem_omaha_tourney( - self.backend, category, hands_id, player_ids, start_cashes, positions - , card_values, card_suits, winnings, rakes, seatNos, tourneys_players_ids - , hudImportData) + hands_players_ids = self.store_hands_players_holdem_omaha_tourney( + self.backend, category, hands_id, player_ids, start_cashes, positions + , card_values, card_suits, winnings, rakes, seatNos, tourneys_players_ids + , hudImportData) - #print "tourney holdem, backend=%d" % backend - if 'dropHudCache' not in settings or settings['dropHudCache'] != 'drop': - self.storeHudCache(self.backend, base, category, gametype_id, hand_start_time, player_ids, hudImportData) - except: - print "tourney_holdem_omaha error: " + str(sys.exc_value) # in case exception doesn't get printed - raise fpdb_simple.FpdbError("tourney_holdem_omaha error: " + str(sys.exc_value)) + #print "tourney holdem, backend=%d" % backend + if 'dropHudCache' not in settings or settings['dropHudCache'] != 'drop': + self.storeHudCache(self.backend, base, category, gametype_id, hand_start_time, player_ids, hudImportData) return hands_id #end def tourney_holdem_omaha @@ -565,26 +554,22 @@ class Database: ,actionNos, hudImportData, maxSeats, tableName, seatNos): #stores a tourney stud/razz hand into the database - try: - fpdb_simple.fillCardArrays(len(names), base, category, cardValues, cardSuits) + fpdb_simple.fillCardArrays(len(names), base, category, cardValues, cardSuits) - tourney_id = self.store_tourneys(tourneyTypeId, siteTourneyNo, entries, prizepool, tourneyStartTime) + tourney_id = self.store_tourneys(tourneyTypeId, siteTourneyNo, entries, prizepool, tourneyStartTime) - tourneys_players_ids = self.store_tourneys_players(tourney_id, playerIds, payin_amounts, ranks, winnings) + tourneys_players_ids = self.store_tourneys_players(tourney_id, playerIds, payin_amounts, ranks, winnings) - hands_id = self.storeHands( self.backend, siteHandNo, gametypeId - , handStartTime, names, tableName, maxSeats - , hudImportData, board_values, board_suits ) + hands_id = self.storeHands( self.backend, siteHandNo, gametypeId + , handStartTime, names, tableName, maxSeats + , hudImportData, board_values, board_suits ) - hands_players_ids = self.store_hands_players_stud_tourney(self.backend, hands_id - , playerIds, startCashes, antes, cardValues, cardSuits - , winnings, rakes, seatNos, tourneys_players_ids) + hands_players_ids = self.store_hands_players_stud_tourney(self.backend, hands_id + , playerIds, startCashes, antes, cardValues, cardSuits + , winnings, rakes, seatNos, tourneys_players_ids) - if 'dropHudCache' not in settings or settings['dropHudCache'] != 'drop': - self.storeHudCache(self.backend, base, category, gametypeId, hand_start_time, playerIds, hudImportData) - except: - print "tourney_stud error: " + str(sys.exc_value) # in case exception doesn't get printed - raise fpdb_simple.FpdbError("tourney_stud error: " + str(sys.exc_value)) + if 'dropHudCache' not in settings or settings['dropHudCache'] != 'drop': + self.storeHudCache(self.backend, base, category, gametypeId, hand_start_time, playerIds, hudImportData) return hands_id #end def tourney_stud @@ -613,7 +598,7 @@ class Database: "AND referenced_column_name = %s ", (fk['fktab'], fk['fkcol'], fk['rtab'], fk['rcol']) ) cons = c.fetchone() - print "preparebulk find fk: cons=", cons + #print "preparebulk find fk: cons=", cons if cons: print "dropping mysql fk", cons[0], fk['fktab'], fk['fkcol'] try: @@ -994,7 +979,7 @@ class Database: result = self.tourney_holdem_omaha( h.config, h.settings, h.base, h.category, h.siteTourneyNo, h.buyin , h.fee, h.knockout, h.entries, h.prizepool, h.tourneyStartTime - , h.payin_amounts, h.ranks, h.tourneyTypeId, h.siteID, h.siteHandNo + , payin_amounts, ranks, h.tourneyTypeId, h.siteID, h.siteHandNo , h.gametypeID, h.handStartTime, h.names, h.playerIDs, h.startCashes , h.positions, h.cardValues, h.cardSuits, h.boardValues, h.boardSuits , h.winnings, h.rakes, h.actionTypes, h.allIns, h.actionAmounts @@ -1003,13 +988,13 @@ class Database: result = self.tourney_stud( h.config, h.settings, h.base, h.category, h.siteTourneyNo , h.buyin, h.fee, h.knockout, h.entries, h.prizepool, h.tourneyStartTime - , h.payin_amounts, h.ranks, h.tourneyTypeId, h.siteID, h.siteHandNo + , payin_amounts, ranks, h.tourneyTypeId, h.siteID, h.siteHandNo , h.gametypeID, h.handStartTime, h.names, h.playerIDs, h.startCashes , h.antes, h.cardValues, h.cardSuits, h.winnings, h.rakes, h.actionTypes , h.allIns, h.actionAmounts, h.actionNos, h.hudImportData, h.maxSeats , h.tableName, h.seatNos) else: - raise fpself.simple.Fpself.rror("unrecognised category") + raise fpdb_simple.FpdbError("unrecognised category") else: if h.base == "hold": result = self.ring_holdem_omaha( @@ -1027,11 +1012,13 @@ class Database: , h.actionAmounts, h.actionNos, h.hudImportData, h.maxSeats, h.tableName , h.seatNos) else: - raise fpself.simple.Fpself.rror ("unrecognised category") - self.commit() + raise fpdb_simple.FpdbError("unrecognised category") except: print "Error storing hand: " + str(sys.exc_value) self.rollback() + # re-raise the exception so that the calling routine can decide what to do: + # (e.g. a write thread might try again) + raise return result #end def store_the_hand @@ -1576,8 +1563,85 @@ class Database: #end def store_tourneys_players + # read HandToWrite objects from q and insert into database + def insert_queue_hands(self, q, maxwait=10, commitEachHand=True): + n,fails,maxTries,firstWait = 0,0,4,0.1 + sendFinal = False + t0 = time() + while True: + try: + h = q.get(True) # (True,maxWait) has probs if 1st part of import is all dups + except Queue.Empty: + # Queue.Empty exception thrown if q was empty for + # if q.empty() also possible - no point if testing for Queue.Empty exception + # maybe increment a counter and only break after a few times? + # could also test threading.active_count() or look through threading.enumerate() + # so break immediately if no threads, but count up to X exceptions if a writer + # thread is still alive??? + print "queue empty too long - writer stopping ..." + break + except: + print "writer stopping, error reading queue: " + str(sys.exc_info()) + break + #print "got hand", str(h.get_finished()) + + tries,wait,again = 0,firstWait,True + while again: + try: + again = False # set this immediately to avoid infinite loops! + if h.get_finished(): + # all items on queue processed + sendFinal = True + else: + self.store_the_hand(h) + # optional commit, could be every hand / every N hands / every time a + # commit message received?? mark flag to indicate if commits outstanding + if commitEachHand: + self.commit() + n = n + 1 + except: + #print "iqh store error", sys.exc_value # debug + self.rollback() + if re.search('deadlock', str(sys.exc_info()[1]), re.I): + # deadlocks only a problem if hudcache is being updated + tries = tries + 1 + if tries < maxTries and wait < 5: # wait < 5 just to make sure + print "deadlock detected - trying again ..." + time.sleep(wait) + wait = wait + wait + again = True + else: + print "too many deadlocks - failed to store hand " + h.get_siteHandNo() + if not again: + fails = fails + 1 + err = traceback.extract_tb(sys.exc_info()[2])[-1] + print "***Error storing hand: "+err[2]+"("+str(err[1])+"): "+str(sys.exc_info()[1]) + # finished trying to store hand + + # always reduce q count, whether or not this hand was saved ok + q.task_done() + # while True loop + + self.commit() + if sendFinal: + q.task_done() + print "db writer finished: stored %d hands (%d fails) in %.1f seconds" % (n, fails, time()-t0) + # end def insert_queue_hands(): + + + def send_finish_msg(self, q): + try: + h = HandToWrite(True) + q.put(h) + except: + err = traceback.extract_tb(sys.exc_info()[2])[-1] + print "***Error sending finish: "+err[2]+"("+str(err[1])+"): "+str(sys.exc_info()[1]) + # end def send_finish_msg(): + + # Class used to hold all the data needed to write a hand to the db # mainParser() in fpdb_parse_logic.py creates one of these and then passes it to +# self.insert_queue_hands() class HandToWrite: @@ -1675,49 +1739,6 @@ class HandToWrite: raise # end def set_hand - def set_ring_holdem_omaha( self, config, settings, base, category, siteHandNo - , gametypeID, handStartTime, names, playerIDs - , startCashes, positions, cardValues, cardSuits - , boardValues, boardSuits, winnings, rakes - , actionTypes, allIns, actionAmounts, actionNos - , hudImportData, maxSeats, tableName, seatNos ): - self.config = config - self.settings = settings - self.base = base - self.category = category - self.siteHandNo = siteHandNo - self.gametypeID = gametypeID - self.handStartTime = handStartTime - self.names = names - self.playerIDs = playerIDs - self.startCashes = startCashes - self.positions = positions - self.cardValues = cardValues - self.cardSuits = cardSuits - self.boardValues = boardValues - self.boardSuits = boardSuits - self.winnings = winnings - self.rakes = rakes - self.actionTypes = actionTypes - self.allIns = allIns - self.actionAmounts = actionAmounts - self.actionNos = actionNos - self.hudImportData = hudImportData - self.maxSeats = maxSeats - self.tableName = tableName - self.seatNos = seatNos - # end def set_ring_holdem_omaha - - def send_ring_holdem_omaha(self, db): - result = db.ring_holdem_omaha( - self.config, self.settings, self.base, self.category, self.siteHandNo - , self.gametypeID, self.handStartTime, self.names, self.playerIDs - , self.startCashes, self.positions, self.cardValues, self.cardSuits - , self.boardValues, self.boardSuits, self.winnings, self.rakes - , self.actionTypes, self.allIns, self.actionAmounts, self.actionNos - , self.hudImportData, self.maxSeats, self.tableName, self.seatNos) - # end def send_ring_holdem_omaha - def get_finished(self): return( self.finished ) # end def get_finished diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index ed87731b..8ad617f2 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -21,12 +21,15 @@ import os # todo: remove this once import_dir is in fpdb_import import sys -from time import time, strftime +from time import time, strftime, sleep import logging import traceback import math import datetime import re +import Queue +from collections import deque # using Queue for now +import threading # fpdb/FreePokerTools modules @@ -42,11 +45,11 @@ try: mysqlLibFound=True except: pass - + try: import psycopg2 pgsqlLibFound=True - import psycopg2.extensions + import psycopg2.extensions psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) except: @@ -61,7 +64,6 @@ class Importer: self.config = config self.sql = sql - self.database = None # database will be the main db interface eventually self.filelist = {} self.dirlist = {} self.siteIds = {} @@ -74,14 +76,24 @@ class Importer: self.pos_in_file = {} # dict to remember how far we have read in the file #Set defaults self.callHud = self.config.get_import_parameters().get("callFpdbHud") - + + # CONFIGURATION OPTIONS - update allowHudcacheRebuild and forceThreads for faster imports self.settings.setdefault("minPrint", 30) self.settings.setdefault("handCount", 0) - - self.database = Database.Database(self.config, sql = self.sql) # includes .connection and .sql variables + self.settings.setdefault("allowHudcacheRebuild", False) # if True speeds up big imports a lot, also + # stops deadlock problems with threaded imports + self.settings.setdefault("forceThreads", 0) # set to 1/2/more for faster imports + self.settings.setdefault("writeQSize", 1000) # no need to change + self.settings.setdefault("writeQMaxWait", 10) # not used + + self.writeq = None + self.database = Database.Database(self.config, sql = self.sql) + self.writerdbs = [] + self.settings.setdefault("threads", 1) # value overridden by GuiBulkImport - use forceThreads above + for i in xrange(self.settings['threads']): + self.writerdbs.append( Database.Database(self.config, sql = self.sql) ) self.NEWIMPORT = False - self.allow_hudcache_rebuild = False #Set functions def setCallHud(self, value): @@ -104,6 +116,9 @@ class Importer: def setThreads(self, value): self.settings['threads'] = value + if self.settings["threads"] > len(self.writerdbs): + for i in xrange(self.settings['threads'] - len(self.writerdbs)): + self.writerdbs.append( Database.Database(self.config, sql = self.sql) ) def setDropIndexes(self, value): self.settings['dropIndexes'] = value @@ -114,6 +129,11 @@ class Importer: def clearFileList(self): self.filelist = {} + def closeDBs(self): + self.database.disconnect() + for i in xrange(len(self.writerdbs)): + self.writerdbs[i].disconnect() + #Add an individual file to filelist def addImportFile(self, filename, site = "default", filter = "passthrough"): #TODO: test it is a valid file -> put that in config!! @@ -164,61 +184,109 @@ class Importer: print "Warning: Attempted to add non-directory: '" + str(dir) + "' as an import directory" def runImport(self): - """"Run full import on self.filelist.""" + """"Run full import on self.filelist. This is called from GuiBulkImport.py""" + if self.settings['forceThreads'] > 0: # use forceThreads until threading enabled in GuiBulkImport + self.setThreads(self.settings['forceThreads']) + # Initial setup start = datetime.datetime.now() + starttime = time() print "Started at", start, "--", len(self.filelist), "files to import.", self.settings['dropIndexes'] if self.settings['dropIndexes'] == 'auto': - self.settings['dropIndexes'] = self.calculate_auto2(12.0, 500.0) - if self.allow_hudcache_rebuild: - self.settings['dropHudCache'] = self.calculate_auto2(25.0, 500.0) # returns "drop"/"don't drop" + self.settings['dropIndexes'] = self.calculate_auto2(self.database, 12.0, 500.0) + if self.settings['allowHudcacheRebuild']: + self.settings['dropHudCache'] = self.calculate_auto2(self.database, 25.0, 500.0) # returns "drop"/"don't drop" if self.settings['dropIndexes'] == 'drop': self.database.prepareBulkImport() else: print "No need to drop indexes." #print "dropInd =", self.settings['dropIndexes'], " dropHudCache =", self.settings['dropHudCache'] + + if self.settings['threads'] <= 0: + (totstored, totdups, totpartial, toterrors) = self.importFiles(self.database, None) + else: + # create queue (will probably change to deque at some point): + self.writeq = Queue.Queue( self.settings['writeQSize'] ) + # start separate thread(s) to read hands from queue and write to db: + for i in xrange(self.settings['threads']): + t = threading.Thread( target=self.writerdbs[i].insert_queue_hands + , args=(self.writeq, self.settings["writeQMaxWait"]) + , name="dbwriter-"+str(i) ) + t.setDaemon(True) + t.start() + # read hands and write to q: + (totstored, totdups, totpartial, toterrors) = self.importFiles(self.database, self.writeq) + + if self.writeq.empty(): + print "writers finished already" + pass + else: + print "waiting for writers to finish ..." + #for t in threading.enumerate(): + # print " "+str(t) + #self.writeq.join() + #using empty() might be more reliable: + while not self.writeq.empty() and len(threading.enumerate()) > 1: + sleep(0.5) + print " ... writers finished" + + # Tidying up after import + if self.settings['dropIndexes'] == 'drop': + self.database.afterBulkImport() + else: + print "No need to rebuild indexes." + if self.settings['allowHudcacheRebuild'] and self.settings['dropHudCache'] == 'drop': + self.database.rebuild_hudcache() + else: + print "No need to rebuild hudcache." + self.database.analyzeDB() + endtime = time() + return (totstored, totdups, totpartial, toterrors, endtime-starttime) + # end def runImport + + def importFiles(self, db, q): + """"Read filenames in self.filelist and pass to import_file_dict(). + Uses a separate database connection if created as a thread (caller + passes None or no param as db).""" + totstored = 0 totdups = 0 totpartial = 0 toterrors = 0 tottime = 0 -# if threads <= 1: do this bit for file in self.filelist: - (stored, duplicates, partial, errors, ttime) = self.import_file_dict(file, self.filelist[file][0], self.filelist[file][1]) + (stored, duplicates, partial, errors, ttime) = self.import_file_dict(db, file + ,self.filelist[file][0], self.filelist[file][1], q) totstored += stored totdups += duplicates totpartial += partial toterrors += errors - tottime += ttime - if self.settings['dropIndexes'] == 'drop': - self.database.afterBulkImport() - else: - print "No need to rebuild indexes." - if self.allow_hudcache_rebuild and self.settings['dropHudCache'] == 'drop': - self.database.rebuild_hudcache() - else: - print "No need to rebuild hudcache." - self.database.analyzeDB() - return (totstored, totdups, totpartial, toterrors, tottime) -# else: import threaded - def calculate_auto(self): + for i in xrange( self.settings['threads'] ): + print "sending finish msg qlen =", q.qsize() + db.send_finish_msg(q) + + return (totstored, totdups, totpartial, toterrors) + # end def importFiles + + # not used currently + def calculate_auto(self, db): """An heuristic to determine a reasonable value of drop/don't drop""" if len(self.filelist) == 1: return "don't drop" if 'handsInDB' not in self.settings: try: - tmpcursor = self.database.get_cursor() + tmpcursor = db.get_cursor() tmpcursor.execute("Select count(1) from Hands;") self.settings['handsInDB'] = tmpcursor.fetchone()[0] except: pass # if this fails we're probably doomed anyway if self.settings['handsInDB'] < 5000: return "drop" - if len(self.filelist) < 50: return "don't drop" + if len(self.filelist) < 50: return "don't drop" if self.settings['handsInDB'] > 50000: return "don't drop" return "drop" - def calculate_auto2(self, scale, increment): + def calculate_auto2(self, db, scale, increment): """A second heuristic to determine a reasonable value of drop/don't drop This one adds up size of files to import to guess number of hands in them Example values of scale and increment params might be 10 and 500 meaning @@ -231,7 +299,7 @@ class Importer: # get number of hands in db if 'handsInDB' not in self.settings: try: - tmpcursor = self.database.get_cursor() + tmpcursor = db.get_cursor() tmpcursor.execute("Select count(1) from Hands;") self.settings['handsInDB'] = tmpcursor.fetchone()[0] except: @@ -244,7 +312,7 @@ class Importer: stat_info = os.stat(file) total_size += stat_info.st_size - # if hands_in_db is zero or very low, we want to drop indexes, otherwise compare + # if hands_in_db is zero or very low, we want to drop indexes, otherwise compare # import size with db size somehow: ret = "don't drop" if self.settings['handsInDB'] < scale * (total_size/size_per_hand) + increment: @@ -253,7 +321,7 @@ class Importer: # size_per_hand, "inc =", increment, "return:", ret return ret - #Run import on updated files, then store latest update time. + #Run import on updated files, then store latest update time. Called from GuiAutoImport.py def runUpdated(self): #Check for new files in monitored directories #todo: make efficient - always checks for new file, should be able to use mtime of directory @@ -268,11 +336,11 @@ class Importer: if os.path.exists(file): stat_info = os.stat(file) #rulog.writelines("path exists ") - try: + try: lastupdate = self.updated[file] #rulog.writelines("lastupdate = %d, mtime = %d" % (lastupdate,stat_info.st_mtime)) if stat_info.st_mtime > lastupdate: - self.import_file_dict(file, self.filelist[file][0], self.filelist[file][1]) + self.import_file_dict(self.database, file, self.filelist[file][0], self.filelist[file][1]) self.updated[file] = time() except: self.updated[file] = time() @@ -281,10 +349,10 @@ class Importer: if os.path.isdir(file) or (time() - stat_info.st_mtime) < 60: # TODO attach a HHC thread to the file # TODO import the output of the HHC thread -- this needs to wait for the HHC to block? - self.import_file_dict(file, self.filelist[file][0], self.filelist[file][1]) + self.import_file_dict(self.database, file, self.filelist[file][0], self.filelist[file][1], None) # TODO we also test if directory, why? #if os.path.isdir(file): - #self.import_file_dict(file, self.filelist[file][0], self.filelist[file][1]) + #self.import_file_dict(self.database, file, self.filelist[file][0], self.filelist[file][1]) else: self.removeFromFileList[file] = True self.addToDirList = filter(lambda x: self.addImportDirectory(x, True, self.addToDirList[x][0], self.addToDirList[x][1]), self.addToDirList) @@ -292,7 +360,7 @@ class Importer: for file in self.removeFromFileList: if file in self.filelist: del self.filelist[file] - + self.addToDirList = {} self.removeFromFileList = {} self.database.rollback() @@ -300,13 +368,15 @@ class Importer: #rulog.close() # This is now an internal function that should not be called directly. - def import_file_dict(self, file, site, filter): + def import_file_dict(self, db, file, site, filter, q=None): #print "import_file_dict" if os.path.isdir(file): self.addToDirList[file] = [site] + [filter] return conv = None + (stored, duplicates, partial, errors, ttime) = (0, 0, 0, 0, 0) + # Load filter, process file, pass returned filename to import_fpdb_file print "\nConverting %s" % file @@ -325,7 +395,7 @@ class Importer: if callable(obj): conv = obj(in_path = file, out_path = out_path, index = 0) # Index into file 0 until changeover if(conv.getStatus() and self.NEWIMPORT == False): - (stored, duplicates, partial, errors, ttime) = self.import_fpdb_file(out_path, site) + (stored, duplicates, partial, errors, ttime) = self.import_fpdb_file(db, out_path, site, q) elif (conv.getStatus() and self.NEWIMPORT == True): #This code doesn't do anything yet handlist = hhc.getProcessedHands() @@ -346,11 +416,11 @@ class Importer: return (stored, duplicates, partial, errors, ttime) - def import_fpdb_file(self, file, site): - #print "import_fpdb_file" + def import_fpdb_file(self, db, file, site, q): starttime = time() last_read_hand = 0 loc = 0 + (stored, duplicates, partial, errors, ttime) = (0, 0, 0, 0, 0) #print "file =", file if file == "stdin": inputFile = sys.stdin @@ -377,13 +447,39 @@ class Importer: self.pos_in_file[file] = inputFile.tell() inputFile.close() - #self.database.lock_for_insert() # should be ok when using one thread + (stored, duplicates, partial, errors, ttime, handsId) = self.import_fpdb_lines(db, self.lines, starttime, file, site, q) + + db.commit() + ttime = time() - starttime + if q == None: + print "\rTotal stored:", stored, " duplicates:", duplicates, "errors:", errors, " time:", ttime + + if not stored: + if duplicates: + for line_no in xrange(len(self.lines)): + if self.lines[line_no].find("Game #")!=-1: + final_game_line=self.lines[line_no] + handsId=fpdb_simple.parseSiteHandNo(final_game_line) + else: + print "failed to read a single hand from file:", inputFile + handsId=0 + #todo: this will cause return of an unstored hand number if the last hand was error + self.handsId=handsId + + return (stored, duplicates, partial, errors, ttime) + # end def import_fpdb_file + + + def import_fpdb_lines(self, db, lines, starttime, file, site, q = None): + """Import an fpdb hand history held in the list lines, could be one hand or many""" + + #db.lock_for_insert() # should be ok when using one thread, but doesn't help?? try: # sometimes we seem to be getting an empty self.lines, in which case, we just want to return. - firstline = self.lines[0] + firstline = lines[0] except: # just skip the debug message and return silently: - #print "DEBUG: import_fpdb_file: failed on self.lines[0]: '%s' '%s' '%s' '%s' " %( file, site, self.lines, loc) + #print "DEBUG: import_fpdb_file: failed on lines[0]: '%s' '%s' '%s' '%s' " %( file, site, lines, loc) return (0,0,0,1,0) if firstline.find("Tournament Summary")!=-1: @@ -399,16 +495,18 @@ class Importer: duplicates = 0 #counter partial = 0 #counter errors = 0 #counter + ttime = None + handsId = -1 - for i in xrange (len(self.lines)): - if (len(self.lines[i])<2): #Wierd way to detect for '\r\n' or '\n' + for i in xrange (len(lines)): + if (len(lines[i])<2): #Wierd way to detect for '\r\n' or '\n' endpos=i - hand=self.lines[startpos:endpos] - + hand=lines[startpos:endpos] + if (len(hand[0])<2): hand=hand[1:] - + if (len(hand)<3): pass #TODO: This is ugly - we didn't actually find the start of the @@ -420,10 +518,10 @@ class Importer: self.hand=hand try: - handsId = fpdb_parse_logic.mainParser( self.settings - , self.siteIds[site], category, hand - , self.config, self.database ) - self.database.commit() + handsId = fpdb_parse_logic.mainParser( self.settings, self.siteIds[site] + , category, hand, self.config + , db, q ) + db.commit() stored += 1 if self.callHud: @@ -433,29 +531,29 @@ class Importer: self.caller.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep) except fpdb_simple.DuplicateError: duplicates += 1 - self.database.rollback() + db.rollback() except (ValueError), fe: errors += 1 self.printEmailErrorMessage(errors, file, hand) if (self.settings['failOnError']): - self.database.commit() #dont remove this, in case hand processing was cancelled. + db.commit() #dont remove this, in case hand processing was cancelled. raise else: - self.database.rollback() + db.rollback() except (fpdb_simple.FpdbError), fe: errors += 1 self.printEmailErrorMessage(errors, file, hand) - self.database.rollback() + db.rollback() if self.settings['failOnError']: - self.database.commit() #dont remove this, in case hand processing was cancelled. + db.commit() #dont remove this, in case hand processing was cancelled. raise if self.settings['minPrint']: if not ((stored+duplicates+errors) % self.settings['minPrint']): - print "stored:", stored, "duplicates:", duplicates, "errors:", errors - + print "stored:", stored, " duplicates:", duplicates, "errors:", errors + if self.settings['handCount']: if ((stored+duplicates+errors) >= self.settings['handCount']): if not self.settings['quiet']: @@ -463,22 +561,8 @@ class Importer: print "Total stored:", stored, "duplicates:", duplicates, "errors:", errors, " time:", (time() - starttime) sys.exit(0) startpos = endpos - ttime = time() - starttime - print "\rTotal stored:", stored, "duplicates:", duplicates, "errors:", errors, " time:", ttime - - if not stored: - if duplicates: - for line_no in xrange(len(self.lines)): - if self.lines[line_no].find("Game #")!=-1: - final_game_line=self.lines[line_no] - handsId=fpdb_simple.parseSiteHandNo(final_game_line) - else: - print "failed to read a single hand from file:", inputFile - handsId=0 - #todo: this will cause return of an unstored hand number if the last hand was error - self.database.commit() - self.handsId=handsId - return (stored, duplicates, partial, errors, ttime) + return (stored, duplicates, partial, errors, ttime, handsId) + # end def import_fpdb_lines def printEmailErrorMessage(self, errors, filename, line): traceback.print_exc(file=sys.stderr) diff --git a/pyfpdb/fpdb_parse_logic.py b/pyfpdb/fpdb_parse_logic.py index fd2d6796..de21439b 100644 --- a/pyfpdb/fpdb_parse_logic.py +++ b/pyfpdb/fpdb_parse_logic.py @@ -25,13 +25,12 @@ from time import time, strftime #parses a holdem hand -def mainParser(settings, siteID, category, hand, config, db = None): - #print "mainparser" - # fdb is not used now - to be removed ... +def mainParser(settings, siteID, category, hand, config, db = None, writeq = None): t0 = time() #print "mainparser" backend = settings['db-backend'] + # Ideally db connection is passed in, if not use sql list if passed in, otherwise start from scratch if db == None: db = Database.Database(c = config, sql = None) category = fpdb_simple.recogniseCategory(hand[0]) @@ -80,7 +79,6 @@ def mainParser(settings, siteID, category, hand, config, db = None): rebuyOrAddon = -1 tourneyTypeId = 1 - fpdb_simple.isAlreadyInDB(db.get_cursor(), gametypeID, siteHandNo) hand = fpdb_simple.filterCrap(hand, isTourney) @@ -177,7 +175,13 @@ def mainParser(settings, siteID, category, hand, config, db = None): , positions, antes, cardValues, cardSuits, boardValues, boardSuits , winnings, rakes, actionTypes, allIns, actionAmounts , actionNos, hudImportData, maxSeats, tableName, seatNos) - result = db.store_the_hand(htw) + + # save hand in db via direct call or via q if in a thread + if writeq == None: + result = db.store_the_hand(htw) + else: + writeq.put(htw) + result = -999 # meaning unknown t9 = time() #print "parse and save=(%4.3f)" % (t9-t0) From f3a4a8b0d0cb9d233e3756e1607a7c4c6a62ec74 Mon Sep 17 00:00:00 2001 From: sqlcoder Date: Fri, 31 Jul 2009 22:30:41 +0100 Subject: [PATCH 07/42] fix previous merge --- pyfpdb/fpdb_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 5e777217..9749f6d7 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -339,7 +339,7 @@ class Importer: #rulog.writelines("path exists ") if file in self.updatedsize: # we should be able to assume that if we're in size, we're in time as well if stat_info.st_size > self.updatedsize[file] or stat_info.st_mtime > self.updatedtime[file]: - self.import_file_dict(file, self.filelist[file][0], self.filelist[file][1]) + self.import_file_dict(self.database, file, self.filelist[file][0], self.filelist[file][1], None) self.updatedsize[file] = stat_info.st_size self.updatedtime[file] = time() else: From 3b9fa3b177b62b4fd205bc89651750acefee80c0 Mon Sep 17 00:00:00 2001 From: sqlcoder Date: Fri, 31 Jul 2009 22:34:26 +0100 Subject: [PATCH 08/42] add useful (hopefully) exception handler --- pyfpdb/GuiAutoImport.py | 46 ++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/pyfpdb/GuiAutoImport.py b/pyfpdb/GuiAutoImport.py index 3095512c..aefe17d1 100755 --- a/pyfpdb/GuiAutoImport.py +++ b/pyfpdb/GuiAutoImport.py @@ -159,29 +159,33 @@ class GuiAutoImport (threading.Thread): # - Ideally we want to release the lock if the auto-import is killed by some # kind of exception - is this possible? if self.settings['global_lock'].acquire(False): # returns false immediately if lock not acquired - print "\nGlobal lock taken ..." - self.doAutoImportBool = True - widget.set_label(u' _Stop Autoimport ') - if self.pipe_to_hud is None: - if os.name == 'nt': - command = "python HUD_main.py" + " " + self.settings['cl_options'] - bs = 0 # windows is not happy with line buffing here - self.pipe_to_hud = subprocess.Popen(command, bufsize = bs, stdin = subprocess.PIPE, - universal_newlines=True) - else: - command = os.path.join(sys.path[0], 'HUD_main.py') - cl = [command, ] + string.split(self.settings['cl_options']) - self.pipe_to_hud = subprocess.Popen(cl, bufsize = 1, stdin = subprocess.PIPE, - universal_newlines=True) + try: + print "\nGlobal lock taken ..." + self.doAutoImportBool = True + widget.set_label(u' _Stop Autoimport ') + if self.pipe_to_hud is None: + if os.name == 'nt': + command = "python HUD_main.py" + " " + self.settings['cl_options'] + bs = 0 # windows is not happy with line buffing here + self.pipe_to_hud = subprocess.Popen(command, bufsize = bs, stdin = subprocess.PIPE, + universal_newlines=True) + else: + command = os.path.join(sys.path[0], 'HUD_main.py') + cl = [command, ] + string.split(self.settings['cl_options']) + self.pipe_to_hud = subprocess.Popen(cl, bufsize = 1, stdin = subprocess.PIPE, + universal_newlines=True) - # Add directories to importer object. - for site in self.input_settings: - self.importer.addImportDirectory(self.input_settings[site][0], True, site, self.input_settings[site][1]) - print "Adding import directories - Site: " + site + " dir: "+ str(self.input_settings[site][0]) - self.do_import() + # Add directories to importer object. + for site in self.input_settings: + self.importer.addImportDirectory(self.input_settings[site][0], True, site, self.input_settings[site][1]) + print "Adding import directories - Site: " + site + " dir: "+ str(self.input_settings[site][0]) + self.do_import() - interval=int(self.intervalEntry.get_text()) - gobject.timeout_add(interval*1000, self.do_import) + interval=int(self.intervalEntry.get_text()) + gobject.timeout_add(interval*1000, self.do_import) + except: + err = traceback.extract_tb(sys.exc_info()[2])[-1] + print "***Error: "+err[2]+"("+str(err[1])+"): "+str(sys.exc_info()[1]) else: print "auto-import aborted - global lock not available" else: # toggled off From 0756345f3ec302b742bc23391df66726f810646e Mon Sep 17 00:00:00 2001 From: sqlcoder Date: Fri, 31 Jul 2009 22:49:46 +0100 Subject: [PATCH 09/42] allow nulls in handsplayer fields so that stud can import (but most fields are missing) --- pyfpdb/SQL.py | 217 +++++++++++++++++++++++++------------------------- 1 file changed, 108 insertions(+), 109 deletions(-) diff --git a/pyfpdb/SQL.py b/pyfpdb/SQL.py index 9b480628..39315ee1 100644 --- a/pyfpdb/SQL.py +++ b/pyfpdb/SQL.py @@ -165,7 +165,6 @@ class Sql: ################################ # List tables ################################ - print "db_server =", db_server if db_server == 'mysql': self.query['list_tables'] = """SHOW TABLES""" elif db_server == 'postgresql': # what is the correct value here? @@ -495,82 +494,82 @@ class Sql: ante INT, winnings int NOT NULL, rake int NOT NULL, - totalProfit INT NOT NULL, + totalProfit INT, comment text, commentTs DATETIME, tourneysPlayersId BIGINT UNSIGNED, tourneyTypeId SMALLINT UNSIGNED NOT NULL, FOREIGN KEY (tourneyTypeId) REFERENCES TourneyTypes(id), - wonWhenSeenStreet1 FLOAT NOT NULL, + wonWhenSeenStreet1 FLOAT, wonWhenSeenStreet2 FLOAT, wonWhenSeenStreet3 FLOAT, wonWhenSeenStreet4 FLOAT, - wonAtSD FLOAT NOT NULL, + wonAtSD FLOAT, - street0VPI BOOLEAN NOT NULL, - street0Aggr BOOLEAN NOT NULL, - street0_3BChance BOOLEAN NOT NULL, - street0_3BDone BOOLEAN NOT NULL, + street0VPI BOOLEAN, + street0Aggr BOOLEAN, + street0_3BChance BOOLEAN, + street0_3BDone BOOLEAN, street0_4BChance BOOLEAN, street0_4BDone BOOLEAN, other3BStreet0 BOOLEAN, other4BStreet0 BOOLEAN, - street1Seen BOOLEAN NOT NULL, - street2Seen BOOLEAN NOT NULL, - street3Seen BOOLEAN NOT NULL, - street4Seen BOOLEAN NOT NULL, - sawShowdown BOOLEAN NOT NULL, + street1Seen BOOLEAN, + street2Seen BOOLEAN, + street3Seen BOOLEAN, + street4Seen BOOLEAN, + sawShowdown BOOLEAN, - street1Aggr BOOLEAN NOT NULL, - street2Aggr BOOLEAN NOT NULL, - street3Aggr BOOLEAN NOT NULL, - street4Aggr BOOLEAN NOT NULL, + street1Aggr BOOLEAN, + street2Aggr BOOLEAN, + street3Aggr BOOLEAN, + street4Aggr BOOLEAN, otherRaisedStreet0 BOOLEAN, - otherRaisedStreet1 BOOLEAN NOT NULL, - otherRaisedStreet2 BOOLEAN NOT NULL, - otherRaisedStreet3 BOOLEAN NOT NULL, - otherRaisedStreet4 BOOLEAN NOT NULL, + otherRaisedStreet1 BOOLEAN, + otherRaisedStreet2 BOOLEAN, + otherRaisedStreet3 BOOLEAN, + otherRaisedStreet4 BOOLEAN, foldToOtherRaisedStreet0 BOOLEAN, - foldToOtherRaisedStreet1 BOOLEAN NOT NULL, - foldToOtherRaisedStreet2 BOOLEAN NOT NULL, - foldToOtherRaisedStreet3 BOOLEAN NOT NULL, - foldToOtherRaisedStreet4 BOOLEAN NOT NULL, + foldToOtherRaisedStreet1 BOOLEAN, + foldToOtherRaisedStreet2 BOOLEAN, + foldToOtherRaisedStreet3 BOOLEAN, + foldToOtherRaisedStreet4 BOOLEAN, - stealAttemptChance BOOLEAN NOT NULL, - stealAttempted BOOLEAN NOT NULL, - foldBbToStealChance BOOLEAN NOT NULL, - foldedBbToSteal BOOLEAN NOT NULL, - foldSbToStealChance BOOLEAN NOT NULL, - foldedSbToSteal BOOLEAN NOT NULL, + stealAttemptChance BOOLEAN, + stealAttempted BOOLEAN, + foldBbToStealChance BOOLEAN, + foldedBbToSteal BOOLEAN, + foldSbToStealChance BOOLEAN, + foldedSbToSteal BOOLEAN, - street1CBChance BOOLEAN NOT NULL, - street1CBDone BOOLEAN NOT NULL, - street2CBChance BOOLEAN NOT NULL, - street2CBDone BOOLEAN NOT NULL, - street3CBChance BOOLEAN NOT NULL, - street3CBDone BOOLEAN NOT NULL, - street4CBChance BOOLEAN NOT NULL, - street4CBDone BOOLEAN NOT NULL, + street1CBChance BOOLEAN, + street1CBDone BOOLEAN, + street2CBChance BOOLEAN, + street2CBDone BOOLEAN, + street3CBChance BOOLEAN, + street3CBDone BOOLEAN, + street4CBChance BOOLEAN, + street4CBDone BOOLEAN, - foldToStreet1CBChance BOOLEAN NOT NULL, - foldToStreet1CBDone BOOLEAN NOT NULL, - foldToStreet2CBChance BOOLEAN NOT NULL, - foldToStreet2CBDone BOOLEAN NOT NULL, - foldToStreet3CBChance BOOLEAN NOT NULL, - foldToStreet3CBDone BOOLEAN NOT NULL, - foldToStreet4CBChance BOOLEAN NOT NULL, - foldToStreet4CBDone BOOLEAN NOT NULL, + foldToStreet1CBChance BOOLEAN, + foldToStreet1CBDone BOOLEAN, + foldToStreet2CBChance BOOLEAN, + foldToStreet2CBDone BOOLEAN, + foldToStreet3CBChance BOOLEAN, + foldToStreet3CBDone BOOLEAN, + foldToStreet4CBChance BOOLEAN, + foldToStreet4CBDone BOOLEAN, - street1CheckCallRaiseChance BOOLEAN NOT NULL, - street1CheckCallRaiseDone BOOLEAN NOT NULL, - street2CheckCallRaiseChance BOOLEAN NOT NULL, - street2CheckCallRaiseDone BOOLEAN NOT NULL, - street3CheckCallRaiseChance BOOLEAN NOT NULL, - street3CheckCallRaiseDone BOOLEAN NOT NULL, - street4CheckCallRaiseChance BOOLEAN NOT NULL, - street4CheckCallRaiseDone BOOLEAN NOT NULL, + street1CheckCallRaiseChance BOOLEAN, + street1CheckCallRaiseDone BOOLEAN, + street2CheckCallRaiseChance BOOLEAN, + street2CheckCallRaiseDone BOOLEAN, + street3CheckCallRaiseChance BOOLEAN, + street3CheckCallRaiseDone BOOLEAN, + street4CheckCallRaiseChance BOOLEAN, + street4CheckCallRaiseDone BOOLEAN, street0Calls TINYINT, street1Calls TINYINT, @@ -613,82 +612,82 @@ class Sql: ante INT, winnings int NOT NULL, rake int NOT NULL, - totalProfit INT NOT NULL, + totalProfit INT, comment text, commentTs timestamp without time zone, tourneysPlayersId BIGINT, tourneyTypeId INT NOT NULL, FOREIGN KEY (tourneyTypeId) REFERENCES TourneyTypes(id), - wonWhenSeenStreet1 FLOAT NOT NULL, + wonWhenSeenStreet1 FLOAT, wonWhenSeenStreet2 FLOAT, wonWhenSeenStreet3 FLOAT, wonWhenSeenStreet4 FLOAT, - wonAtSD FLOAT NOT NULL, + wonAtSD FLOAT, - street0VPI BOOLEAN NOT NULL, - street0Aggr BOOLEAN NOT NULL, - street0_3BChance BOOLEAN NOT NULL, - street0_3BDone BOOLEAN NOT NULL, + street0VPI BOOLEAN, + street0Aggr BOOLEAN, + street0_3BChance BOOLEAN, + street0_3BDone BOOLEAN, street0_4BChance BOOLEAN, street0_4BDone BOOLEAN, other3BStreet0 BOOLEAN, other4BStreet0 BOOLEAN, - street1Seen BOOLEAN NOT NULL, - street2Seen BOOLEAN NOT NULL, - street3Seen BOOLEAN NOT NULL, - street4Seen BOOLEAN NOT NULL, - sawShowdown BOOLEAN NOT NULL, + street1Seen BOOLEAN, + street2Seen BOOLEAN, + street3Seen BOOLEAN, + street4Seen BOOLEAN, + sawShowdown BOOLEAN, - street1Aggr BOOLEAN NOT NULL, - street2Aggr BOOLEAN NOT NULL, - street3Aggr BOOLEAN NOT NULL, - street4Aggr BOOLEAN NOT NULL, + street1Aggr BOOLEAN, + street2Aggr BOOLEAN, + street3Aggr BOOLEAN, + street4Aggr BOOLEAN, otherRaisedStreet0 BOOLEAN, - otherRaisedStreet1 BOOLEAN NOT NULL, - otherRaisedStreet2 BOOLEAN NOT NULL, - otherRaisedStreet3 BOOLEAN NOT NULL, - otherRaisedStreet4 BOOLEAN NOT NULL, + otherRaisedStreet1 BOOLEAN, + otherRaisedStreet2 BOOLEAN, + otherRaisedStreet3 BOOLEAN, + otherRaisedStreet4 BOOLEAN, foldToOtherRaisedStreet0 BOOLEAN, - foldToOtherRaisedStreet1 BOOLEAN NOT NULL, - foldToOtherRaisedStreet2 BOOLEAN NOT NULL, - foldToOtherRaisedStreet3 BOOLEAN NOT NULL, - foldToOtherRaisedStreet4 BOOLEAN NOT NULL, + foldToOtherRaisedStreet1 BOOLEAN, + foldToOtherRaisedStreet2 BOOLEAN, + foldToOtherRaisedStreet3 BOOLEAN, + foldToOtherRaisedStreet4 BOOLEAN, - stealAttemptChance BOOLEAN NOT NULL, - stealAttempted BOOLEAN NOT NULL, - foldBbToStealChance BOOLEAN NOT NULL, - foldedBbToSteal BOOLEAN NOT NULL, - foldSbToStealChance BOOLEAN NOT NULL, - foldedSbToSteal BOOLEAN NOT NULL, + stealAttemptChance BOOLEAN, + stealAttempted BOOLEAN, + foldBbToStealChance BOOLEAN, + foldedBbToSteal BOOLEAN, + foldSbToStealChance BOOLEAN, + foldedSbToSteal BOOLEAN, - street1CBChance BOOLEAN NOT NULL, - street1CBDone BOOLEAN NOT NULL, - street2CBChance BOOLEAN NOT NULL, - street2CBDone BOOLEAN NOT NULL, - street3CBChance BOOLEAN NOT NULL, - street3CBDone BOOLEAN NOT NULL, - street4CBChance BOOLEAN NOT NULL, - street4CBDone BOOLEAN NOT NULL, + street1CBChance BOOLEAN, + street1CBDone BOOLEAN, + street2CBChance BOOLEAN, + street2CBDone BOOLEAN, + street3CBChance BOOLEAN, + street3CBDone BOOLEAN, + street4CBChance BOOLEAN, + street4CBDone BOOLEAN, - foldToStreet1CBChance BOOLEAN NOT NULL, - foldToStreet1CBDone BOOLEAN NOT NULL, - foldToStreet2CBChance BOOLEAN NOT NULL, - foldToStreet2CBDone BOOLEAN NOT NULL, - foldToStreet3CBChance BOOLEAN NOT NULL, - foldToStreet3CBDone BOOLEAN NOT NULL, - foldToStreet4CBChance BOOLEAN NOT NULL, - foldToStreet4CBDone BOOLEAN NOT NULL, + foldToStreet1CBChance BOOLEAN, + foldToStreet1CBDone BOOLEAN, + foldToStreet2CBChance BOOLEAN, + foldToStreet2CBDone BOOLEAN, + foldToStreet3CBChance BOOLEAN, + foldToStreet3CBDone BOOLEAN, + foldToStreet4CBChance BOOLEAN, + foldToStreet4CBDone BOOLEAN, - street1CheckCallRaiseChance BOOLEAN NOT NULL, - street1CheckCallRaiseDone BOOLEAN NOT NULL, - street2CheckCallRaiseChance BOOLEAN NOT NULL, - street2CheckCallRaiseDone BOOLEAN NOT NULL, - street3CheckCallRaiseChance BOOLEAN NOT NULL, - street3CheckCallRaiseDone BOOLEAN NOT NULL, - street4CheckCallRaiseChance BOOLEAN NOT NULL, - street4CheckCallRaiseDone BOOLEAN NOT NULL, + street1CheckCallRaiseChance BOOLEAN, + street1CheckCallRaiseDone BOOLEAN, + street2CheckCallRaiseChance BOOLEAN, + street2CheckCallRaiseDone BOOLEAN, + street3CheckCallRaiseChance BOOLEAN, + street3CheckCallRaiseDone BOOLEAN, + street4CheckCallRaiseChance BOOLEAN, + street4CheckCallRaiseDone BOOLEAN, street0Calls SMALLINT, street1Calls SMALLINT, From e9873d38a53d6e140030212be8711d889207769b Mon Sep 17 00:00:00 2001 From: sqlcoder Date: Sat, 1 Aug 2009 00:06:07 +0100 Subject: [PATCH 10/42] fix sleep() call --- pyfpdb/Database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index 11f9ef17..82a71331 100755 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -27,7 +27,7 @@ Create and manage the database objects. import sys import traceback from datetime import datetime, date, time, timedelta -from time import time, strftime +from time import time, strftime, sleep import string import re import logging @@ -1607,7 +1607,7 @@ class Database: tries = tries + 1 if tries < maxTries and wait < 5: # wait < 5 just to make sure print "deadlock detected - trying again ..." - time.sleep(wait) + sleep(wait) wait = wait + wait again = True else: From fbab3c7e249c3c74f0d42362c2f8bfea89346fcb Mon Sep 17 00:00:00 2001 From: sqlcoder Date: Sat, 1 Aug 2009 00:07:31 +0100 Subject: [PATCH 11/42] make sure handsid is always returned and show size of queue when threaded --- pyfpdb/fpdb_import.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 9749f6d7..c1d835b8 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -375,7 +375,10 @@ class Importer: # Load filter, process file, pass returned filename to import_fpdb_file - print "\nConverting %s" % file + if self.writeq != None: + print "\nConverting " + file + " (" + str(q.qsize()) + ")" + else: + print "\nConverting " + file hhbase = self.config.get_import_parameters().get("hhArchiveBase") hhbase = os.path.expanduser(hhbase) hhdir = os.path.join(hhbase,site) @@ -403,10 +406,10 @@ class Importer: else: # conversion didn't work # TODO: appropriate response? - return (0, 0, 0, 1, 0) + return (0, 0, 0, 1, 0, -1) else: print "Unknown filter filter_name:'%s' in filter:'%s'" %(filter_name, filter) - return + return (0, 0, 0, 1, 0, -1) #This will barf if conv.getStatus != True return (stored, duplicates, partial, errors, ttime) @@ -476,13 +479,13 @@ class Importer: except: # just skip the debug message and return silently: #print "DEBUG: import_fpdb_file: failed on lines[0]: '%s' '%s' '%s' '%s' " %( file, site, lines, loc) - return (0,0,0,1,0) + return (0,0,0,1,0,0) if firstline.find("Tournament Summary")!=-1: print "TODO: implement importing tournament summaries" #self.faobs = readfile(inputFile) #self.parseTourneyHistory() - return (0,0,0,1,0) + return (0,0,0,1,0,0) category=fpdb_simple.recogniseCategory(firstline) @@ -491,8 +494,8 @@ class Importer: duplicates = 0 #counter partial = 0 #counter errors = 0 #counter - ttime = None - handsId = -1 + ttime = 0 + handsId = 0 for i in xrange (len(lines)): if (len(lines[i])<2): #Wierd way to detect for '\r\n' or '\n' From 921812f5e4e5614c1fbfe3e704a867152d769471 Mon Sep 17 00:00:00 2001 From: sqlcoder Date: Sat, 1 Aug 2009 00:08:41 +0100 Subject: [PATCH 12/42] add config var at top of file to allow threaded option --- pyfpdb/GuiBulkImport.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyfpdb/GuiBulkImport.py b/pyfpdb/GuiBulkImport.py index 72ec362b..c166bbb4 100755 --- a/pyfpdb/GuiBulkImport.py +++ b/pyfpdb/GuiBulkImport.py @@ -35,6 +35,9 @@ import Configuration class GuiBulkImport(): + # CONFIGURATION - update these as preferred: + allowThreads = True # set to True to try out the threads field + # not used def import_dir(self): """imports a directory, non-recursive. todo: move this to fpdb_import so CLI can use it""" @@ -146,7 +149,8 @@ class GuiBulkImport(): self.spin_threads = gtk.SpinButton(adjustment=threads_adj, climb_rate=0.0, digits=0) self.table.attach(self.spin_threads, 4, 5, 0, 1, xpadding = 0, ypadding = 0, yoptions=gtk.SHRINK) self.spin_threads.show() - self.spin_threads.set_sensitive(False) + if not self.allowThreads: + self.spin_threads.set_sensitive(False) # checkbox - fail on error? self.chk_fail = gtk.CheckButton('Fail on error') From fed180d945b580d343c5e3753cf06d80ae2e2c7c Mon Sep 17 00:00:00 2001 From: sqlcoder Date: Sat, 1 Aug 2009 00:37:13 +0100 Subject: [PATCH 13/42] set allow threads option to false --- pyfpdb/GuiBulkImport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyfpdb/GuiBulkImport.py b/pyfpdb/GuiBulkImport.py index c166bbb4..8472f7ed 100755 --- a/pyfpdb/GuiBulkImport.py +++ b/pyfpdb/GuiBulkImport.py @@ -36,7 +36,7 @@ import Configuration class GuiBulkImport(): # CONFIGURATION - update these as preferred: - allowThreads = True # set to True to try out the threads field + allowThreads = False # set to True to try out the threads field # not used def import_dir(self): From 4dd9a8877da112d5c36147d70c3d0355c90cfe8e Mon Sep 17 00:00:00 2001 From: sqlcoder Date: Sat, 1 Aug 2009 12:45:10 +0100 Subject: [PATCH 14/42] restart python if running an old version and 2.5 or 2.6 is available on PATH (Windows) --- pyfpdb/Options.py | 3 +++ pyfpdb/fpdb.py | 27 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/pyfpdb/Options.py b/pyfpdb/Options.py index 38307d40..8cd98f20 100644 --- a/pyfpdb/Options.py +++ b/pyfpdb/Options.py @@ -32,6 +32,9 @@ def fpdb_options(): parser.add_option("-c", "--configFile", dest="config", default=None, help="Specifies a configuration file.") + parser.add_option("-r", "--rerunPython", + action="store_true", + help="Indicates program was restarted with a different path (only allowed once).") (options, sys.argv) = parser.parse_args() return (options, sys.argv) diff --git a/pyfpdb/fpdb.py b/pyfpdb/fpdb.py index 570143cf..7f648b52 100755 --- a/pyfpdb/fpdb.py +++ b/pyfpdb/fpdb.py @@ -17,6 +17,33 @@ import os import sys +import re + +# if path is set to use an old version of python look for a new one: +# (does this work in linux?) +if os.name == 'nt' and sys.version[0:3] not in ('2.5', '2.6') and '-r' not in sys.argv: + #print "old path =", os.environ['PATH'] + dirs = re.split(os.pathsep, os.environ['PATH']) + # remove any trailing / or \ chars from dirs: + dirs = [re.sub('[\\/]$','',p) for p in dirs] + # remove any dirs containing 'python' apart from those ending in 'python25', 'python26' or 'python': + dirs = [p for p in dirs if not re.search('python', p, re.I) or re.search('python25$', p, re.I) or re.search('python26$', p, re.I)] + tmppath = ";".join(dirs) + #print "new path =", tmppath + if re.search('python', tmppath, re.I): + os.environ['PATH'] = tmppath + print "Python " + sys.version[0:3] + ' - press return to continue\n' + sys.stdin.readline() + os.execvpe('python.exe', ('python.exe', 'fpdb.py', '-r'), os.environ) # first arg is ignored (name of program being run) + else: + print "\npython 2.5 not found, please install python 2.5 or 2.6 for fpdb\n" + exit +else: + pass + #print "debug - not changing path" + +print "Python " + sys.version[0:3] + '...\n' + import threading import Options import string From 1d2df984de5ba528ebe5bbfcb668d3f2b5f565a5 Mon Sep 17 00:00:00 2001 From: sqlcoder Date: Sat, 1 Aug 2009 12:47:07 +0100 Subject: [PATCH 15/42] New helper prog to create batch files in windows. Creates a separate .bat file for each GTK dir on PATH so that user can try each one out --- pyfpdb/windows_make_bats.py | 47 +++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100755 pyfpdb/windows_make_bats.py diff --git a/pyfpdb/windows_make_bats.py b/pyfpdb/windows_make_bats.py new file mode 100755 index 00000000..e602314f --- /dev/null +++ b/pyfpdb/windows_make_bats.py @@ -0,0 +1,47 @@ + +# create .bat scripts in windows to try out different gtk dirs + +try: + + import os + import sys + import re + + if os.name != 'nt': + print "\nThis script is only for windows\n" + exit() + + dirs = re.split(os.pathsep, os.environ['PATH']) + # remove any trailing / or \ chars from dirs: + dirs = [re.sub('[\\/]$','',p) for p in dirs] + # remove any dirs containing 'python' apart from those ending in 'python25', 'python26' or 'python': + dirs = [p for p in dirs if not re.search('python', p, re.I) or re.search('python25$', p, re.I) or re.search('python26$', p, re.I)] + # find gtk dirs: + gtkdirs = [p for p in dirs if re.search('gtk', p, re.I)] + + lines = [ '@echo off\n\n' + , '' + , 'python fpdb.py\n\n' + , 'pause\n\n' + ] + if gtkdirs: + i = 1 + for gpath in gtkdirs: # enumerate converts the \\ into \ + tmpdirs = [p for p in dirs if not re.search('gtk', p, re.I) or p == gpath] + tmppath = ";".join(tmpdirs) + lines[1] = 'PATH=' + tmppath + '\n\n' + bat = open('run_fpdb'+str(i)+'.bat', 'w') + bat.writelines(lines) + bat.close() + i = i + 1 + else: + print "\nno gtk directories found in your path - install gtk or edit the path manually\n" + +except SystemExit: + pass + +except: + print "Error:", str(sys.exc_info()) + pass + +# sys.stdin.readline() From 83fbbc63325bc95ffdc422f81fce0c6b5b20d232 Mon Sep 17 00:00:00 2001 From: Ray Date: Sat, 1 Aug 2009 11:51:37 -0400 Subject: [PATCH 16/42] Finished prep for Stars HH format change. All currency symbols are maintained in dicts that are class variables. The currency symbols are compiled in to the regexs. I.e., "\$?" no longer appears in any of the regexs. Will have to be updated with actual Euro symbol and tested against live HHs when Stars makes the change. --- pyfpdb/PokerStarsToFpdb.py | 49 +++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/pyfpdb/PokerStarsToFpdb.py b/pyfpdb/PokerStarsToFpdb.py index 20ba4319..7f746e9e 100755 --- a/pyfpdb/PokerStarsToFpdb.py +++ b/pyfpdb/PokerStarsToFpdb.py @@ -29,9 +29,11 @@ class PokerStars(HandHistoryConverter): ############################################################ # Class Variables + mixes = { 'HORSE': 'horse', '8-Game': '8game', 'HOSE': 'hose'} # Legal mixed games + sym = {'USD': "\$", 'CAD': "\$", 'T$': ""} # ADD Euro, Sterling, etc HERE substitutions = { 'LEGAL_ISO' : "USD|EUR|GBP|CAD", # legal ISO currency codes - 'LS' : "\$" # legal currency symbols + 'LS' : "\$" # legal currency symbols ADD Euro, Sterling, etc HERE } # Static regexes @@ -54,19 +56,26 @@ class PokerStars(HandHistoryConverter): \)\s-\s # close paren of the stakes (?P.*$)""" % substitutions, re.MULTILINE|re.VERBOSE) + + re_PlayerInfo = re.compile(""" + ^Seat\s(?P[0-9]+):\s + (?P.*)\s + \(%(LS)s?(?P[.0-9]+)\sin\schips\)""" % substitutions, + re.MULTILINE|re.VERBOSE) + + re_HandInfo = re.compile(""" + ^Table\s\'(?P[-\ a-zA-Z\d]+)\'\s + ((?P\d+)-max\s)? + (?P\(Play\sMoney\)\s)? + (Seat\s\#(?P
[-\ a-zA-Z\d]+)\'\s - ((?P\d+)-max\s)? - (?P\(Play\sMoney\)\s)? - (Seat\s\#(?P
[ a-zA-Z]+) - \$?(?P[.0-9]+)/\$?(?P[.0-9]+) - (?P.*) - (?P
[0-9]+):(?P[0-9]+) ET - (?P[0-9]+)/(?P[0-9]+)/(?P[0-9]+)Table (?P
[ a-zA-Z]+)\nSeat (?P