From 72dff55f6f38eb984181c7b55e122ad437f73b29 Mon Sep 17 00:00:00 2001 From: Worros Date: Mon, 9 Mar 2009 18:07:04 +0900 Subject: [PATCH 01/19] [KILLFTP 1/x] Tear out FTP specific hand import code --- pyfpdb/fpdb_import.py | 131 +++++++++++++----------------------------- 1 file changed, 41 insertions(+), 90 deletions(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index d86c4214..fde69125 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -293,12 +293,6 @@ class Importer: # print "import_fpdb_file", file, site, self.lines, "\n" return (0,0,0,1,0) - if firstline.find("Tournament Summary")!=-1: - print "TODO: implement importing tournament summaries" - #self.faobs = readfile(inputFile) - #self.parseTourneyHistory() - return 0 - site=fpdb_simple.recogniseSite(firstline) category=fpdb_simple.recogniseCategory(firstline) @@ -316,91 +310,53 @@ class Importer: if (len(hand[0])<2): hand=hand[1:] - cancelled=False - damaged=False - if (site=="ftp"): - for i in range (len(hand)): - if (hand[i].endswith(" has been canceled")): #this is their typo. this is a typo, right? - cancelled=True - - #FTP generates lines looking like: - #Seat 1: IOS Seat 2: kashman59 (big blind) showed [8c 9d] and won ($3.25) with a pair of Eights - #ie. Seat X multiple times on the same line in the summary section, when a new player sits down in the - #middle of the hand. - #TODO: Deal with this properly, either fix the file or make the parsing code work with this line. - if "Seat" in hand[i]: - mo = re.search(" Seat [0-9]+: ", hand[i]) - if mo: - print "mo=", mo, "\nmo.start=", mo.start(),"\nhand[i]=",hand[i] - hand.insert(i+1, hand[i][mo.start()+1:]) - hand[i] = hand[i][0:mo.start()] - - if (len(hand)<3): - pass - #todo: the above 2 lines are kind of a dirty hack, the mentioned circumstances should be handled elsewhere but that doesnt work with DOS/Win EOL. actually this doesnt work. - elif (hand[0].endswith(" (partial)")): #partial hand - do nothing - partial+=1 - elif (hand[1].find("Seat")==-1 and hand[2].find("Seat")==-1 and hand[3].find("Seat")==-1):#todo: should this be or instead of and? - partial+=1 - elif (cancelled or damaged): - partial+=1 - if damaged: - print """ - DEBUG: Partial hand triggered by a line containing 'Seat X:' twice. This is a - bug in the FTP software when a player sits down in the middle of a hand. - Adding a newline after the player name will fix the issue - """ - print "File: %s" %(file) - print "Line: %s" %(startpos) - else: #normal processing - isTourney=fpdb_simple.isTourney(hand[0]) - if not isTourney: - fpdb_simple.filterAnteBlindFold(site,hand) - hand=fpdb_simple.filterCrap(site, hand, isTourney) - self.hand=hand + isTourney=fpdb_simple.isTourney(hand[0]) + if not isTourney: + fpdb_simple.filterAnteBlindFold(site,hand) + hand=fpdb_simple.filterCrap(site, hand, isTourney) + self.hand=hand - try: - handsId=fpdb_parse_logic.mainParser(self.settings['db-backend'], self.fdb.db + try: + handsId=fpdb_parse_logic.mainParser(self.settings['db-backend'], self.fdb.db ,self.fdb.cursor, site, category, hand) - self.fdb.db.commit() + self.fdb.db.commit() - stored+=1 - if self.callHud: - #print "call to HUD here. handsId:",handsId - #pipe the Hands.id out to the HUD - self.caller.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep) - except fpdb_simple.DuplicateError: - duplicates+=1 - except (ValueError), fe: - errors+=1 - self.printEmailErrorMessage(errors, file, hand) - - if (self.settings['failOnError']): - self.fdb.db.commit() #dont remove this, in case hand processing was cancelled. - raise - except (fpdb_simple.FpdbError), fe: - errors+=1 - self.printEmailErrorMessage(errors, file, hand) + stored+=1 + if self.callHud: + #print "call to HUD here. handsId:",handsId + #pipe the Hands.id out to the HUD + self.caller.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep) + except fpdb_simple.DuplicateError: + duplicates+=1 + except (ValueError), fe: + errors+=1 + self.printEmailErrorMessage(errors, file, hand) - #fe.printStackTrace() #todo: get stacktrace - self.fdb.db.rollback() + if (self.settings['failOnError']): + self.fdb.db.commit() #dont remove this, in case hand processing was cancelled. + raise + except (fpdb_simple.FpdbError), fe: + errors+=1 + self.printEmailErrorMessage(errors, file, hand) + + self.fdb.db.rollback() - if (self.settings['failOnError']): - self.fdb.db.commit() #dont remove this, in case hand processing was cancelled. - raise - if (self.settings['minPrint']!=0): - if ((stored+duplicates+partial+errors)%self.settings['minPrint']==0): - print "stored:", stored, "duplicates:", duplicates, "partial:", partial, "errors:", errors + if (self.settings['failOnError']): + self.fdb.db.commit() #dont remove this, in case hand processing was cancelled. + raise + if (self.settings['minPrint']!=0): + if ((stored+duplicates+errors)%self.settings['minPrint']==0): + print "stored:", stored, "duplicates:", duplicates, "errors:", errors - if (self.settings['handCount']!=0): - if ((stored+duplicates+partial+errors)>=self.settings['handCount']): - if (not self.settings['quiet']): - print "quitting due to reaching the amount of hands to be imported" - print "Total stored:", stored, "duplicates:", duplicates, "partial/damaged:", partial, "errors:", errors, " time:", (time() - starttime) - sys.exit(0) - startpos=endpos + if (self.settings['handCount']!=0): + if ((stored+duplicates+errors)>=self.settings['handCount']): + if (not self.settings['quiet']): + print "quitting due to reaching the amount of hands to be imported" + print "Total stored:", stored, "duplicates:", duplicates, "errors:", errors, " time:", (time() - starttime) + sys.exit(0) + startpos=endpos ttime = time() - starttime - print "\rTotal stored:", stored, "duplicates:", duplicates, "partial:", partial, "errors:", errors, " time:", ttime + print "\rTotal stored:", stored, "duplicates:", duplicates, "errors:", errors, " time:", ttime if stored==0: if duplicates>0: @@ -411,16 +367,11 @@ class Importer: else: print "failed to read a single hand from file:", inputFile handsId=0 - #todo: this will cause return of an unstored hand number if the last hand was error or partial + #todo: this will cause return of an unstored hand number if the last hand was error self.fdb.db.commit() self.handsId=handsId return (stored, duplicates, partial, errors, ttime) - def parseTourneyHistory(self): - print "Tourney history parser stub" - #Find tournament boundaries. - #print self.foabs - def printEmailErrorMessage(self, errors, filename, line): traceback.print_exc(file=sys.stderr) print "Error No.",errors,", please send the hand causing this to steffen@sycamoretest.info so I can fix it." From 58445c0a1b72fb16f3db97bb13a798db1d896feb Mon Sep 17 00:00:00 2001 From: Worros Date: Mon, 9 Mar 2009 18:21:22 +0900 Subject: [PATCH 02/19] [KILLFTP 2/x] More ftp specific code removal --- pyfpdb/fpdb_parse_logic.py | 2 - pyfpdb/fpdb_simple.py | 140 ++++++++----------------------------- 2 files changed, 29 insertions(+), 113 deletions(-) diff --git a/pyfpdb/fpdb_parse_logic.py b/pyfpdb/fpdb_parse_logic.py index dab3bcc4..c5076283 100644 --- a/pyfpdb/fpdb_parse_logic.py +++ b/pyfpdb/fpdb_parse_logic.py @@ -111,8 +111,6 @@ def mainParser(backend, db, cursor, site, category, hand): tableResult=fpdb_simple.parseTableLine(site, base, hand[i]) else: raise fpdb_simple.FpdbError("unrecognised lineType:"+lineTypes[i]) - if site=="ftp": - tableResult=fpdb_simple.parseTableLine(site, base, hand[0]) maxSeats=tableResult['maxSeats'] tableName=tableResult['tableName'] #print "before part5, antes:", antes diff --git a/pyfpdb/fpdb_simple.py b/pyfpdb/fpdb_simple.py index dbf53e93..d3bb4097 100644 --- a/pyfpdb/fpdb_simple.py +++ b/pyfpdb/fpdb_simple.py @@ -476,8 +476,6 @@ def convert3B4B(site, category, limit_type, actionTypes, actionAmounts): #todo: run correction for below if (site=="ps" and category=="holdem" and limit_type=="nl" and len(bets)==3): fail=False - if (site=="ftp" and category=="omahahi" and limit_type=="pl" and len(bets)==3): - fail=False if fail: print "len(bets)>2 in convert3B4B, i didnt think this is possible. i:",i,"j:",j,"k:",k @@ -689,18 +687,8 @@ def filterCrap(site, hand, isTourney): toRemove.append(hand[i]) elif (hand[i].endswith("is disconnected")): toRemove.append(hand[i]) - elif (hand[i].endswith(" is feeling angry")): - toRemove.append(hand[i]) - elif (hand[i].endswith(" is feeling confused")): - toRemove.append(hand[i]) - elif (hand[i].endswith(" is feeling happy")): - toRemove.append(hand[i]) - elif (hand[i].endswith(" is feeling normal")): - toRemove.append(hand[i]) elif (hand[i].find(" is low with [")!=-1): toRemove.append(hand[i]) - #elif (hand[i].find("-max Seat #")!=-1 and hand[i].find(" is the button")!=-1): - # toRemove.append(hand[i]) elif (hand[i].endswith(" mucks")): toRemove.append(hand[i]) elif (hand[i].endswith(": mucks hand")): @@ -711,8 +699,6 @@ def filterCrap(site, hand, isTourney): toRemove.append(hand[i]) elif (hand[i].find(" shows ")!=-1 and hand[i].find("[")==-1): toRemove.append(hand[i]) - #elif (hand[i].startswith("Table '") and hand[i].endswith("-max")): - # toRemove.append(hand[i]) elif (hand[i].startswith("The button is in seat #")): toRemove.append(hand[i]) #above is alphabetic, reorder below if bored @@ -747,8 +733,6 @@ def filterCrap(site, hand, isTourney): #site specific variable position filter elif (hand[i].find(" said, \"")!=-1): toRemove.append(hand[i]) - elif (hand[i].find(": ")!=-1 and site=="ftp" and hand[i].find("Seat ")==-1 and hand[i].find(": Table")==-1): #filter ftp chat - toRemove.append(hand[i]) if isTourney: if (hand[i].endswith(" is sitting out") and (not hand[i].startswith("Seat "))): toRemove.append(hand[i]) @@ -857,14 +841,6 @@ def isWinLine(line): return True elif (line.find("ties for the low side pot")!=-1): return True - elif (line.find("ties for the main pot")!=-1): #for ftp tied main pot of split pot - return True - elif (line.find("ties for the pot")!=-1): #for ftp tie - return True - elif (line.find("ties for the side pot")!=-1): #for ftp tied split pots - return True - elif (line.find("wins side pot #")!=-1): #for ftp multi split pots - return True elif (line.find("wins the low main pot")!=-1): return True elif (line.find("wins the low pot")!=-1): @@ -879,8 +855,6 @@ def isWinLine(line): return True elif (line.find("wins the main pot")!=-1): return True - elif (line.find("wins the side pot")!=-1): #for ftp split pots - return True elif (line.find("collected")!=-1): return True else: @@ -904,10 +878,6 @@ def parseActionAmount(line, atype, site, isTourney): amount=0 elif (atype=="check"): amount=0 - elif (atype=="unbet" and site=="ftp"): - pos1=line.find("$")+1 - pos2=line.find(" returned to") - amount=float2int(line[pos1:pos2]) elif (atype=="unbet" and site=="ps"): #print "ps unbet, line:",line pos1=line.find("$")+1 @@ -1129,8 +1099,6 @@ def parseCashesAndSeatNos(lines, site): pos1=lines[i].rfind("($")+2 if pos1==1: #for tourneys - it's 1 instead of -1 due to adding 2 above pos1=lines[i].rfind("(")+1 - if (site=="ftp"): - pos2=lines[i].rfind(")") elif (site=="ps"): pos2=lines[i].find(" in chips") cashes.append(float2int(lines[i][pos1:pos2])) @@ -1157,41 +1125,25 @@ def parseHandStartTime(topline, site): if counter==10: break isUTC=False - if site=="ftp": - # Full Tilt Sit'n'Go - # Full Tilt Poker Game #10311865543: $1 + $0.25 Sit & Go (78057629), Table 1 - 25/50 - No Limit Hold'em - 0:07:45 ET - 2009/01/29 - # Cash Game: - # Full Tilt Poker Game #9403951181: Table CR - tay - $0.05/$0.10 - No Limit Hold'em - 9:40:20 ET - 2008/12/09 - # Full Tilt Poker Game #9468383505: Table Bike (deep 6) - $0.05/$0.10 - No Limit Hold'em - 5:09:36 ET - 2008/12/13 - pos = topline.find(" ", len(topline)-26)+1 - tmp = topline[pos:] - - rexx = '(?P
[0-9]+):(?P[0-9]+):(?P[0-9]+) ET [\- ]+(?P[0-9]{4})\/(?P[0-9]{2})\/(?P[0-9]{2})' - m = re.search(rexx,tmp) - result = datetime.datetime(int(m.group('YEAR')), int(m.group('MON')), int(m.group('DAY')), int(m.group('HR')), int(m.group('MIN')), int(m.group('SEC'))) - elif site=="ps": - if topline.find("UTC")!=-1: - pos1 = topline.find("-")+2 - pos2 = topline.find("UTC") - tmp=topline[pos1:pos2] - isUTC=True - else: - tmp=topline - #print "parsehandStartTime, tmp:", tmp - pos = tmp.find("-")+2 - tmp = tmp[pos:] - #Need to match either - # 2008/09/07 06:23:14 ET or - # 2008/08/17 - 01:14:43 (ET) or - # 2008/11/12 9:33:31 CET [2008/11/12 3:33:31 ET] - rexx = '(?P[0-9]{4})\/(?P[0-9]{2})\/(?P[0-9]{2})[\- ]+(?P
[0-9]+):(?P[0-9]+):(?P[0-9]+)' - m = re.search(rexx,tmp) - #print "year:", int(m.group('YEAR')), "month", int(m.group('MON')), "day", int(m.group('DAY')), "hour", int(m.group('HR')), "minute", int(m.group('MIN')), "second", int(m.group('SEC')) - result = datetime.datetime(int(m.group('YEAR')), int(m.group('MON')), int(m.group('DAY')), int(m.group('HR')), int(m.group('MIN')), int(m.group('SEC'))) + if topline.find("UTC")!=-1: + pos1 = topline.find("-")+2 + pos2 = topline.find("UTC") + tmp=topline[pos1:pos2] + isUTC=True else: - raise FpdbError("invalid site in parseHandStartTime") + tmp=topline + #print "parsehandStartTime, tmp:", tmp + pos = tmp.find("-")+2 + tmp = tmp[pos:] + #Need to match either + # 2008/09/07 06:23:14 ET or + # 2008/08/17 - 01:14:43 (ET) or + # 2008/11/12 9:33:31 CET [2008/11/12 3:33:31 ET] + rexx = '(?P[0-9]{4})\/(?P[0-9]{2})\/(?P[0-9]{2})[\- ]+(?P
[0-9]+):(?P[0-9]+):(?P[0-9]+)' + m = re.search(rexx,tmp) + result = datetime.datetime(int(m.group('YEAR')), int(m.group('MON')), int(m.group('DAY')), int(m.group('HR')), int(m.group('MIN')), int(m.group('SEC'))) - if (site=="ftp" or site=="ps") and not isUTC: #these use US ET + if not isUTC: #these use US ET result+=datetime.timedelta(hours=5) return result @@ -1308,40 +1260,13 @@ def parseSiteHandNo(topline): def parseTableLine(site, base, line): """returns a dictionary with maxSeats and tableName""" - if site=="ps": - pos1=line.find('\'')+1 - pos2=line.find('\'', pos1) - #print "table:",line[pos1:pos2] - pos3=pos2+2 - pos4=line.find("-max") - #print "seats:",line[pos3:pos4] - return {'maxSeats':int(line[pos3:pos4]), 'tableName':line[pos1:pos2]} - elif site=="ftp": - pos1=line.find("Table ")+6 - pos2=line.find("-")-1 - if base=="hold": - maxSeats=9 - elif base=="stud": - maxSeats=8 - - if line.find("6 max")!=-1: - maxSeats=6 - elif line.find("4 max")!=-1: - maxSeats=4 - elif line.find("heads up")!=-1: - maxSeats=2 - - tableName = line[pos1:pos2] - for pattern in [' \(6 max\)', ' \(heads up\)', ' \(deep\)', - ' \(deep hu\)', ' \(deep 6\)', ' \(2\)', - ' \(edu\)', ' \(edu, 6 max\)', ' \(6\)', - ' \(speed\)', - ' no all-in', ' fast', ',', ' 50BB min', '\s+$']: - tableName = re.sub(pattern, '', tableName) - tableName = tableName.rstrip() - return {'maxSeats':maxSeats, 'tableName':tableName} - else: - raise FpdbError("invalid site ID") + pos1=line.find('\'')+1 + pos2=line.find('\'', pos1) + #print "table:",line[pos1:pos2] + pos3=pos2+2 + pos4=line.find("-max") + #print "seats:",line[pos3:pos4] + return {'maxSeats':int(line[pos3:pos4]), 'tableName':line[pos1:pos2]} #end def parseTableLine #returns the hand no assigned by the poker site @@ -1359,17 +1284,11 @@ def parseWinLine(line, site, names, winnings, isTourney): if (line.startswith(names[i].encode("latin-1"))): #found a winner if isTourney: pos1=line.rfind("collected ")+10 - if (site=="ftp"): - pos2=line.find(")", pos1) - elif (site=="ps"): - pos2=line.find(" ", pos1) + pos2=line.find(" ", pos1) winnings[i]+=int(line[pos1:pos2]) else: pos1=line.rfind("$")+1 - if (site=="ftp"): - pos2=line.find(")", pos1) - elif (site=="ps"): - pos2=line.find(" ", pos1) + pos2=line.find(" ", pos1) winnings[i]+=float2int(line[pos1:pos2]) #end def parseWinLine @@ -1413,10 +1332,8 @@ def recogniseGametypeID(backend, db, cursor, topline, smallBlindLine, site_id, c pos1=pos2+2 if isTourney: pos1-=1 - if (site_id==1): #ftp - pos2=topline.find(" ", pos1) - elif (site_id==2): #ps - pos2=topline.find(")") + #ps + pos2=topline.find(")") if pos2<=pos1: pos2=topline.find(")", pos1) @@ -1558,6 +1475,7 @@ def recognisePlayerNo(line, names, atype): #returns the site abbreviation for the given site def recogniseSite(line): + # TODO: Make site agnostic if (line.startswith("Full Tilt Poker")): return "ftp" elif (line.startswith("PokerStars")): From 14b58d6ad538dfb7e930e07b84176d12b1cb8d47 Mon Sep 17 00:00:00 2001 From: Worros Date: Tue, 10 Mar 2009 21:36:52 +0900 Subject: [PATCH 03/19] [KILLFTP 3/x] Fix screwup from patch 1 --- pyfpdb/fpdb_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index fde69125..03de8554 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -354,7 +354,7 @@ class Importer: print "quitting due to reaching the amount of hands to be imported" print "Total stored:", stored, "duplicates:", duplicates, "errors:", errors, " time:", (time() - starttime) sys.exit(0) - startpos=endpos + startpos=endpos ttime = time() - starttime print "\rTotal stored:", stored, "duplicates:", duplicates, "errors:", errors, " time:", ttime From 161485e44e5886d21d5ea3e67ee673e4ce6f7bb6 Mon Sep 17 00:00:00 2001 From: Worros Date: Tue, 10 Mar 2009 22:00:37 +0900 Subject: [PATCH 04/19] [KILLFTP 4/x] paper bag fix caused iun patch 1. This code needs replacing --- pyfpdb/fpdb_import.py | 98 +++++++++++++++++++++++-------------------- 1 file changed, 52 insertions(+), 46 deletions(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 03de8554..c4c4e261 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -208,7 +208,7 @@ class Importer: #if os.path.isdir(file): #self.import_file_dict(file, self.filelist[file][0], self.filelist[file][1]) - self.addToDirList = filter(lambda x: self.addImportDirectory(x, True, self.addToDirList[x][0], self.addToDirList[x][1]), self.addToDirList) + self.addToDirList = filter(lambda x: self.addImportDirectory(x, True, self.addToDirList[x][0], self.addToDirList[x][1]), self.addToDirList) for file in self.removeFromFileList: if file in self.filelist: @@ -302,58 +302,64 @@ class Importer: partial=0 #counter errors=0 #counter - for i in xrange (len(self.lines)): #main loop, iterates through the lines of a file and calls the appropriate parser method - if (len(self.lines[i])<2): + for i in xrange (len(self.lines)): + if (len(self.lines[i])<2): #Wierd way to detect for '\r\n' or '\n' endpos=i hand=self.lines[startpos:endpos] if (len(hand[0])<2): hand=hand[1:] + - isTourney=fpdb_simple.isTourney(hand[0]) - if not isTourney: - fpdb_simple.filterAnteBlindFold(site,hand) - hand=fpdb_simple.filterCrap(site, hand, isTourney) - self.hand=hand + if (len(hand)<3): + pass + #TODO: This is ugly - we didn't actually find the start of the + # hand with the outer loop so we test again... + else: + isTourney=fpdb_simple.isTourney(hand[0]) + if not isTourney: + fpdb_simple.filterAnteBlindFold(site,hand) + hand=fpdb_simple.filterCrap(site, hand, isTourney) + self.hand=hand + + try: + handsId=fpdb_parse_logic.mainParser(self.settings['db-backend'], self.fdb.db + ,self.fdb.cursor, site, category, hand) + self.fdb.db.commit() + + stored+=1 + if self.callHud: + #print "call to HUD here. handsId:",handsId + #pipe the Hands.id out to the HUD + self.caller.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep) + except fpdb_simple.DuplicateError: + duplicates+=1 + except (ValueError), fe: + errors+=1 + self.printEmailErrorMessage(errors, file, hand) + + if (self.settings['failOnError']): + self.fdb.db.commit() #dont remove this, in case hand processing was cancelled. + raise + except (fpdb_simple.FpdbError), fe: + errors+=1 + self.printEmailErrorMessage(errors, file, hand) + + self.fdb.db.rollback() + + if (self.settings['failOnError']): + self.fdb.db.commit() #dont remove this, in case hand processing was cancelled. + raise + if (self.settings['minPrint']!=0): + if ((stored+duplicates+errors)%self.settings['minPrint']==0): + print "stored:", stored, "duplicates:", duplicates, "errors:", errors - try: - handsId=fpdb_parse_logic.mainParser(self.settings['db-backend'], self.fdb.db - ,self.fdb.cursor, site, category, hand) - self.fdb.db.commit() - - stored+=1 - if self.callHud: - #print "call to HUD here. handsId:",handsId - #pipe the Hands.id out to the HUD - self.caller.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep) - except fpdb_simple.DuplicateError: - duplicates+=1 - except (ValueError), fe: - errors+=1 - self.printEmailErrorMessage(errors, file, hand) - - if (self.settings['failOnError']): - self.fdb.db.commit() #dont remove this, in case hand processing was cancelled. - raise - except (fpdb_simple.FpdbError), fe: - errors+=1 - self.printEmailErrorMessage(errors, file, hand) - - self.fdb.db.rollback() - - if (self.settings['failOnError']): - self.fdb.db.commit() #dont remove this, in case hand processing was cancelled. - raise - if (self.settings['minPrint']!=0): - if ((stored+duplicates+errors)%self.settings['minPrint']==0): - print "stored:", stored, "duplicates:", duplicates, "errors:", errors - - if (self.settings['handCount']!=0): - if ((stored+duplicates+errors)>=self.settings['handCount']): - if (not self.settings['quiet']): - print "quitting due to reaching the amount of hands to be imported" - print "Total stored:", stored, "duplicates:", duplicates, "errors:", errors, " time:", (time() - starttime) - sys.exit(0) + if (self.settings['handCount']!=0): + if ((stored+duplicates+errors)>=self.settings['handCount']): + if (not self.settings['quiet']): + print "quitting due to reaching the amount of hands to be imported" + print "Total stored:", stored, "duplicates:", duplicates, "errors:", errors, " time:", (time() - starttime) + sys.exit(0) startpos=endpos ttime = time() - starttime print "\rTotal stored:", stored, "duplicates:", duplicates, "errors:", errors, " time:", ttime From 264b18863339013ee65956a58c490650b05e7496 Mon Sep 17 00:00:00 2001 From: Worros Date: Tue, 10 Mar 2009 23:03:02 +0900 Subject: [PATCH 05/19] [KILLFTP 5/x] Remove references to the named site "PokerStars" or "Full Tilt Poker" --- pyfpdb/fpdb_import.py | 8 ++--- pyfpdb/fpdb_parse_logic.py | 22 +++++++------- pyfpdb/fpdb_simple.py | 61 +++++++++++--------------------------- 3 files changed, 32 insertions(+), 59 deletions(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index c4c4e261..dde5ec6f 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -293,7 +293,7 @@ class Importer: # print "import_fpdb_file", file, site, self.lines, "\n" return (0,0,0,1,0) - site=fpdb_simple.recogniseSite(firstline) + # site=fpdb_simple.recogniseSite(firstline) category=fpdb_simple.recogniseCategory(firstline) startpos=0 @@ -318,13 +318,13 @@ class Importer: else: isTourney=fpdb_simple.isTourney(hand[0]) if not isTourney: - fpdb_simple.filterAnteBlindFold(site,hand) - hand=fpdb_simple.filterCrap(site, hand, isTourney) + fpdb_simple.filterAnteBlindFold(hand) + hand=fpdb_simple.filterCrap(hand, isTourney) self.hand=hand try: handsId=fpdb_parse_logic.mainParser(self.settings['db-backend'], self.fdb.db - ,self.fdb.cursor, site, category, hand) + ,self.fdb.cursor, category, hand) self.fdb.db.commit() stored+=1 diff --git a/pyfpdb/fpdb_parse_logic.py b/pyfpdb/fpdb_parse_logic.py index c5076283..a1f2540f 100644 --- a/pyfpdb/fpdb_parse_logic.py +++ b/pyfpdb/fpdb_parse_logic.py @@ -21,7 +21,7 @@ import fpdb_simple import fpdb_save_to_db #parses a holdem hand -def mainParser(backend, db, cursor, site, category, hand): +def mainParser(backend, db, cursor, category, hand): category=fpdb_simple.recogniseCategory(hand[0]) if (category=="holdem" or category=="omahahi" or category=="omahahilo"): base="hold" @@ -35,8 +35,8 @@ def mainParser(backend, db, cursor, site, category, hand): #part 1: read hand no and check for duplicate siteHandNo=fpdb_simple.parseSiteHandNo(hand[0]) - handStartTime=fpdb_simple.parseHandStartTime(hand[0], site) - siteID=fpdb_simple.recogniseSiteID(cursor, site) + handStartTime=fpdb_simple.parseHandStartTime(hand[0]) + siteID=fpdb_simple.recogniseSiteID() #print "parse logic, siteID:",siteID,"site:",site isTourney=fpdb_simple.isTourney(hand[0]) @@ -51,8 +51,6 @@ def mainParser(backend, db, cursor, site, category, hand): #print "small blind line:",smallBlindLine gametypeID=fpdb_simple.recogniseGametypeID(backend, db, cursor, hand[0], hand[smallBlindLine], siteID, category, isTourney) if isTourney: - if site!="ps": - raise fpdb_simple.FpdbError("tourneys are only supported on PS right now") siteTourneyNo=fpdb_simple.parseTourneyNo(hand[0]) buyin=fpdb_simple.parseBuyin(hand[0]) fee=fpdb_simple.parseFee(hand[0]) @@ -75,7 +73,7 @@ def mainParser(backend, db, cursor, site, category, hand): seatLines.append(hand[i]) names=fpdb_simple.parseNames(seatLines) playerIDs = fpdb_simple.recognisePlayerIDs(cursor, names, siteID) - tmp=fpdb_simple.parseCashesAndSeatNos(seatLines, site) + tmp=fpdb_simple.parseCashesAndSeatNos(seatLines) startCashes=tmp['startCashes'] seatNos=tmp['seatNos'] @@ -88,15 +86,15 @@ def mainParser(backend, db, cursor, site, category, hand): #part 4: take appropriate action for each line based on linetype for i in range(len(hand)): if (lineTypes[i]=="cards"): - fpdb_simple.parseCardLine (site, category, lineStreets[i], hand[i], names, cardValues, cardSuits, boardValues, boardSuits) + fpdb_simple.parseCardLine (category, lineStreets[i], hand[i], names, cardValues, cardSuits, boardValues, boardSuits) #if category=="studhilo": # print "hand[i]:", hand[i] # print "cardValues:", cardValues # print "cardSuits:", cardSuits elif (lineTypes[i]=="action"): - fpdb_simple.parseActionLine (site, base, isTourney, hand[i], lineStreets[i], playerIDs, names, actionTypes, allIns, actionAmounts, actionNos, actionTypeByNo) + fpdb_simple.parseActionLine (base, isTourney, hand[i], lineStreets[i], playerIDs, names, actionTypes, allIns, actionAmounts, actionNos, actionTypeByNo) elif (lineTypes[i]=="win"): - fpdb_simple.parseWinLine (hand[i], site, names, winnings, isTourney) + fpdb_simple.parseWinLine (hand[i], names, winnings, isTourney) elif (lineTypes[i]=="rake"): if isTourney: totalRake=0 @@ -106,9 +104,9 @@ def mainParser(backend, db, cursor, site, category, hand): elif (lineTypes[i]=="header" or lineTypes[i]=="rake" or lineTypes[i]=="name" or lineTypes[i]=="ignore"): pass elif (lineTypes[i]=="ante"): - fpdb_simple.parseAnteLine(hand[i], site, isTourney, names, antes) + fpdb_simple.parseAnteLine(hand[i], isTourney, names, antes) elif (lineTypes[i]=="table"): - tableResult=fpdb_simple.parseTableLine(site, base, hand[i]) + tableResult=fpdb_simple.parseTableLine(base, hand[i]) else: raise fpdb_simple.FpdbError("unrecognised lineType:"+lineTypes[i]) maxSeats=tableResult['maxSeats'] @@ -125,7 +123,7 @@ def mainParser(backend, db, cursor, site, category, hand): cursor.execute("SELECT limitType FROM Gametypes WHERE id=%s",(gametypeID, )) limit_type=cursor.fetchone()[0] - fpdb_simple.convert3B4B(site, category, limit_type, actionTypes, actionAmounts) + fpdb_simple.convert3B4B(category, limit_type, actionTypes, actionAmounts) totalWinnings=0 for i in range(len(winnings)): diff --git a/pyfpdb/fpdb_simple.py b/pyfpdb/fpdb_simple.py index d3bb4097..cad7e046 100644 --- a/pyfpdb/fpdb_simple.py +++ b/pyfpdb/fpdb_simple.py @@ -458,7 +458,7 @@ def classifyLines(hand, category, lineTypes, lineStreets): lineStreets.append(currentStreet) #end def classifyLines -def convert3B4B(site, category, limit_type, actionTypes, actionAmounts): +def convert3B4B(category, limit_type, actionTypes, actionAmounts): """calculates the actual bet amounts in the given amount array and changes it accordingly.""" for i in range (len(actionTypes)): for j in range (len(actionTypes[i])): @@ -474,7 +474,7 @@ def convert3B4B(site, category, limit_type, actionTypes, actionAmounts): elif (len(bets)>2): fail=True #todo: run correction for below - if (site=="ps" and category=="holdem" and limit_type=="nl" and len(bets)==3): + if (limit_type=="nl" or limit_type == "pl"): fail=False if fail: @@ -611,7 +611,7 @@ def fillCardArrays(player_count, base, category, card_values, card_suits): #filters out a player that folded before paying ante or blinds. This should be called #before calling the actual hand parser. manipulates hand, no return. -def filterAnteBlindFold(site,hand): +def filterAnteBlindFold(hand): #todo: this'll only get rid of one ante folder, not multiple ones #todo: in tourneys this should not be removed but #print "start of filterAnteBlindFold" @@ -649,7 +649,7 @@ def filterAnteBlindFold(site,hand): #end def filterAnteFold #removes useless lines as well as trailing spaces -def filterCrap(site, hand, isTourney): +def filterCrap(hand, isTourney): #remove two trailing spaces at end of line for i in range (len(hand)): if (hand[i][-1]==' '): @@ -714,12 +714,8 @@ def filterCrap(site, hand, isTourney): toRemove.append(hand[i]) elif (hand[i].find("joins the table at seat ")!=-1): toRemove.append(hand[i]) - elif (hand[i].endswith(" sits down")): - toRemove.append(hand[i]) elif (hand[i].endswith("leaves the table")): toRemove.append(hand[i]) - elif (hand[i].endswith(" stands up")): - toRemove.append(hand[i]) elif (hand[i].find("is high with ")!=-1): toRemove.append(hand[i]) elif (hand[i].endswith("doesn't show hand")): @@ -862,7 +858,7 @@ def isWinLine(line): #end def isWinLine #returns the amount of cash/chips put into the put in the given action line -def parseActionAmount(line, atype, site, isTourney): +def parseActionAmount(line, atype, isTourney): #if (line.endswith(" and is all-in")): # line=line[:-14] #elif (line.endswith(", and is all in")): @@ -878,14 +874,14 @@ def parseActionAmount(line, atype, site, isTourney): amount=0 elif (atype=="check"): amount=0 - elif (atype=="unbet" and site=="ps"): + elif (atype=="unbet"): #print "ps unbet, line:",line pos1=line.find("$")+1 if pos1==0: pos1=line.find("(")+1 pos2=line.find(")") amount=float2int(line[pos1:pos2]) - elif (atype=="bet" and site=="ps" and line.find(": raises $")!=-1 and line.find("to $")!=-1): + elif (atype=="bet" and line.find(": raises $")!=-1 and line.find("to $")!=-1): pos=line.find("to $")+4 amount=float2int(line[pos:]) else: @@ -908,7 +904,7 @@ def parseActionAmount(line, atype, site, isTourney): #doesnt return anything, simply changes the passed arrays action_types and # action_amounts. For stud this expects numeric streets (3-7), for # holdem/omaha it expects predeal, preflop, flop, turn or river -def parseActionLine(site, base, isTourney, line, street, playerIDs, names, action_types, allIns, action_amounts, actionNos, actionTypeByNo): +def parseActionLine(base, isTourney, line, street, playerIDs, names, action_types, allIns, action_amounts, actionNos, actionTypeByNo): if (street=="predeal" or street=="preflop"): street=0 elif (street=="flop"): @@ -927,7 +923,7 @@ def parseActionLine(site, base, isTourney, line, street, playerIDs, names, actio line, allIn=goesAllInOnThisLine(line) atype=parseActionType(line) playerno=recognisePlayerNo(line, names, atype) - amount=parseActionAmount(line, atype, site, isTourney) + amount=parseActionAmount(line, atype, isTourney) action_types[street][playerno].append(atype) allIns[street][playerno].append(allIn) @@ -988,7 +984,7 @@ def parseActionType(line): #end def parseActionType #parses the ante out of the given line and checks which player paid it, updates antes accordingly. -def parseAnteLine(line, site, isTourney, names, antes): +def parseAnteLine(line, isTourney, names, antes): for i in range(len(names)): if (line.startswith(names[i].encode("latin-1"))): #found the ante'er pos=line.rfind("$")+1 @@ -1014,7 +1010,7 @@ def parseBuyin(topline): #parses a card line and changes the passed arrays accordingly #todo: reorganise this messy method -def parseCardLine(site, category, street, line, names, cardValues, cardSuits, boardValues, boardSuits): +def parseCardLine(category, street, line, names, cardValues, cardSuits, boardValues, boardSuits): if (line.startswith("Dealt to ") or line.find(" shows [")!=-1 or line.find("mucked [")!=-1): playerNo=recognisePlayerNo(line, names, "card") #anything but unbet will be ok for that string @@ -1088,7 +1084,7 @@ def parseCardLine(site, category, street, line, names, cardValues, cardSuits, bo raise FpdbError ("unrecognised line:"+line) #end def parseCardLine -def parseCashesAndSeatNos(lines, site): +def parseCashesAndSeatNos(lines): """parses the startCashes and seatNos of each player out of the given lines and returns them as a dictionary of two arrays""" cashes = [] seatNos = [] @@ -1099,8 +1095,7 @@ def parseCashesAndSeatNos(lines, site): pos1=lines[i].rfind("($")+2 if pos1==1: #for tourneys - it's 1 instead of -1 due to adding 2 above pos1=lines[i].rfind("(")+1 - elif (site=="ps"): - pos2=lines[i].find(" in chips") + pos2=lines[i].find(" in chips") cashes.append(float2int(lines[i][pos1:pos2])) return {'startCashes':cashes, 'seatNos':seatNos} #end def parseCashesAndSeatNos @@ -1114,7 +1109,7 @@ def parseFee(topline): #end def parsefee #returns a datetime object with the starttime indicated in the given topline -def parseHandStartTime(topline, site): +def parseHandStartTime(topline): #convert x:13:35 to 0x:13:35 counter=0 while (True): @@ -1258,7 +1253,7 @@ def parseSiteHandNo(topline): return topline[pos1:pos2] #end def parseSiteHandNo -def parseTableLine(site, base, line): +def parseTableLine(base, line): """returns a dictionary with maxSeats and tableName""" pos1=line.find('\'')+1 pos2=line.find('\'', pos1) @@ -1278,7 +1273,7 @@ def parseTourneyNo(topline): #end def parseTourneyNo #parses a win/collect line. manipulates the passed array winnings, no explicit return -def parseWinLine(line, site, names, winnings, isTourney): +def parseWinLine(line, names, winnings, isTourney): #print "parseWinLine: line:",line for i in range(len(names)): if (line.startswith(names[i].encode("latin-1"))): #found a winner @@ -1473,28 +1468,8 @@ def recognisePlayerNo(line, names, atype): raise FpdbError ("failed to recognise player in: "+line+" atype:"+atype) #end def recognisePlayerNo -#returns the site abbreviation for the given site -def recogniseSite(line): - # TODO: Make site agnostic - if (line.startswith("Full Tilt Poker")): - return "ftp" - elif (line.startswith("PokerStars")): - return "ps" - else: - raise FpdbError("failed to recognise site, line:"+line) -#end def recogniseSite - -#returns the ID of the given site -def recogniseSiteID(cursor, site): - if (site=="ftp"): - return 1 - #cursor.execute("SELECT id FROM Sites WHERE name = ('Full Tilt Poker')") - elif (site=="ps"): - return 2 - #cursor.execute("SELECT id FROM Sites WHERE name = ('PokerStars')") - else: - raise FpdbError("invalid site in recogniseSiteID: "+site) - return cursor.fetchall()[0][0] +def recogniseSiteID(): + return 2 #end def recogniseSiteID #removes trailing \n from the given array From ae4704b8dc003a035ab38c44e6bbec6a480c9b58 Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 21 Mar 2009 21:23:51 +0900 Subject: [PATCH 06/19] First working(?) version of siteneutral import --- pyfpdb/FpdbSQLQueries.py | 4 ++-- pyfpdb/fpdb_import.py | 24 ++++++++++++++++++------ pyfpdb/fpdb_parse_logic.py | 4 +--- pyfpdb/fpdb_save_to_db.py | 11 +++++++---- pyfpdb/fpdb_simple.py | 8 ++------ 5 files changed, 30 insertions(+), 21 deletions(-) diff --git a/pyfpdb/FpdbSQLQueries.py b/pyfpdb/FpdbSQLQueries.py index c3260466..55bf8c2e 100644 --- a/pyfpdb/FpdbSQLQueries.py +++ b/pyfpdb/FpdbSQLQueries.py @@ -41,8 +41,8 @@ class FpdbSQLQueries: self.query['list_tables'] = """ """ ################################################################## - # Drop Tables - MySQL, PostgreSQL and SQLite all share same syntax - ################################################################## + # Drop Tables - MySQL, PostgreSQL and SQLite all share same syntax + ################################################################## if(self.dbname == 'MySQL InnoDB') or (self.dbname == 'PostgreSQL') or (self.dbname == 'SQLite'): self.query['drop_table'] = """DROP TABLE IF EXISTS """ diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index e6baea29..fc878746 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -58,13 +58,14 @@ class Importer: self.cursor = None self.filelist = {} self.dirlist = {} + self.siteIds = {} self.addToDirList = {} self.removeFromFileList = {} # to remove deleted files self.monitor = False - self.updated = {} #Time last import was run {file:mtime} + self.updated = {} #Time last import was run {file:mtime} self.lines = None - self.faobs = None #File as one big string - self.pos_in_file = {} # dict to remember how far we have read in the file + self.faobs = None # File as one big string + self.pos_in_file = {} # dict to remember how far we have read in the file #Set defaults self.callHud = self.config.get_import_parameters().get("callFpdbHud") if 'minPrint' not in self.settings: @@ -111,6 +112,18 @@ class Importer: def addImportFile(self, filename, site = "default", filter = "passthrough"): #TODO: test it is a valid file -> put that in config!! self.filelist[filename] = [site] + [filter] + if site not in self.siteIds: + # Get id from Sites table in DB + self.fdb.cursor.execute(self.fdb.sql.query['getSiteId'], (site,)) + result = self.fdb.cursor.fetchall() + if len(result) == 1: + self.siteIds[site] = result[0][0] + else: + if len(result) == 0: + print "[ERROR] Database ID for %s not found" % site + else: + print "[ERROR] More than 1 Database ID found for %s - Multiple currencies not implemented yet" % site + # Called from GuiBulkImport to add a file or directory. def addBulkImportImportFileOrDir(self, inputPath,filter = "passthrough"): @@ -289,9 +302,8 @@ class Importer: print "TODO: implement importing tournament summaries" #self.faobs = readfile(inputFile) #self.parseTourneyHistory() - return 0 + return (0,0,0,1,0) - site=fpdb_simple.recogniseSite(firstline) category=fpdb_simple.recogniseCategory(firstline) startpos=0 @@ -322,7 +334,7 @@ class Importer: try: handsId=fpdb_parse_logic.mainParser(self.settings['db-backend'], self.fdb.db - ,self.fdb.cursor, site, category, hand, self.config) + ,self.fdb.cursor, self.siteIds[site], category, hand, self.config) self.fdb.db.commit() stored+=1 diff --git a/pyfpdb/fpdb_parse_logic.py b/pyfpdb/fpdb_parse_logic.py index dd5d84ea..58cc94df 100644 --- a/pyfpdb/fpdb_parse_logic.py +++ b/pyfpdb/fpdb_parse_logic.py @@ -21,7 +21,7 @@ import fpdb_simple import fpdb_save_to_db #parses a holdem hand -def mainParser(backend, db, cursor, category, hand, config): +def mainParser(backend, db, cursor, siteID, category, hand, config): category = fpdb_simple.recogniseCategory(hand[0]) base = "hold" if category == "holdem" or category == "omahahi" or category == "omahahilo" else "stud" @@ -35,8 +35,6 @@ def mainParser(backend, db, cursor, category, hand, config): #part 1: read hand no and check for duplicate siteHandNo = fpdb_simple.parseSiteHandNo(hand[0]) handStartTime = fpdb_simple.parseHandStartTime(hand[0]) - siteID = fpdb_simple.recogniseSiteID() - #print "parse logic, siteID:",siteID,"site:",site isTourney = fpdb_simple.isTourney(hand[0]) smallBlindLine = 0 diff --git a/pyfpdb/fpdb_save_to_db.py b/pyfpdb/fpdb_save_to_db.py index f13c8032..85468c2b 100644 --- a/pyfpdb/fpdb_save_to_db.py +++ b/pyfpdb/fpdb_save_to_db.py @@ -28,11 +28,12 @@ SQLITE = 4 fastStoreHudCache = True # set this to True to test the new storeHudCache routine -saveActions = False # set this to False to avoid storing action data +saveActions = True # set this to False to avoid storing action data # Pros: speeds up imports # Cons: no action data is saved, so you need to keep the hand histories # variance not available on stats page - + # : No graphs + #stores a stud/razz hand into the database def ring_stud(config, backend, db, cursor, base, category, site_hand_no, gametype_id, hand_start_time ,names, player_ids, start_cashes, antes, card_values, card_suits, winnings, rakes @@ -41,7 +42,7 @@ def ring_stud(config, backend, db, cursor, base, category, site_hand_no, gametyp import_options = config.get_import_parameters() - saveActions = True if import_options['saveActions'] == 'True' else False + saveActions = False if import_options['saveActions'] == 'False' else True fastStoreHudCache = True if import_options['fastStoreHudCache'] == 'True' else False fpdb_simple.fillCardArrays(len(names), base, category, card_values, card_suits) @@ -69,9 +70,11 @@ def ring_holdem_omaha(config, backend, db, cursor, base, category, site_hand_no, """stores a holdem/omaha hand into the database""" import_options = config.get_import_parameters() - saveActions = True if import_options['saveActions'] == 'True' else False + saveActions = False if import_options['saveActions'] == 'False' else True fastStoreHudCache = True if import_options['fastStoreHudCache'] == 'True' else False + #print "DEBUG: saveActions: %s fastStoreHudCache: %s" %(saveActions, fastStoreHudCache) + t0 = time() fpdb_simple.fillCardArrays(len(names), base, category, card_values, card_suits) t1 = time() diff --git a/pyfpdb/fpdb_simple.py b/pyfpdb/fpdb_simple.py index b6e7e237..5a75af45 100644 --- a/pyfpdb/fpdb_simple.py +++ b/pyfpdb/fpdb_simple.py @@ -691,14 +691,13 @@ def filterCrap(hand, isTourney): hand[i] = False elif (hand[i]=="Betting is capped"): hand[i] = False - #site specific variable position filter elif (hand[i].find(" said, \"")!=-1): hand[i] = False + if isTourney: if (hand[i].endswith(" is sitting out") and (not hand[i].startswith("Seat "))): hand[i] = False - elif: - if (hand[i].endswith(": sits out")): + elif (hand[i].endswith(": sits out")): hand[i] = False elif (hand[i].endswith(" is sitting out")): hand[i] = False @@ -1375,9 +1374,6 @@ def recognisePlayerNo(line, names, atype): raise FpdbError ("failed to recognise player in: "+line+" atype:"+atype) #end def recognisePlayerNo -def recogniseSiteID(): - return 2 -#end def recogniseSiteID #removes trailing \n from the given array def removeTrailingEOL(arr): From 497354d4f46becc19a861d8559f7e7ced735c293 Mon Sep 17 00:00:00 2001 From: Worros Date: Tue, 24 Mar 2009 22:58:45 +0900 Subject: [PATCH 07/19] Make BulkImport work only with HHCs Added new section to config for converters for ease of creating the gui. Removed the section in the site line for filter and supported games --- pyfpdb/Configuration.py | 20 ++++++++++++++++++++ pyfpdb/GuiBulkImport.py | 16 +++++++--------- pyfpdb/HUD_config.xml.example | 11 ++++++++--- pyfpdb/fpdb_import.py | 8 ++++---- pyfpdb/fpdb_parse_logic.py | 2 +- 5 files changed, 40 insertions(+), 17 deletions(-) diff --git a/pyfpdb/Configuration.py b/pyfpdb/Configuration.py index 19e62038..3e0ca3dd 100755 --- a/pyfpdb/Configuration.py +++ b/pyfpdb/Configuration.py @@ -194,6 +194,15 @@ class Aux_window: temp = temp + "%s" % self.layout[layout] return temp +class HHC: + def __init__(self, node): + self.site = node.getAttribute("site") + self.converter = node.getAttribute("converter") + + def __str__(self): + return "%s:\t%s" % (self.site, self.converter) + + class Popup: def __init__(self, node): self.name = node.getAttribute("pu_name") @@ -273,6 +282,7 @@ class Config: self.supported_games = {} self.supported_databases = {} self.aux_windows = {} + self.hhcs = {} self.popup_windows = {} # s_sites = doc.getElementsByTagName("supported_sites") @@ -295,6 +305,11 @@ class Config: aw = Aux_window(node = aw_node) self.aux_windows[aw.name] = aw +# s_dbs = doc.getElementsByTagName("mucked_windows") + for hhc_node in doc.getElementsByTagName("hhc"): + hhc = HHC(node = hhc_node) + self.hhcs[hhc.site] = hhc + # s_dbs = doc.getElementsByTagName("popup_windows") for pu_node in doc.getElementsByTagName("pu"): pu = Popup(node = pu_node) @@ -696,6 +711,11 @@ if __name__== "__main__": for w in c.aux_windows.keys(): print c.aux_windows[w] print "----------- END AUX WINDOW FORMATS -----------" + + print "\n----------- HAND HISTORY CONVERTERS -----------" + for w in c.hhcs.keys(): + print c.hhcs[w] + print "----------- END HAND HISTORY CONVERTERS -----------" print "\n----------- POPUP WINDOW FORMATS -----------" for w in c.popup_windows.keys(): diff --git a/pyfpdb/GuiBulkImport.py b/pyfpdb/GuiBulkImport.py index 1a62a37d..35563fe3 100755 --- a/pyfpdb/GuiBulkImport.py +++ b/pyfpdb/GuiBulkImport.py @@ -66,10 +66,10 @@ class GuiBulkImport(): self.importer.setDropIndexes(cb_model[cb_index][0]) else: self.importer.setDropIndexes("auto") - hhc=self.cbfilter.get_model()[self.cbfilter.get_active()][0] + sitename = self.cbfilter.get_model()[self.cbfilter.get_active()][0] self.lab_info.set_text("Importing") - self.importer.addBulkImportImportFileOrDir(self.inputFile,filter=hhc) + self.importer.addBulkImportImportFileOrDir(self.inputFile, site = sitename) self.importer.setCallHud(False) starttime = time() (stored, dups, partial, errs, ttime) = self.importer.runImport() @@ -175,11 +175,9 @@ class GuiBulkImport(): # ComboBox - filter self.cbfilter = gtk.combo_box_new_text() - self.cbfilter.append_text("passthrough") - self.cbfilter.append_text("BetfairToFpdb") - self.cbfilter.append_text("EverleafToFpdb") - self.cbfilter.append_text("FulltiltToFpdb") - self.cbfilter.append_text("PokerStarsToFpdb") + for w in self.config.hhcs: + print w + self.cbfilter.append_text(w) self.cbfilter.set_active(0) self.table.attach(self.cbfilter, 3, 4, 2, 3, xpadding = 10, ypadding = 0, yoptions=gtk.SHRINK) self.cbfilter.show() @@ -223,7 +221,7 @@ def main(argv=None): parser.add_option("-q", "--quiet", action="store_false", dest="gui", default=True, help="don't start gui; deprecated (just give a filename with -f).") parser.add_option("-c", "--convert", dest="filtername", default="passthrough", metavar="FILTER", - help="Conversion filter (*passthrough, FullTiltToFpdb, PokerStarsToFpdb, EverleafToFpdb)") + help="Conversion filter (*Full Tilt Poker, PokerStars, Everleaf)") parser.add_option("-x", "--failOnError", action="store_true", default=False, help="If this option is passed it quits when it encounters any error") parser.add_option("-m", "--minPrint", "--status", dest="minPrint", default="0", type="int", @@ -258,7 +256,7 @@ def main(argv=None): importer = fpdb_import.Importer(False,settings, config) importer.setDropIndexes("auto") importer.setFailOnError(options.failOnError) - importer.addBulkImportImportFileOrDir(os.path.expanduser(options.filename), filter=options.filtername) + importer.addBulkImportImportFileOrDir(os.path.expanduser(options.filename), site=options.filtername) importer.setCallHud(False) importer.runImport() importer.clearFileList() diff --git a/pyfpdb/HUD_config.xml.example b/pyfpdb/HUD_config.xml.example index f7351526..ad7d440b 100644 --- a/pyfpdb/HUD_config.xml.example +++ b/pyfpdb/HUD_config.xml.example @@ -2,7 +2,7 @@ - + @@ -49,7 +49,7 @@ - + @@ -84,7 +84,7 @@ - + @@ -198,6 +198,11 @@ + + + + + diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 5ce23928..68373fb0 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -126,18 +126,18 @@ class Importer: # Called from GuiBulkImport to add a file or directory. - def addBulkImportImportFileOrDir(self, inputPath,filter = "passthrough"): + def addBulkImportImportFileOrDir(self, inputPath, site = "PokerStars"): """Add a file or directory for bulk import""" - + filter = self.config.hhcs[site].converter # Bulk import never monitors # if directory, add all files in it. Otherwise add single file. # TODO: only add sane files? if os.path.isdir(inputPath): for subdir in os.walk(inputPath): for file in subdir[2]: - self.addImportFile(os.path.join(inputPath, subdir[0], file), site="default", filter=filter) + self.addImportFile(os.path.join(inputPath, subdir[0], file), site=site, filter=filter) else: - self.addImportFile(inputPath, site="default", filter=filter) + self.addImportFile(inputPath, site=site, filter=filter) #Add a directory of files to filelist #Only one import directory per site supported. #dirlist is a hash of lists: diff --git a/pyfpdb/fpdb_parse_logic.py b/pyfpdb/fpdb_parse_logic.py index 0badbb82..750b1512 100644 --- a/pyfpdb/fpdb_parse_logic.py +++ b/pyfpdb/fpdb_parse_logic.py @@ -60,7 +60,7 @@ def mainParser(backend, db, cursor, siteID, category, hand, config): fpdb_simple.isAlreadyInDB(cursor, gametypeID, siteHandNo) - hand=fpdb_simple.filterCrap(site, hand, isTourney) + hand=fpdb_simple.filterCrap(hand, isTourney) #part 2: classify lines by type (e.g. cards, action, win, sectionchange) and street fpdb_simple.classifyLines(hand, category, lineTypes, lineStreets) From 20dd3e1aec8295369f4f2af671807b9608a1f6d7 Mon Sep 17 00:00:00 2001 From: Worros Date: Wed, 25 Mar 2009 18:35:19 +0900 Subject: [PATCH 08/19] Make CET timezones work with Stars hands --- pyfpdb/PokerStarsToFpdb.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pyfpdb/PokerStarsToFpdb.py b/pyfpdb/PokerStarsToFpdb.py index 8b905005..f521bbd0 100755 --- a/pyfpdb/PokerStarsToFpdb.py +++ b/pyfpdb/PokerStarsToFpdb.py @@ -128,9 +128,11 @@ follow : whether to tail -f the input""" logging.debug("readHandInfo: %s" % info) for key in info: if key == 'DATETIME': - datetime = info[key].replace(" - "," ") # some are like "2009/02/26 - 15:22:55 ET" - datetime = datetime.replace(" (ET)","") # kludge for now. - datetime = datetime.replace(" ET","") # kludge for now. + #2008/11/12 10:00:48 CET [2008/11/12 4:00:48 ET] + #2008/08/17 - 01:14:43 (ET) + #2008/09/07 06:23:14 ET + m2 = re.search("(?P[0-9]{4})\/(?P[0-9]{2})\/(?P[0-9]{2})[\- ]+(?P[0-9]+):(?P[0-9]+):(?P[0-9]+)", info[key]) + datetime = "%s/%s/%s %s:%s:%s" % (m2.group('Y'), m2.group('M'),m2.group('D'),m2.group('H'),m2.group('MIN'),m2.group('S')) hand.starttime = time.strptime(datetime, "%Y/%m/%d %H:%M:%S") if key == 'HID': hand.handid = info[key] From 710893b186edf4fba36ea1c3749daa38ed37eb48 Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 28 Mar 2009 00:54:30 +0900 Subject: [PATCH 09/19] Fix merge breakage --- pyfpdb/fpdb_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 17bf04f1..1e5049d0 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -338,7 +338,7 @@ class Importer: try: handsId = fpdb_parse_logic.mainParser(self.settings['db-backend'], self.fdb.db - ,self.fdb.cursor, site, category, hand, self.config) + ,self.fdb.cursor, self.siteIds[site], category, hand, self.config) self.fdb.db.commit() stored += 1 From f6ca8c1bc10fce6eb61206ecbbb708eee278f910 Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 28 Mar 2009 19:38:37 +0900 Subject: [PATCH 10/19] GuiBulkImport cli breakage fix --- pyfpdb/GuiBulkImport.py | 2 +- pyfpdb/fpdb_import.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyfpdb/GuiBulkImport.py b/pyfpdb/GuiBulkImport.py index 35563fe3..e90be69a 100755 --- a/pyfpdb/GuiBulkImport.py +++ b/pyfpdb/GuiBulkImport.py @@ -236,7 +236,7 @@ def main(argv=None): if os.name == 'nt': settings['os'] = 'windows' else: settings['os'] = 'linuxmac' - settings.update(config.get_db_parameters('fpdb')) + settings.update(config.get_db_parameters()) settings.update(config.get_tv_parameters()) settings.update(config.get_import_parameters()) settings.update(config.get_default_paths()) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 1e5049d0..2b421e19 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -21,7 +21,7 @@ import os # todo: remove this once import_dir is in fpdb_import import sys -from time import time +from time import time, strftime import traceback import math import datetime From 95add137311ac0f173fb90d64fe62b4a43ed91ce Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 28 Mar 2009 19:39:13 +0900 Subject: [PATCH 11/19] No passthough converter in siteneutral --- pyfpdb/GuiBulkImport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyfpdb/GuiBulkImport.py b/pyfpdb/GuiBulkImport.py index e90be69a..7dbb9858 100755 --- a/pyfpdb/GuiBulkImport.py +++ b/pyfpdb/GuiBulkImport.py @@ -220,7 +220,7 @@ def main(argv=None): help="Input file in quiet mode") parser.add_option("-q", "--quiet", action="store_false", dest="gui", default=True, help="don't start gui; deprecated (just give a filename with -f).") - parser.add_option("-c", "--convert", dest="filtername", default="passthrough", metavar="FILTER", + parser.add_option("-c", "--convert", dest="filtername", default="PokerStars", metavar="FILTER", help="Conversion filter (*Full Tilt Poker, PokerStars, Everleaf)") parser.add_option("-x", "--failOnError", action="store_true", default=False, help="If this option is passed it quits when it encounters any error") From bdd615ede9ec20a6ccac58eb48ed806e0b305f2f Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 28 Mar 2009 20:32:30 +0900 Subject: [PATCH 12/19] utf8 fix to db connection Change mysql connection type to use_unicode. --- pyfpdb/fpdb_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyfpdb/fpdb_db.py b/pyfpdb/fpdb_db.py index 771b79fa..eba87f52 100644 --- a/pyfpdb/fpdb_db.py +++ b/pyfpdb/fpdb_db.py @@ -59,7 +59,7 @@ class fpdb_db: self.database=database if backend==self.MYSQL_INNODB: import MySQLdb - self.db=MySQLdb.connect(host = host, user = user, passwd = password, db = database) + self.db=MySQLdb.connect(host = host, user = user, passwd = password, db = database, use_unicode=True) elif backend==self.PGSQL: import psycopg2 # If DB connection is made over TCP, then the variables From 584c658590e6c78d8ed42e635133a96160aa42b3 Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 11 Apr 2009 03:51:45 +0800 Subject: [PATCH 13/19] Forgot the converter line in merge --- pyfpdb/HUD_config.xml.example | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyfpdb/HUD_config.xml.example b/pyfpdb/HUD_config.xml.example index 32017934..ce397e42 100644 --- a/pyfpdb/HUD_config.xml.example +++ b/pyfpdb/HUD_config.xml.example @@ -13,6 +13,7 @@ site_path="C:/Program Files/PokerStars/" HH_path="C:/Program Files/PokerStars/HandHistory/YOUR SCREEN NAME HERE/" decoder="pokerstars_decode_table" + converter="PokerStarsToFpdb" bgcolor="#000000" fgcolor="#FFFFFF" hudopacity="1.0" @@ -73,6 +74,7 @@ site_path="C:/Program Files/Full Tilt Poker/" HH_path="C:/Program Files/Full Tilt Poker/HandHistory/YOUR SCREEN NAME HERE/" decoder="fulltilt_decode_table" + converter="FullTiltToFpdb" bgcolor="#000000" fgcolor="#FFFFFF" hudopacity="1.0" @@ -121,6 +123,7 @@ site_path="" HH_path="" decoder="everleaf_decode_table" + converter="EverleafToFpdb" supported_games="holdem"> From 5a842c52fa0e6d3eea0b3db140c5f200344b18a8 Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 11 Apr 2009 04:16:36 +0800 Subject: [PATCH 14/19] Add section to example config and remove converter from Change to make building GUI easier from config. --- pyfpdb/HUD_config.xml.example | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pyfpdb/HUD_config.xml.example b/pyfpdb/HUD_config.xml.example index ce397e42..20f38235 100644 --- a/pyfpdb/HUD_config.xml.example +++ b/pyfpdb/HUD_config.xml.example @@ -13,7 +13,6 @@ site_path="C:/Program Files/PokerStars/" HH_path="C:/Program Files/PokerStars/HandHistory/YOUR SCREEN NAME HERE/" decoder="pokerstars_decode_table" - converter="PokerStarsToFpdb" bgcolor="#000000" fgcolor="#FFFFFF" hudopacity="1.0" @@ -74,7 +73,6 @@ site_path="C:/Program Files/Full Tilt Poker/" HH_path="C:/Program Files/Full Tilt Poker/HandHistory/YOUR SCREEN NAME HERE/" decoder="fulltilt_decode_table" - converter="FullTiltToFpdb" bgcolor="#000000" fgcolor="#FFFFFF" hudopacity="1.0" @@ -123,7 +121,6 @@ site_path="" HH_path="" decoder="everleaf_decode_table" - converter="EverleafToFpdb" supported_games="holdem"> @@ -288,6 +285,12 @@ + + + + + + From 2bc564e29255e4364e0c277e490b81510bd49b4c Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 11 Apr 2009 04:18:10 +0800 Subject: [PATCH 15/19] Remove draw from supported list. Olld import code doesn't support draw yet, this patch prevents the bulk importer from crashing if you have a draw hh file in the list accidentally. --- pyfpdb/PokerStarsToFpdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyfpdb/PokerStarsToFpdb.py b/pyfpdb/PokerStarsToFpdb.py index f81d4dc7..300c6071 100755 --- a/pyfpdb/PokerStarsToFpdb.py +++ b/pyfpdb/PokerStarsToFpdb.py @@ -73,7 +73,7 @@ follow : whether to tail -f the input""" ["ring", "hold", "pl"], ["ring", "hold", "fl"], ["ring", "stud", "fl"], - ["ring", "draw", "fl"], + #["ring", "draw", "fl"], ["ring", "omaha", "pl"] ] From b1aade624f182c61fef906753fee348588e118c9 Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 11 Apr 2009 04:28:54 +0800 Subject: [PATCH 16/19] Change default window size because it bugged me --- pyfpdb/fpdb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyfpdb/fpdb.py b/pyfpdb/fpdb.py index 49b3ad82..80fcedae 100755 --- a/pyfpdb/fpdb.py +++ b/pyfpdb/fpdb.py @@ -381,7 +381,7 @@ This program is licensed under the AGPL3, see docs"""+os.sep+"agpl-3.0.txt") self.window.connect("destroy", self.destroy) self.window.set_title("Free Poker DB - v%s or higher" % (VERSION, )) self.window.set_border_width(1) - self.window.set_size_request(1020,400) + self.window.set_default_size(900,720) self.window.set_resizable(True) self.menu_items = ( @@ -449,7 +449,7 @@ This program is licensed under the AGPL3, see docs"""+os.sep+"agpl-3.0.txt") self.tab_main_help(None, None) - self.status_bar = gtk.Label("Status: Connected to "+self.db.get_backend_name()+" database named "+self.db.database+" on host "+self.db.host) + self.status_bar = gtk.Label("Status: Connected to %s database named %s on host %s"%(self.db.get_backend_name(),self.db.database, self.db.host)) self.main_vbox.pack_end(self.status_bar, False, True, 0) self.status_bar.show() From 702cb5851a30f234b735f883584fb7a1f15c54ae Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 11 Apr 2009 11:14:01 +0800 Subject: [PATCH 17/19] Oops - revert converter removal --- pyfpdb/HUD_config.xml.example | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyfpdb/HUD_config.xml.example b/pyfpdb/HUD_config.xml.example index 20f38235..ef30fb9b 100644 --- a/pyfpdb/HUD_config.xml.example +++ b/pyfpdb/HUD_config.xml.example @@ -12,7 +12,8 @@ screen_name="YOUR SCREEN NAME HERE" site_path="C:/Program Files/PokerStars/" HH_path="C:/Program Files/PokerStars/HandHistory/YOUR SCREEN NAME HERE/" - decoder="pokerstars_decode_table" + decoder="pokerstars_decode_table" + converter="PokerStarsToFpdb" bgcolor="#000000" fgcolor="#FFFFFF" hudopacity="1.0" @@ -73,6 +74,7 @@ site_path="C:/Program Files/Full Tilt Poker/" HH_path="C:/Program Files/Full Tilt Poker/HandHistory/YOUR SCREEN NAME HERE/" decoder="fulltilt_decode_table" + converter="FulltiltToFpdb" bgcolor="#000000" fgcolor="#FFFFFF" hudopacity="1.0" @@ -121,6 +123,7 @@ site_path="" HH_path="" decoder="everleaf_decode_table" + converter="EverleafToFpdb" supported_games="holdem"> From 76d7b63b817e7a3afec054a5818f0d88af8b6dc3 Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 11 Apr 2009 14:38:46 +0800 Subject: [PATCH 18/19] Make Grapher use get_supported_sites() properly --- pyfpdb/GuiGraphViewer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyfpdb/GuiGraphViewer.py b/pyfpdb/GuiGraphViewer.py index 5afc4a92..8899c576 100644 --- a/pyfpdb/GuiGraphViewer.py +++ b/pyfpdb/GuiGraphViewer.py @@ -185,7 +185,7 @@ class GuiGraphViewer (threading.Thread): print "self.sites[%s] set to %s" %(site, self.sites[site]) def fillPlayerFrame(self, vbox): - for site in self.conf.supported_sites.keys(): + for site in self.conf.get_supported_sites(): pathHBox = gtk.HBox(False, 0) vbox.pack_start(pathHBox, False, True, 0) pathHBox.show() @@ -194,7 +194,7 @@ class GuiGraphViewer (threading.Thread): self.createPlayerLine(pathHBox, site, player) def fillSitesFrame(self, vbox): - for site in self.conf.supported_sites.keys(): + for site in self.conf.get_supported_sites(): hbox = gtk.HBox(False, 0) vbox.pack_start(hbox, False, True, 0) hbox.show() From 15d7e1fab50bddee5620f6ce60bef01ad2be4211 Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 11 Apr 2009 17:10:00 +0800 Subject: [PATCH 19/19] Add a couple of queries for GUI building --- pyfpdb/FpdbSQLQueries.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pyfpdb/FpdbSQLQueries.py b/pyfpdb/FpdbSQLQueries.py index f1981d71..fe04aec6 100644 --- a/pyfpdb/FpdbSQLQueries.py +++ b/pyfpdb/FpdbSQLQueries.py @@ -1153,6 +1153,12 @@ class FpdbSQLQueries: and hprof2.PlPosition = stats.PlPosition) order by stats.category, stats.limittype, stats.bigBlind, cast(stats.PlPosition as signed) """ + if(self.dbname == 'MySQL InnoDB') or (self.dbname == 'PostgreSQL') or (self.dbname == 'SQLite'): + self.query['getGames'] = """SELECT DISTINCT category from Gametypes""" + + if(self.dbname == 'MySQL InnoDB') or (self.dbname == 'PostgreSQL') or (self.dbname == 'SQLite'): + self.query['getLimits'] = """SELECT DISTINCT bigBlind from Gametypes ORDER by bigBlind DESC""" + if __name__== "__main__": from optparse import OptionParser