From 72dff55f6f38eb984181c7b55e122ad437f73b29 Mon Sep 17 00:00:00 2001 From: Worros Date: Mon, 9 Mar 2009 18:07:04 +0900 Subject: [PATCH 001/118] [KILLFTP 1/x] Tear out FTP specific hand import code --- pyfpdb/fpdb_import.py | 131 +++++++++++++----------------------------- 1 file changed, 41 insertions(+), 90 deletions(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index d86c4214..fde69125 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -293,12 +293,6 @@ class Importer: # print "import_fpdb_file", file, site, self.lines, "\n" return (0,0,0,1,0) - if firstline.find("Tournament Summary")!=-1: - print "TODO: implement importing tournament summaries" - #self.faobs = readfile(inputFile) - #self.parseTourneyHistory() - return 0 - site=fpdb_simple.recogniseSite(firstline) category=fpdb_simple.recogniseCategory(firstline) @@ -316,91 +310,53 @@ class Importer: if (len(hand[0])<2): hand=hand[1:] - cancelled=False - damaged=False - if (site=="ftp"): - for i in range (len(hand)): - if (hand[i].endswith(" has been canceled")): #this is their typo. this is a typo, right? - cancelled=True - - #FTP generates lines looking like: - #Seat 1: IOS Seat 2: kashman59 (big blind) showed [8c 9d] and won ($3.25) with a pair of Eights - #ie. Seat X multiple times on the same line in the summary section, when a new player sits down in the - #middle of the hand. - #TODO: Deal with this properly, either fix the file or make the parsing code work with this line. - if "Seat" in hand[i]: - mo = re.search(" Seat [0-9]+: ", hand[i]) - if mo: - print "mo=", mo, "\nmo.start=", mo.start(),"\nhand[i]=",hand[i] - hand.insert(i+1, hand[i][mo.start()+1:]) - hand[i] = hand[i][0:mo.start()] - - if (len(hand)<3): - pass - #todo: the above 2 lines are kind of a dirty hack, the mentioned circumstances should be handled elsewhere but that doesnt work with DOS/Win EOL. actually this doesnt work. - elif (hand[0].endswith(" (partial)")): #partial hand - do nothing - partial+=1 - elif (hand[1].find("Seat")==-1 and hand[2].find("Seat")==-1 and hand[3].find("Seat")==-1):#todo: should this be or instead of and? - partial+=1 - elif (cancelled or damaged): - partial+=1 - if damaged: - print """ - DEBUG: Partial hand triggered by a line containing 'Seat X:' twice. This is a - bug in the FTP software when a player sits down in the middle of a hand. - Adding a newline after the player name will fix the issue - """ - print "File: %s" %(file) - print "Line: %s" %(startpos) - else: #normal processing - isTourney=fpdb_simple.isTourney(hand[0]) - if not isTourney: - fpdb_simple.filterAnteBlindFold(site,hand) - hand=fpdb_simple.filterCrap(site, hand, isTourney) - self.hand=hand + isTourney=fpdb_simple.isTourney(hand[0]) + if not isTourney: + fpdb_simple.filterAnteBlindFold(site,hand) + hand=fpdb_simple.filterCrap(site, hand, isTourney) + self.hand=hand - try: - handsId=fpdb_parse_logic.mainParser(self.settings['db-backend'], self.fdb.db + try: + handsId=fpdb_parse_logic.mainParser(self.settings['db-backend'], self.fdb.db ,self.fdb.cursor, site, category, hand) - self.fdb.db.commit() + self.fdb.db.commit() - stored+=1 - if self.callHud: - #print "call to HUD here. handsId:",handsId - #pipe the Hands.id out to the HUD - self.caller.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep) - except fpdb_simple.DuplicateError: - duplicates+=1 - except (ValueError), fe: - errors+=1 - self.printEmailErrorMessage(errors, file, hand) - - if (self.settings['failOnError']): - self.fdb.db.commit() #dont remove this, in case hand processing was cancelled. - raise - except (fpdb_simple.FpdbError), fe: - errors+=1 - self.printEmailErrorMessage(errors, file, hand) + stored+=1 + if self.callHud: + #print "call to HUD here. handsId:",handsId + #pipe the Hands.id out to the HUD + self.caller.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep) + except fpdb_simple.DuplicateError: + duplicates+=1 + except (ValueError), fe: + errors+=1 + self.printEmailErrorMessage(errors, file, hand) - #fe.printStackTrace() #todo: get stacktrace - self.fdb.db.rollback() + if (self.settings['failOnError']): + self.fdb.db.commit() #dont remove this, in case hand processing was cancelled. + raise + except (fpdb_simple.FpdbError), fe: + errors+=1 + self.printEmailErrorMessage(errors, file, hand) + + self.fdb.db.rollback() - if (self.settings['failOnError']): - self.fdb.db.commit() #dont remove this, in case hand processing was cancelled. - raise - if (self.settings['minPrint']!=0): - if ((stored+duplicates+partial+errors)%self.settings['minPrint']==0): - print "stored:", stored, "duplicates:", duplicates, "partial:", partial, "errors:", errors + if (self.settings['failOnError']): + self.fdb.db.commit() #dont remove this, in case hand processing was cancelled. + raise + if (self.settings['minPrint']!=0): + if ((stored+duplicates+errors)%self.settings['minPrint']==0): + print "stored:", stored, "duplicates:", duplicates, "errors:", errors - if (self.settings['handCount']!=0): - if ((stored+duplicates+partial+errors)>=self.settings['handCount']): - if (not self.settings['quiet']): - print "quitting due to reaching the amount of hands to be imported" - print "Total stored:", stored, "duplicates:", duplicates, "partial/damaged:", partial, "errors:", errors, " time:", (time() - starttime) - sys.exit(0) - startpos=endpos + if (self.settings['handCount']!=0): + if ((stored+duplicates+errors)>=self.settings['handCount']): + if (not self.settings['quiet']): + print "quitting due to reaching the amount of hands to be imported" + print "Total stored:", stored, "duplicates:", duplicates, "errors:", errors, " time:", (time() - starttime) + sys.exit(0) + startpos=endpos ttime = time() - starttime - print "\rTotal stored:", stored, "duplicates:", duplicates, "partial:", partial, "errors:", errors, " time:", ttime + print "\rTotal stored:", stored, "duplicates:", duplicates, "errors:", errors, " time:", ttime if stored==0: if duplicates>0: @@ -411,16 +367,11 @@ class Importer: else: print "failed to read a single hand from file:", inputFile handsId=0 - #todo: this will cause return of an unstored hand number if the last hand was error or partial + #todo: this will cause return of an unstored hand number if the last hand was error self.fdb.db.commit() self.handsId=handsId return (stored, duplicates, partial, errors, ttime) - def parseTourneyHistory(self): - print "Tourney history parser stub" - #Find tournament boundaries. - #print self.foabs - def printEmailErrorMessage(self, errors, filename, line): traceback.print_exc(file=sys.stderr) print "Error No.",errors,", please send the hand causing this to steffen@sycamoretest.info so I can fix it." From 58445c0a1b72fb16f3db97bb13a798db1d896feb Mon Sep 17 00:00:00 2001 From: Worros Date: Mon, 9 Mar 2009 18:21:22 +0900 Subject: [PATCH 002/118] [KILLFTP 2/x] More ftp specific code removal --- pyfpdb/fpdb_parse_logic.py | 2 - pyfpdb/fpdb_simple.py | 140 ++++++++----------------------------- 2 files changed, 29 insertions(+), 113 deletions(-) diff --git a/pyfpdb/fpdb_parse_logic.py b/pyfpdb/fpdb_parse_logic.py index dab3bcc4..c5076283 100644 --- a/pyfpdb/fpdb_parse_logic.py +++ b/pyfpdb/fpdb_parse_logic.py @@ -111,8 +111,6 @@ def mainParser(backend, db, cursor, site, category, hand): tableResult=fpdb_simple.parseTableLine(site, base, hand[i]) else: raise fpdb_simple.FpdbError("unrecognised lineType:"+lineTypes[i]) - if site=="ftp": - tableResult=fpdb_simple.parseTableLine(site, base, hand[0]) maxSeats=tableResult['maxSeats'] tableName=tableResult['tableName'] #print "before part5, antes:", antes diff --git a/pyfpdb/fpdb_simple.py b/pyfpdb/fpdb_simple.py index dbf53e93..d3bb4097 100644 --- a/pyfpdb/fpdb_simple.py +++ b/pyfpdb/fpdb_simple.py @@ -476,8 +476,6 @@ def convert3B4B(site, category, limit_type, actionTypes, actionAmounts): #todo: run correction for below if (site=="ps" and category=="holdem" and limit_type=="nl" and len(bets)==3): fail=False - if (site=="ftp" and category=="omahahi" and limit_type=="pl" and len(bets)==3): - fail=False if fail: print "len(bets)>2 in convert3B4B, i didnt think this is possible. i:",i,"j:",j,"k:",k @@ -689,18 +687,8 @@ def filterCrap(site, hand, isTourney): toRemove.append(hand[i]) elif (hand[i].endswith("is disconnected")): toRemove.append(hand[i]) - elif (hand[i].endswith(" is feeling angry")): - toRemove.append(hand[i]) - elif (hand[i].endswith(" is feeling confused")): - toRemove.append(hand[i]) - elif (hand[i].endswith(" is feeling happy")): - toRemove.append(hand[i]) - elif (hand[i].endswith(" is feeling normal")): - toRemove.append(hand[i]) elif (hand[i].find(" is low with [")!=-1): toRemove.append(hand[i]) - #elif (hand[i].find("-max Seat #")!=-1 and hand[i].find(" is the button")!=-1): - # toRemove.append(hand[i]) elif (hand[i].endswith(" mucks")): toRemove.append(hand[i]) elif (hand[i].endswith(": mucks hand")): @@ -711,8 +699,6 @@ def filterCrap(site, hand, isTourney): toRemove.append(hand[i]) elif (hand[i].find(" shows ")!=-1 and hand[i].find("[")==-1): toRemove.append(hand[i]) - #elif (hand[i].startswith("Table '") and hand[i].endswith("-max")): - # toRemove.append(hand[i]) elif (hand[i].startswith("The button is in seat #")): toRemove.append(hand[i]) #above is alphabetic, reorder below if bored @@ -747,8 +733,6 @@ def filterCrap(site, hand, isTourney): #site specific variable position filter elif (hand[i].find(" said, \"")!=-1): toRemove.append(hand[i]) - elif (hand[i].find(": ")!=-1 and site=="ftp" and hand[i].find("Seat ")==-1 and hand[i].find(": Table")==-1): #filter ftp chat - toRemove.append(hand[i]) if isTourney: if (hand[i].endswith(" is sitting out") and (not hand[i].startswith("Seat "))): toRemove.append(hand[i]) @@ -857,14 +841,6 @@ def isWinLine(line): return True elif (line.find("ties for the low side pot")!=-1): return True - elif (line.find("ties for the main pot")!=-1): #for ftp tied main pot of split pot - return True - elif (line.find("ties for the pot")!=-1): #for ftp tie - return True - elif (line.find("ties for the side pot")!=-1): #for ftp tied split pots - return True - elif (line.find("wins side pot #")!=-1): #for ftp multi split pots - return True elif (line.find("wins the low main pot")!=-1): return True elif (line.find("wins the low pot")!=-1): @@ -879,8 +855,6 @@ def isWinLine(line): return True elif (line.find("wins the main pot")!=-1): return True - elif (line.find("wins the side pot")!=-1): #for ftp split pots - return True elif (line.find("collected")!=-1): return True else: @@ -904,10 +878,6 @@ def parseActionAmount(line, atype, site, isTourney): amount=0 elif (atype=="check"): amount=0 - elif (atype=="unbet" and site=="ftp"): - pos1=line.find("$")+1 - pos2=line.find(" returned to") - amount=float2int(line[pos1:pos2]) elif (atype=="unbet" and site=="ps"): #print "ps unbet, line:",line pos1=line.find("$")+1 @@ -1129,8 +1099,6 @@ def parseCashesAndSeatNos(lines, site): pos1=lines[i].rfind("($")+2 if pos1==1: #for tourneys - it's 1 instead of -1 due to adding 2 above pos1=lines[i].rfind("(")+1 - if (site=="ftp"): - pos2=lines[i].rfind(")") elif (site=="ps"): pos2=lines[i].find(" in chips") cashes.append(float2int(lines[i][pos1:pos2])) @@ -1157,41 +1125,25 @@ def parseHandStartTime(topline, site): if counter==10: break isUTC=False - if site=="ftp": - # Full Tilt Sit'n'Go - # Full Tilt Poker Game #10311865543: $1 + $0.25 Sit & Go (78057629), Table 1 - 25/50 - No Limit Hold'em - 0:07:45 ET - 2009/01/29 - # Cash Game: - # Full Tilt Poker Game #9403951181: Table CR - tay - $0.05/$0.10 - No Limit Hold'em - 9:40:20 ET - 2008/12/09 - # Full Tilt Poker Game #9468383505: Table Bike (deep 6) - $0.05/$0.10 - No Limit Hold'em - 5:09:36 ET - 2008/12/13 - pos = topline.find(" ", len(topline)-26)+1 - tmp = topline[pos:] - - rexx = '(?P
[0-9]+):(?P[0-9]+):(?P[0-9]+) ET [\- ]+(?P[0-9]{4})\/(?P[0-9]{2})\/(?P[0-9]{2})' - m = re.search(rexx,tmp) - result = datetime.datetime(int(m.group('YEAR')), int(m.group('MON')), int(m.group('DAY')), int(m.group('HR')), int(m.group('MIN')), int(m.group('SEC'))) - elif site=="ps": - if topline.find("UTC")!=-1: - pos1 = topline.find("-")+2 - pos2 = topline.find("UTC") - tmp=topline[pos1:pos2] - isUTC=True - else: - tmp=topline - #print "parsehandStartTime, tmp:", tmp - pos = tmp.find("-")+2 - tmp = tmp[pos:] - #Need to match either - # 2008/09/07 06:23:14 ET or - # 2008/08/17 - 01:14:43 (ET) or - # 2008/11/12 9:33:31 CET [2008/11/12 3:33:31 ET] - rexx = '(?P[0-9]{4})\/(?P[0-9]{2})\/(?P[0-9]{2})[\- ]+(?P
[0-9]+):(?P[0-9]+):(?P[0-9]+)' - m = re.search(rexx,tmp) - #print "year:", int(m.group('YEAR')), "month", int(m.group('MON')), "day", int(m.group('DAY')), "hour", int(m.group('HR')), "minute", int(m.group('MIN')), "second", int(m.group('SEC')) - result = datetime.datetime(int(m.group('YEAR')), int(m.group('MON')), int(m.group('DAY')), int(m.group('HR')), int(m.group('MIN')), int(m.group('SEC'))) + if topline.find("UTC")!=-1: + pos1 = topline.find("-")+2 + pos2 = topline.find("UTC") + tmp=topline[pos1:pos2] + isUTC=True else: - raise FpdbError("invalid site in parseHandStartTime") + tmp=topline + #print "parsehandStartTime, tmp:", tmp + pos = tmp.find("-")+2 + tmp = tmp[pos:] + #Need to match either + # 2008/09/07 06:23:14 ET or + # 2008/08/17 - 01:14:43 (ET) or + # 2008/11/12 9:33:31 CET [2008/11/12 3:33:31 ET] + rexx = '(?P[0-9]{4})\/(?P[0-9]{2})\/(?P[0-9]{2})[\- ]+(?P
[0-9]+):(?P[0-9]+):(?P[0-9]+)' + m = re.search(rexx,tmp) + result = datetime.datetime(int(m.group('YEAR')), int(m.group('MON')), int(m.group('DAY')), int(m.group('HR')), int(m.group('MIN')), int(m.group('SEC'))) - if (site=="ftp" or site=="ps") and not isUTC: #these use US ET + if not isUTC: #these use US ET result+=datetime.timedelta(hours=5) return result @@ -1308,40 +1260,13 @@ def parseSiteHandNo(topline): def parseTableLine(site, base, line): """returns a dictionary with maxSeats and tableName""" - if site=="ps": - pos1=line.find('\'')+1 - pos2=line.find('\'', pos1) - #print "table:",line[pos1:pos2] - pos3=pos2+2 - pos4=line.find("-max") - #print "seats:",line[pos3:pos4] - return {'maxSeats':int(line[pos3:pos4]), 'tableName':line[pos1:pos2]} - elif site=="ftp": - pos1=line.find("Table ")+6 - pos2=line.find("-")-1 - if base=="hold": - maxSeats=9 - elif base=="stud": - maxSeats=8 - - if line.find("6 max")!=-1: - maxSeats=6 - elif line.find("4 max")!=-1: - maxSeats=4 - elif line.find("heads up")!=-1: - maxSeats=2 - - tableName = line[pos1:pos2] - for pattern in [' \(6 max\)', ' \(heads up\)', ' \(deep\)', - ' \(deep hu\)', ' \(deep 6\)', ' \(2\)', - ' \(edu\)', ' \(edu, 6 max\)', ' \(6\)', - ' \(speed\)', - ' no all-in', ' fast', ',', ' 50BB min', '\s+$']: - tableName = re.sub(pattern, '', tableName) - tableName = tableName.rstrip() - return {'maxSeats':maxSeats, 'tableName':tableName} - else: - raise FpdbError("invalid site ID") + pos1=line.find('\'')+1 + pos2=line.find('\'', pos1) + #print "table:",line[pos1:pos2] + pos3=pos2+2 + pos4=line.find("-max") + #print "seats:",line[pos3:pos4] + return {'maxSeats':int(line[pos3:pos4]), 'tableName':line[pos1:pos2]} #end def parseTableLine #returns the hand no assigned by the poker site @@ -1359,17 +1284,11 @@ def parseWinLine(line, site, names, winnings, isTourney): if (line.startswith(names[i].encode("latin-1"))): #found a winner if isTourney: pos1=line.rfind("collected ")+10 - if (site=="ftp"): - pos2=line.find(")", pos1) - elif (site=="ps"): - pos2=line.find(" ", pos1) + pos2=line.find(" ", pos1) winnings[i]+=int(line[pos1:pos2]) else: pos1=line.rfind("$")+1 - if (site=="ftp"): - pos2=line.find(")", pos1) - elif (site=="ps"): - pos2=line.find(" ", pos1) + pos2=line.find(" ", pos1) winnings[i]+=float2int(line[pos1:pos2]) #end def parseWinLine @@ -1413,10 +1332,8 @@ def recogniseGametypeID(backend, db, cursor, topline, smallBlindLine, site_id, c pos1=pos2+2 if isTourney: pos1-=1 - if (site_id==1): #ftp - pos2=topline.find(" ", pos1) - elif (site_id==2): #ps - pos2=topline.find(")") + #ps + pos2=topline.find(")") if pos2<=pos1: pos2=topline.find(")", pos1) @@ -1558,6 +1475,7 @@ def recognisePlayerNo(line, names, atype): #returns the site abbreviation for the given site def recogniseSite(line): + # TODO: Make site agnostic if (line.startswith("Full Tilt Poker")): return "ftp" elif (line.startswith("PokerStars")): From 14b58d6ad538dfb7e930e07b84176d12b1cb8d47 Mon Sep 17 00:00:00 2001 From: Worros Date: Tue, 10 Mar 2009 21:36:52 +0900 Subject: [PATCH 003/118] [KILLFTP 3/x] Fix screwup from patch 1 --- pyfpdb/fpdb_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index fde69125..03de8554 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -354,7 +354,7 @@ class Importer: print "quitting due to reaching the amount of hands to be imported" print "Total stored:", stored, "duplicates:", duplicates, "errors:", errors, " time:", (time() - starttime) sys.exit(0) - startpos=endpos + startpos=endpos ttime = time() - starttime print "\rTotal stored:", stored, "duplicates:", duplicates, "errors:", errors, " time:", ttime From 161485e44e5886d21d5ea3e67ee673e4ce6f7bb6 Mon Sep 17 00:00:00 2001 From: Worros Date: Tue, 10 Mar 2009 22:00:37 +0900 Subject: [PATCH 004/118] [KILLFTP 4/x] paper bag fix caused iun patch 1. This code needs replacing --- pyfpdb/fpdb_import.py | 98 +++++++++++++++++++++++-------------------- 1 file changed, 52 insertions(+), 46 deletions(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 03de8554..c4c4e261 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -208,7 +208,7 @@ class Importer: #if os.path.isdir(file): #self.import_file_dict(file, self.filelist[file][0], self.filelist[file][1]) - self.addToDirList = filter(lambda x: self.addImportDirectory(x, True, self.addToDirList[x][0], self.addToDirList[x][1]), self.addToDirList) + self.addToDirList = filter(lambda x: self.addImportDirectory(x, True, self.addToDirList[x][0], self.addToDirList[x][1]), self.addToDirList) for file in self.removeFromFileList: if file in self.filelist: @@ -302,58 +302,64 @@ class Importer: partial=0 #counter errors=0 #counter - for i in xrange (len(self.lines)): #main loop, iterates through the lines of a file and calls the appropriate parser method - if (len(self.lines[i])<2): + for i in xrange (len(self.lines)): + if (len(self.lines[i])<2): #Wierd way to detect for '\r\n' or '\n' endpos=i hand=self.lines[startpos:endpos] if (len(hand[0])<2): hand=hand[1:] + - isTourney=fpdb_simple.isTourney(hand[0]) - if not isTourney: - fpdb_simple.filterAnteBlindFold(site,hand) - hand=fpdb_simple.filterCrap(site, hand, isTourney) - self.hand=hand + if (len(hand)<3): + pass + #TODO: This is ugly - we didn't actually find the start of the + # hand with the outer loop so we test again... + else: + isTourney=fpdb_simple.isTourney(hand[0]) + if not isTourney: + fpdb_simple.filterAnteBlindFold(site,hand) + hand=fpdb_simple.filterCrap(site, hand, isTourney) + self.hand=hand + + try: + handsId=fpdb_parse_logic.mainParser(self.settings['db-backend'], self.fdb.db + ,self.fdb.cursor, site, category, hand) + self.fdb.db.commit() + + stored+=1 + if self.callHud: + #print "call to HUD here. handsId:",handsId + #pipe the Hands.id out to the HUD + self.caller.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep) + except fpdb_simple.DuplicateError: + duplicates+=1 + except (ValueError), fe: + errors+=1 + self.printEmailErrorMessage(errors, file, hand) + + if (self.settings['failOnError']): + self.fdb.db.commit() #dont remove this, in case hand processing was cancelled. + raise + except (fpdb_simple.FpdbError), fe: + errors+=1 + self.printEmailErrorMessage(errors, file, hand) + + self.fdb.db.rollback() + + if (self.settings['failOnError']): + self.fdb.db.commit() #dont remove this, in case hand processing was cancelled. + raise + if (self.settings['minPrint']!=0): + if ((stored+duplicates+errors)%self.settings['minPrint']==0): + print "stored:", stored, "duplicates:", duplicates, "errors:", errors - try: - handsId=fpdb_parse_logic.mainParser(self.settings['db-backend'], self.fdb.db - ,self.fdb.cursor, site, category, hand) - self.fdb.db.commit() - - stored+=1 - if self.callHud: - #print "call to HUD here. handsId:",handsId - #pipe the Hands.id out to the HUD - self.caller.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep) - except fpdb_simple.DuplicateError: - duplicates+=1 - except (ValueError), fe: - errors+=1 - self.printEmailErrorMessage(errors, file, hand) - - if (self.settings['failOnError']): - self.fdb.db.commit() #dont remove this, in case hand processing was cancelled. - raise - except (fpdb_simple.FpdbError), fe: - errors+=1 - self.printEmailErrorMessage(errors, file, hand) - - self.fdb.db.rollback() - - if (self.settings['failOnError']): - self.fdb.db.commit() #dont remove this, in case hand processing was cancelled. - raise - if (self.settings['minPrint']!=0): - if ((stored+duplicates+errors)%self.settings['minPrint']==0): - print "stored:", stored, "duplicates:", duplicates, "errors:", errors - - if (self.settings['handCount']!=0): - if ((stored+duplicates+errors)>=self.settings['handCount']): - if (not self.settings['quiet']): - print "quitting due to reaching the amount of hands to be imported" - print "Total stored:", stored, "duplicates:", duplicates, "errors:", errors, " time:", (time() - starttime) - sys.exit(0) + if (self.settings['handCount']!=0): + if ((stored+duplicates+errors)>=self.settings['handCount']): + if (not self.settings['quiet']): + print "quitting due to reaching the amount of hands to be imported" + print "Total stored:", stored, "duplicates:", duplicates, "errors:", errors, " time:", (time() - starttime) + sys.exit(0) startpos=endpos ttime = time() - starttime print "\rTotal stored:", stored, "duplicates:", duplicates, "errors:", errors, " time:", ttime From 264b18863339013ee65956a58c490650b05e7496 Mon Sep 17 00:00:00 2001 From: Worros Date: Tue, 10 Mar 2009 23:03:02 +0900 Subject: [PATCH 005/118] [KILLFTP 5/x] Remove references to the named site "PokerStars" or "Full Tilt Poker" --- pyfpdb/fpdb_import.py | 8 ++--- pyfpdb/fpdb_parse_logic.py | 22 +++++++------- pyfpdb/fpdb_simple.py | 61 +++++++++++--------------------------- 3 files changed, 32 insertions(+), 59 deletions(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index c4c4e261..dde5ec6f 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -293,7 +293,7 @@ class Importer: # print "import_fpdb_file", file, site, self.lines, "\n" return (0,0,0,1,0) - site=fpdb_simple.recogniseSite(firstline) + # site=fpdb_simple.recogniseSite(firstline) category=fpdb_simple.recogniseCategory(firstline) startpos=0 @@ -318,13 +318,13 @@ class Importer: else: isTourney=fpdb_simple.isTourney(hand[0]) if not isTourney: - fpdb_simple.filterAnteBlindFold(site,hand) - hand=fpdb_simple.filterCrap(site, hand, isTourney) + fpdb_simple.filterAnteBlindFold(hand) + hand=fpdb_simple.filterCrap(hand, isTourney) self.hand=hand try: handsId=fpdb_parse_logic.mainParser(self.settings['db-backend'], self.fdb.db - ,self.fdb.cursor, site, category, hand) + ,self.fdb.cursor, category, hand) self.fdb.db.commit() stored+=1 diff --git a/pyfpdb/fpdb_parse_logic.py b/pyfpdb/fpdb_parse_logic.py index c5076283..a1f2540f 100644 --- a/pyfpdb/fpdb_parse_logic.py +++ b/pyfpdb/fpdb_parse_logic.py @@ -21,7 +21,7 @@ import fpdb_simple import fpdb_save_to_db #parses a holdem hand -def mainParser(backend, db, cursor, site, category, hand): +def mainParser(backend, db, cursor, category, hand): category=fpdb_simple.recogniseCategory(hand[0]) if (category=="holdem" or category=="omahahi" or category=="omahahilo"): base="hold" @@ -35,8 +35,8 @@ def mainParser(backend, db, cursor, site, category, hand): #part 1: read hand no and check for duplicate siteHandNo=fpdb_simple.parseSiteHandNo(hand[0]) - handStartTime=fpdb_simple.parseHandStartTime(hand[0], site) - siteID=fpdb_simple.recogniseSiteID(cursor, site) + handStartTime=fpdb_simple.parseHandStartTime(hand[0]) + siteID=fpdb_simple.recogniseSiteID() #print "parse logic, siteID:",siteID,"site:",site isTourney=fpdb_simple.isTourney(hand[0]) @@ -51,8 +51,6 @@ def mainParser(backend, db, cursor, site, category, hand): #print "small blind line:",smallBlindLine gametypeID=fpdb_simple.recogniseGametypeID(backend, db, cursor, hand[0], hand[smallBlindLine], siteID, category, isTourney) if isTourney: - if site!="ps": - raise fpdb_simple.FpdbError("tourneys are only supported on PS right now") siteTourneyNo=fpdb_simple.parseTourneyNo(hand[0]) buyin=fpdb_simple.parseBuyin(hand[0]) fee=fpdb_simple.parseFee(hand[0]) @@ -75,7 +73,7 @@ def mainParser(backend, db, cursor, site, category, hand): seatLines.append(hand[i]) names=fpdb_simple.parseNames(seatLines) playerIDs = fpdb_simple.recognisePlayerIDs(cursor, names, siteID) - tmp=fpdb_simple.parseCashesAndSeatNos(seatLines, site) + tmp=fpdb_simple.parseCashesAndSeatNos(seatLines) startCashes=tmp['startCashes'] seatNos=tmp['seatNos'] @@ -88,15 +86,15 @@ def mainParser(backend, db, cursor, site, category, hand): #part 4: take appropriate action for each line based on linetype for i in range(len(hand)): if (lineTypes[i]=="cards"): - fpdb_simple.parseCardLine (site, category, lineStreets[i], hand[i], names, cardValues, cardSuits, boardValues, boardSuits) + fpdb_simple.parseCardLine (category, lineStreets[i], hand[i], names, cardValues, cardSuits, boardValues, boardSuits) #if category=="studhilo": # print "hand[i]:", hand[i] # print "cardValues:", cardValues # print "cardSuits:", cardSuits elif (lineTypes[i]=="action"): - fpdb_simple.parseActionLine (site, base, isTourney, hand[i], lineStreets[i], playerIDs, names, actionTypes, allIns, actionAmounts, actionNos, actionTypeByNo) + fpdb_simple.parseActionLine (base, isTourney, hand[i], lineStreets[i], playerIDs, names, actionTypes, allIns, actionAmounts, actionNos, actionTypeByNo) elif (lineTypes[i]=="win"): - fpdb_simple.parseWinLine (hand[i], site, names, winnings, isTourney) + fpdb_simple.parseWinLine (hand[i], names, winnings, isTourney) elif (lineTypes[i]=="rake"): if isTourney: totalRake=0 @@ -106,9 +104,9 @@ def mainParser(backend, db, cursor, site, category, hand): elif (lineTypes[i]=="header" or lineTypes[i]=="rake" or lineTypes[i]=="name" or lineTypes[i]=="ignore"): pass elif (lineTypes[i]=="ante"): - fpdb_simple.parseAnteLine(hand[i], site, isTourney, names, antes) + fpdb_simple.parseAnteLine(hand[i], isTourney, names, antes) elif (lineTypes[i]=="table"): - tableResult=fpdb_simple.parseTableLine(site, base, hand[i]) + tableResult=fpdb_simple.parseTableLine(base, hand[i]) else: raise fpdb_simple.FpdbError("unrecognised lineType:"+lineTypes[i]) maxSeats=tableResult['maxSeats'] @@ -125,7 +123,7 @@ def mainParser(backend, db, cursor, site, category, hand): cursor.execute("SELECT limitType FROM Gametypes WHERE id=%s",(gametypeID, )) limit_type=cursor.fetchone()[0] - fpdb_simple.convert3B4B(site, category, limit_type, actionTypes, actionAmounts) + fpdb_simple.convert3B4B(category, limit_type, actionTypes, actionAmounts) totalWinnings=0 for i in range(len(winnings)): diff --git a/pyfpdb/fpdb_simple.py b/pyfpdb/fpdb_simple.py index d3bb4097..cad7e046 100644 --- a/pyfpdb/fpdb_simple.py +++ b/pyfpdb/fpdb_simple.py @@ -458,7 +458,7 @@ def classifyLines(hand, category, lineTypes, lineStreets): lineStreets.append(currentStreet) #end def classifyLines -def convert3B4B(site, category, limit_type, actionTypes, actionAmounts): +def convert3B4B(category, limit_type, actionTypes, actionAmounts): """calculates the actual bet amounts in the given amount array and changes it accordingly.""" for i in range (len(actionTypes)): for j in range (len(actionTypes[i])): @@ -474,7 +474,7 @@ def convert3B4B(site, category, limit_type, actionTypes, actionAmounts): elif (len(bets)>2): fail=True #todo: run correction for below - if (site=="ps" and category=="holdem" and limit_type=="nl" and len(bets)==3): + if (limit_type=="nl" or limit_type == "pl"): fail=False if fail: @@ -611,7 +611,7 @@ def fillCardArrays(player_count, base, category, card_values, card_suits): #filters out a player that folded before paying ante or blinds. This should be called #before calling the actual hand parser. manipulates hand, no return. -def filterAnteBlindFold(site,hand): +def filterAnteBlindFold(hand): #todo: this'll only get rid of one ante folder, not multiple ones #todo: in tourneys this should not be removed but #print "start of filterAnteBlindFold" @@ -649,7 +649,7 @@ def filterAnteBlindFold(site,hand): #end def filterAnteFold #removes useless lines as well as trailing spaces -def filterCrap(site, hand, isTourney): +def filterCrap(hand, isTourney): #remove two trailing spaces at end of line for i in range (len(hand)): if (hand[i][-1]==' '): @@ -714,12 +714,8 @@ def filterCrap(site, hand, isTourney): toRemove.append(hand[i]) elif (hand[i].find("joins the table at seat ")!=-1): toRemove.append(hand[i]) - elif (hand[i].endswith(" sits down")): - toRemove.append(hand[i]) elif (hand[i].endswith("leaves the table")): toRemove.append(hand[i]) - elif (hand[i].endswith(" stands up")): - toRemove.append(hand[i]) elif (hand[i].find("is high with ")!=-1): toRemove.append(hand[i]) elif (hand[i].endswith("doesn't show hand")): @@ -862,7 +858,7 @@ def isWinLine(line): #end def isWinLine #returns the amount of cash/chips put into the put in the given action line -def parseActionAmount(line, atype, site, isTourney): +def parseActionAmount(line, atype, isTourney): #if (line.endswith(" and is all-in")): # line=line[:-14] #elif (line.endswith(", and is all in")): @@ -878,14 +874,14 @@ def parseActionAmount(line, atype, site, isTourney): amount=0 elif (atype=="check"): amount=0 - elif (atype=="unbet" and site=="ps"): + elif (atype=="unbet"): #print "ps unbet, line:",line pos1=line.find("$")+1 if pos1==0: pos1=line.find("(")+1 pos2=line.find(")") amount=float2int(line[pos1:pos2]) - elif (atype=="bet" and site=="ps" and line.find(": raises $")!=-1 and line.find("to $")!=-1): + elif (atype=="bet" and line.find(": raises $")!=-1 and line.find("to $")!=-1): pos=line.find("to $")+4 amount=float2int(line[pos:]) else: @@ -908,7 +904,7 @@ def parseActionAmount(line, atype, site, isTourney): #doesnt return anything, simply changes the passed arrays action_types and # action_amounts. For stud this expects numeric streets (3-7), for # holdem/omaha it expects predeal, preflop, flop, turn or river -def parseActionLine(site, base, isTourney, line, street, playerIDs, names, action_types, allIns, action_amounts, actionNos, actionTypeByNo): +def parseActionLine(base, isTourney, line, street, playerIDs, names, action_types, allIns, action_amounts, actionNos, actionTypeByNo): if (street=="predeal" or street=="preflop"): street=0 elif (street=="flop"): @@ -927,7 +923,7 @@ def parseActionLine(site, base, isTourney, line, street, playerIDs, names, actio line, allIn=goesAllInOnThisLine(line) atype=parseActionType(line) playerno=recognisePlayerNo(line, names, atype) - amount=parseActionAmount(line, atype, site, isTourney) + amount=parseActionAmount(line, atype, isTourney) action_types[street][playerno].append(atype) allIns[street][playerno].append(allIn) @@ -988,7 +984,7 @@ def parseActionType(line): #end def parseActionType #parses the ante out of the given line and checks which player paid it, updates antes accordingly. -def parseAnteLine(line, site, isTourney, names, antes): +def parseAnteLine(line, isTourney, names, antes): for i in range(len(names)): if (line.startswith(names[i].encode("latin-1"))): #found the ante'er pos=line.rfind("$")+1 @@ -1014,7 +1010,7 @@ def parseBuyin(topline): #parses a card line and changes the passed arrays accordingly #todo: reorganise this messy method -def parseCardLine(site, category, street, line, names, cardValues, cardSuits, boardValues, boardSuits): +def parseCardLine(category, street, line, names, cardValues, cardSuits, boardValues, boardSuits): if (line.startswith("Dealt to ") or line.find(" shows [")!=-1 or line.find("mucked [")!=-1): playerNo=recognisePlayerNo(line, names, "card") #anything but unbet will be ok for that string @@ -1088,7 +1084,7 @@ def parseCardLine(site, category, street, line, names, cardValues, cardSuits, bo raise FpdbError ("unrecognised line:"+line) #end def parseCardLine -def parseCashesAndSeatNos(lines, site): +def parseCashesAndSeatNos(lines): """parses the startCashes and seatNos of each player out of the given lines and returns them as a dictionary of two arrays""" cashes = [] seatNos = [] @@ -1099,8 +1095,7 @@ def parseCashesAndSeatNos(lines, site): pos1=lines[i].rfind("($")+2 if pos1==1: #for tourneys - it's 1 instead of -1 due to adding 2 above pos1=lines[i].rfind("(")+1 - elif (site=="ps"): - pos2=lines[i].find(" in chips") + pos2=lines[i].find(" in chips") cashes.append(float2int(lines[i][pos1:pos2])) return {'startCashes':cashes, 'seatNos':seatNos} #end def parseCashesAndSeatNos @@ -1114,7 +1109,7 @@ def parseFee(topline): #end def parsefee #returns a datetime object with the starttime indicated in the given topline -def parseHandStartTime(topline, site): +def parseHandStartTime(topline): #convert x:13:35 to 0x:13:35 counter=0 while (True): @@ -1258,7 +1253,7 @@ def parseSiteHandNo(topline): return topline[pos1:pos2] #end def parseSiteHandNo -def parseTableLine(site, base, line): +def parseTableLine(base, line): """returns a dictionary with maxSeats and tableName""" pos1=line.find('\'')+1 pos2=line.find('\'', pos1) @@ -1278,7 +1273,7 @@ def parseTourneyNo(topline): #end def parseTourneyNo #parses a win/collect line. manipulates the passed array winnings, no explicit return -def parseWinLine(line, site, names, winnings, isTourney): +def parseWinLine(line, names, winnings, isTourney): #print "parseWinLine: line:",line for i in range(len(names)): if (line.startswith(names[i].encode("latin-1"))): #found a winner @@ -1473,28 +1468,8 @@ def recognisePlayerNo(line, names, atype): raise FpdbError ("failed to recognise player in: "+line+" atype:"+atype) #end def recognisePlayerNo -#returns the site abbreviation for the given site -def recogniseSite(line): - # TODO: Make site agnostic - if (line.startswith("Full Tilt Poker")): - return "ftp" - elif (line.startswith("PokerStars")): - return "ps" - else: - raise FpdbError("failed to recognise site, line:"+line) -#end def recogniseSite - -#returns the ID of the given site -def recogniseSiteID(cursor, site): - if (site=="ftp"): - return 1 - #cursor.execute("SELECT id FROM Sites WHERE name = ('Full Tilt Poker')") - elif (site=="ps"): - return 2 - #cursor.execute("SELECT id FROM Sites WHERE name = ('PokerStars')") - else: - raise FpdbError("invalid site in recogniseSiteID: "+site) - return cursor.fetchall()[0][0] +def recogniseSiteID(): + return 2 #end def recogniseSiteID #removes trailing \n from the given array From ae4704b8dc003a035ab38c44e6bbec6a480c9b58 Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 21 Mar 2009 21:23:51 +0900 Subject: [PATCH 006/118] First working(?) version of siteneutral import --- pyfpdb/FpdbSQLQueries.py | 4 ++-- pyfpdb/fpdb_import.py | 24 ++++++++++++++++++------ pyfpdb/fpdb_parse_logic.py | 4 +--- pyfpdb/fpdb_save_to_db.py | 11 +++++++---- pyfpdb/fpdb_simple.py | 8 ++------ 5 files changed, 30 insertions(+), 21 deletions(-) diff --git a/pyfpdb/FpdbSQLQueries.py b/pyfpdb/FpdbSQLQueries.py index c3260466..55bf8c2e 100644 --- a/pyfpdb/FpdbSQLQueries.py +++ b/pyfpdb/FpdbSQLQueries.py @@ -41,8 +41,8 @@ class FpdbSQLQueries: self.query['list_tables'] = """ """ ################################################################## - # Drop Tables - MySQL, PostgreSQL and SQLite all share same syntax - ################################################################## + # Drop Tables - MySQL, PostgreSQL and SQLite all share same syntax + ################################################################## if(self.dbname == 'MySQL InnoDB') or (self.dbname == 'PostgreSQL') or (self.dbname == 'SQLite'): self.query['drop_table'] = """DROP TABLE IF EXISTS """ diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index e6baea29..fc878746 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -58,13 +58,14 @@ class Importer: self.cursor = None self.filelist = {} self.dirlist = {} + self.siteIds = {} self.addToDirList = {} self.removeFromFileList = {} # to remove deleted files self.monitor = False - self.updated = {} #Time last import was run {file:mtime} + self.updated = {} #Time last import was run {file:mtime} self.lines = None - self.faobs = None #File as one big string - self.pos_in_file = {} # dict to remember how far we have read in the file + self.faobs = None # File as one big string + self.pos_in_file = {} # dict to remember how far we have read in the file #Set defaults self.callHud = self.config.get_import_parameters().get("callFpdbHud") if 'minPrint' not in self.settings: @@ -111,6 +112,18 @@ class Importer: def addImportFile(self, filename, site = "default", filter = "passthrough"): #TODO: test it is a valid file -> put that in config!! self.filelist[filename] = [site] + [filter] + if site not in self.siteIds: + # Get id from Sites table in DB + self.fdb.cursor.execute(self.fdb.sql.query['getSiteId'], (site,)) + result = self.fdb.cursor.fetchall() + if len(result) == 1: + self.siteIds[site] = result[0][0] + else: + if len(result) == 0: + print "[ERROR] Database ID for %s not found" % site + else: + print "[ERROR] More than 1 Database ID found for %s - Multiple currencies not implemented yet" % site + # Called from GuiBulkImport to add a file or directory. def addBulkImportImportFileOrDir(self, inputPath,filter = "passthrough"): @@ -289,9 +302,8 @@ class Importer: print "TODO: implement importing tournament summaries" #self.faobs = readfile(inputFile) #self.parseTourneyHistory() - return 0 + return (0,0,0,1,0) - site=fpdb_simple.recogniseSite(firstline) category=fpdb_simple.recogniseCategory(firstline) startpos=0 @@ -322,7 +334,7 @@ class Importer: try: handsId=fpdb_parse_logic.mainParser(self.settings['db-backend'], self.fdb.db - ,self.fdb.cursor, site, category, hand, self.config) + ,self.fdb.cursor, self.siteIds[site], category, hand, self.config) self.fdb.db.commit() stored+=1 diff --git a/pyfpdb/fpdb_parse_logic.py b/pyfpdb/fpdb_parse_logic.py index dd5d84ea..58cc94df 100644 --- a/pyfpdb/fpdb_parse_logic.py +++ b/pyfpdb/fpdb_parse_logic.py @@ -21,7 +21,7 @@ import fpdb_simple import fpdb_save_to_db #parses a holdem hand -def mainParser(backend, db, cursor, category, hand, config): +def mainParser(backend, db, cursor, siteID, category, hand, config): category = fpdb_simple.recogniseCategory(hand[0]) base = "hold" if category == "holdem" or category == "omahahi" or category == "omahahilo" else "stud" @@ -35,8 +35,6 @@ def mainParser(backend, db, cursor, category, hand, config): #part 1: read hand no and check for duplicate siteHandNo = fpdb_simple.parseSiteHandNo(hand[0]) handStartTime = fpdb_simple.parseHandStartTime(hand[0]) - siteID = fpdb_simple.recogniseSiteID() - #print "parse logic, siteID:",siteID,"site:",site isTourney = fpdb_simple.isTourney(hand[0]) smallBlindLine = 0 diff --git a/pyfpdb/fpdb_save_to_db.py b/pyfpdb/fpdb_save_to_db.py index f13c8032..85468c2b 100644 --- a/pyfpdb/fpdb_save_to_db.py +++ b/pyfpdb/fpdb_save_to_db.py @@ -28,11 +28,12 @@ SQLITE = 4 fastStoreHudCache = True # set this to True to test the new storeHudCache routine -saveActions = False # set this to False to avoid storing action data +saveActions = True # set this to False to avoid storing action data # Pros: speeds up imports # Cons: no action data is saved, so you need to keep the hand histories # variance not available on stats page - + # : No graphs + #stores a stud/razz hand into the database def ring_stud(config, backend, db, cursor, base, category, site_hand_no, gametype_id, hand_start_time ,names, player_ids, start_cashes, antes, card_values, card_suits, winnings, rakes @@ -41,7 +42,7 @@ def ring_stud(config, backend, db, cursor, base, category, site_hand_no, gametyp import_options = config.get_import_parameters() - saveActions = True if import_options['saveActions'] == 'True' else False + saveActions = False if import_options['saveActions'] == 'False' else True fastStoreHudCache = True if import_options['fastStoreHudCache'] == 'True' else False fpdb_simple.fillCardArrays(len(names), base, category, card_values, card_suits) @@ -69,9 +70,11 @@ def ring_holdem_omaha(config, backend, db, cursor, base, category, site_hand_no, """stores a holdem/omaha hand into the database""" import_options = config.get_import_parameters() - saveActions = True if import_options['saveActions'] == 'True' else False + saveActions = False if import_options['saveActions'] == 'False' else True fastStoreHudCache = True if import_options['fastStoreHudCache'] == 'True' else False + #print "DEBUG: saveActions: %s fastStoreHudCache: %s" %(saveActions, fastStoreHudCache) + t0 = time() fpdb_simple.fillCardArrays(len(names), base, category, card_values, card_suits) t1 = time() diff --git a/pyfpdb/fpdb_simple.py b/pyfpdb/fpdb_simple.py index b6e7e237..5a75af45 100644 --- a/pyfpdb/fpdb_simple.py +++ b/pyfpdb/fpdb_simple.py @@ -691,14 +691,13 @@ def filterCrap(hand, isTourney): hand[i] = False elif (hand[i]=="Betting is capped"): hand[i] = False - #site specific variable position filter elif (hand[i].find(" said, \"")!=-1): hand[i] = False + if isTourney: if (hand[i].endswith(" is sitting out") and (not hand[i].startswith("Seat "))): hand[i] = False - elif: - if (hand[i].endswith(": sits out")): + elif (hand[i].endswith(": sits out")): hand[i] = False elif (hand[i].endswith(" is sitting out")): hand[i] = False @@ -1375,9 +1374,6 @@ def recognisePlayerNo(line, names, atype): raise FpdbError ("failed to recognise player in: "+line+" atype:"+atype) #end def recognisePlayerNo -def recogniseSiteID(): - return 2 -#end def recogniseSiteID #removes trailing \n from the given array def removeTrailingEOL(arr): From 497354d4f46becc19a861d8559f7e7ced735c293 Mon Sep 17 00:00:00 2001 From: Worros Date: Tue, 24 Mar 2009 22:58:45 +0900 Subject: [PATCH 007/118] Make BulkImport work only with HHCs Added new section to config for converters for ease of creating the gui. Removed the section in the site line for filter and supported games --- pyfpdb/Configuration.py | 20 ++++++++++++++++++++ pyfpdb/GuiBulkImport.py | 16 +++++++--------- pyfpdb/HUD_config.xml.example | 11 ++++++++--- pyfpdb/fpdb_import.py | 8 ++++---- pyfpdb/fpdb_parse_logic.py | 2 +- 5 files changed, 40 insertions(+), 17 deletions(-) diff --git a/pyfpdb/Configuration.py b/pyfpdb/Configuration.py index 19e62038..3e0ca3dd 100755 --- a/pyfpdb/Configuration.py +++ b/pyfpdb/Configuration.py @@ -194,6 +194,15 @@ class Aux_window: temp = temp + "%s" % self.layout[layout] return temp +class HHC: + def __init__(self, node): + self.site = node.getAttribute("site") + self.converter = node.getAttribute("converter") + + def __str__(self): + return "%s:\t%s" % (self.site, self.converter) + + class Popup: def __init__(self, node): self.name = node.getAttribute("pu_name") @@ -273,6 +282,7 @@ class Config: self.supported_games = {} self.supported_databases = {} self.aux_windows = {} + self.hhcs = {} self.popup_windows = {} # s_sites = doc.getElementsByTagName("supported_sites") @@ -295,6 +305,11 @@ class Config: aw = Aux_window(node = aw_node) self.aux_windows[aw.name] = aw +# s_dbs = doc.getElementsByTagName("mucked_windows") + for hhc_node in doc.getElementsByTagName("hhc"): + hhc = HHC(node = hhc_node) + self.hhcs[hhc.site] = hhc + # s_dbs = doc.getElementsByTagName("popup_windows") for pu_node in doc.getElementsByTagName("pu"): pu = Popup(node = pu_node) @@ -696,6 +711,11 @@ if __name__== "__main__": for w in c.aux_windows.keys(): print c.aux_windows[w] print "----------- END AUX WINDOW FORMATS -----------" + + print "\n----------- HAND HISTORY CONVERTERS -----------" + for w in c.hhcs.keys(): + print c.hhcs[w] + print "----------- END HAND HISTORY CONVERTERS -----------" print "\n----------- POPUP WINDOW FORMATS -----------" for w in c.popup_windows.keys(): diff --git a/pyfpdb/GuiBulkImport.py b/pyfpdb/GuiBulkImport.py index 1a62a37d..35563fe3 100755 --- a/pyfpdb/GuiBulkImport.py +++ b/pyfpdb/GuiBulkImport.py @@ -66,10 +66,10 @@ class GuiBulkImport(): self.importer.setDropIndexes(cb_model[cb_index][0]) else: self.importer.setDropIndexes("auto") - hhc=self.cbfilter.get_model()[self.cbfilter.get_active()][0] + sitename = self.cbfilter.get_model()[self.cbfilter.get_active()][0] self.lab_info.set_text("Importing") - self.importer.addBulkImportImportFileOrDir(self.inputFile,filter=hhc) + self.importer.addBulkImportImportFileOrDir(self.inputFile, site = sitename) self.importer.setCallHud(False) starttime = time() (stored, dups, partial, errs, ttime) = self.importer.runImport() @@ -175,11 +175,9 @@ class GuiBulkImport(): # ComboBox - filter self.cbfilter = gtk.combo_box_new_text() - self.cbfilter.append_text("passthrough") - self.cbfilter.append_text("BetfairToFpdb") - self.cbfilter.append_text("EverleafToFpdb") - self.cbfilter.append_text("FulltiltToFpdb") - self.cbfilter.append_text("PokerStarsToFpdb") + for w in self.config.hhcs: + print w + self.cbfilter.append_text(w) self.cbfilter.set_active(0) self.table.attach(self.cbfilter, 3, 4, 2, 3, xpadding = 10, ypadding = 0, yoptions=gtk.SHRINK) self.cbfilter.show() @@ -223,7 +221,7 @@ def main(argv=None): parser.add_option("-q", "--quiet", action="store_false", dest="gui", default=True, help="don't start gui; deprecated (just give a filename with -f).") parser.add_option("-c", "--convert", dest="filtername", default="passthrough", metavar="FILTER", - help="Conversion filter (*passthrough, FullTiltToFpdb, PokerStarsToFpdb, EverleafToFpdb)") + help="Conversion filter (*Full Tilt Poker, PokerStars, Everleaf)") parser.add_option("-x", "--failOnError", action="store_true", default=False, help="If this option is passed it quits when it encounters any error") parser.add_option("-m", "--minPrint", "--status", dest="minPrint", default="0", type="int", @@ -258,7 +256,7 @@ def main(argv=None): importer = fpdb_import.Importer(False,settings, config) importer.setDropIndexes("auto") importer.setFailOnError(options.failOnError) - importer.addBulkImportImportFileOrDir(os.path.expanduser(options.filename), filter=options.filtername) + importer.addBulkImportImportFileOrDir(os.path.expanduser(options.filename), site=options.filtername) importer.setCallHud(False) importer.runImport() importer.clearFileList() diff --git a/pyfpdb/HUD_config.xml.example b/pyfpdb/HUD_config.xml.example index f7351526..ad7d440b 100644 --- a/pyfpdb/HUD_config.xml.example +++ b/pyfpdb/HUD_config.xml.example @@ -2,7 +2,7 @@ - + @@ -49,7 +49,7 @@ - + @@ -84,7 +84,7 @@ - + @@ -198,6 +198,11 @@ + + + + + diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 5ce23928..68373fb0 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -126,18 +126,18 @@ class Importer: # Called from GuiBulkImport to add a file or directory. - def addBulkImportImportFileOrDir(self, inputPath,filter = "passthrough"): + def addBulkImportImportFileOrDir(self, inputPath, site = "PokerStars"): """Add a file or directory for bulk import""" - + filter = self.config.hhcs[site].converter # Bulk import never monitors # if directory, add all files in it. Otherwise add single file. # TODO: only add sane files? if os.path.isdir(inputPath): for subdir in os.walk(inputPath): for file in subdir[2]: - self.addImportFile(os.path.join(inputPath, subdir[0], file), site="default", filter=filter) + self.addImportFile(os.path.join(inputPath, subdir[0], file), site=site, filter=filter) else: - self.addImportFile(inputPath, site="default", filter=filter) + self.addImportFile(inputPath, site=site, filter=filter) #Add a directory of files to filelist #Only one import directory per site supported. #dirlist is a hash of lists: diff --git a/pyfpdb/fpdb_parse_logic.py b/pyfpdb/fpdb_parse_logic.py index 0badbb82..750b1512 100644 --- a/pyfpdb/fpdb_parse_logic.py +++ b/pyfpdb/fpdb_parse_logic.py @@ -60,7 +60,7 @@ def mainParser(backend, db, cursor, siteID, category, hand, config): fpdb_simple.isAlreadyInDB(cursor, gametypeID, siteHandNo) - hand=fpdb_simple.filterCrap(site, hand, isTourney) + hand=fpdb_simple.filterCrap(hand, isTourney) #part 2: classify lines by type (e.g. cards, action, win, sectionchange) and street fpdb_simple.classifyLines(hand, category, lineTypes, lineStreets) From 20dd3e1aec8295369f4f2af671807b9608a1f6d7 Mon Sep 17 00:00:00 2001 From: Worros Date: Wed, 25 Mar 2009 18:35:19 +0900 Subject: [PATCH 008/118] Make CET timezones work with Stars hands --- pyfpdb/PokerStarsToFpdb.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pyfpdb/PokerStarsToFpdb.py b/pyfpdb/PokerStarsToFpdb.py index 8b905005..f521bbd0 100755 --- a/pyfpdb/PokerStarsToFpdb.py +++ b/pyfpdb/PokerStarsToFpdb.py @@ -128,9 +128,11 @@ follow : whether to tail -f the input""" logging.debug("readHandInfo: %s" % info) for key in info: if key == 'DATETIME': - datetime = info[key].replace(" - "," ") # some are like "2009/02/26 - 15:22:55 ET" - datetime = datetime.replace(" (ET)","") # kludge for now. - datetime = datetime.replace(" ET","") # kludge for now. + #2008/11/12 10:00:48 CET [2008/11/12 4:00:48 ET] + #2008/08/17 - 01:14:43 (ET) + #2008/09/07 06:23:14 ET + m2 = re.search("(?P[0-9]{4})\/(?P[0-9]{2})\/(?P[0-9]{2})[\- ]+(?P[0-9]+):(?P[0-9]+):(?P[0-9]+)", info[key]) + datetime = "%s/%s/%s %s:%s:%s" % (m2.group('Y'), m2.group('M'),m2.group('D'),m2.group('H'),m2.group('MIN'),m2.group('S')) hand.starttime = time.strptime(datetime, "%Y/%m/%d %H:%M:%S") if key == 'HID': hand.handid = info[key] From 710893b186edf4fba36ea1c3749daa38ed37eb48 Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 28 Mar 2009 00:54:30 +0900 Subject: [PATCH 009/118] Fix merge breakage --- pyfpdb/fpdb_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 17bf04f1..1e5049d0 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -338,7 +338,7 @@ class Importer: try: handsId = fpdb_parse_logic.mainParser(self.settings['db-backend'], self.fdb.db - ,self.fdb.cursor, site, category, hand, self.config) + ,self.fdb.cursor, self.siteIds[site], category, hand, self.config) self.fdb.db.commit() stored += 1 From f6ca8c1bc10fce6eb61206ecbbb708eee278f910 Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 28 Mar 2009 19:38:37 +0900 Subject: [PATCH 010/118] GuiBulkImport cli breakage fix --- pyfpdb/GuiBulkImport.py | 2 +- pyfpdb/fpdb_import.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyfpdb/GuiBulkImport.py b/pyfpdb/GuiBulkImport.py index 35563fe3..e90be69a 100755 --- a/pyfpdb/GuiBulkImport.py +++ b/pyfpdb/GuiBulkImport.py @@ -236,7 +236,7 @@ def main(argv=None): if os.name == 'nt': settings['os'] = 'windows' else: settings['os'] = 'linuxmac' - settings.update(config.get_db_parameters('fpdb')) + settings.update(config.get_db_parameters()) settings.update(config.get_tv_parameters()) settings.update(config.get_import_parameters()) settings.update(config.get_default_paths()) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 1e5049d0..2b421e19 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -21,7 +21,7 @@ import os # todo: remove this once import_dir is in fpdb_import import sys -from time import time +from time import time, strftime import traceback import math import datetime From 95add137311ac0f173fb90d64fe62b4a43ed91ce Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 28 Mar 2009 19:39:13 +0900 Subject: [PATCH 011/118] No passthough converter in siteneutral --- pyfpdb/GuiBulkImport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyfpdb/GuiBulkImport.py b/pyfpdb/GuiBulkImport.py index e90be69a..7dbb9858 100755 --- a/pyfpdb/GuiBulkImport.py +++ b/pyfpdb/GuiBulkImport.py @@ -220,7 +220,7 @@ def main(argv=None): help="Input file in quiet mode") parser.add_option("-q", "--quiet", action="store_false", dest="gui", default=True, help="don't start gui; deprecated (just give a filename with -f).") - parser.add_option("-c", "--convert", dest="filtername", default="passthrough", metavar="FILTER", + parser.add_option("-c", "--convert", dest="filtername", default="PokerStars", metavar="FILTER", help="Conversion filter (*Full Tilt Poker, PokerStars, Everleaf)") parser.add_option("-x", "--failOnError", action="store_true", default=False, help="If this option is passed it quits when it encounters any error") From bdd615ede9ec20a6ccac58eb48ed806e0b305f2f Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 28 Mar 2009 20:32:30 +0900 Subject: [PATCH 012/118] utf8 fix to db connection Change mysql connection type to use_unicode. --- pyfpdb/fpdb_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyfpdb/fpdb_db.py b/pyfpdb/fpdb_db.py index 771b79fa..eba87f52 100644 --- a/pyfpdb/fpdb_db.py +++ b/pyfpdb/fpdb_db.py @@ -59,7 +59,7 @@ class fpdb_db: self.database=database if backend==self.MYSQL_INNODB: import MySQLdb - self.db=MySQLdb.connect(host = host, user = user, passwd = password, db = database) + self.db=MySQLdb.connect(host = host, user = user, passwd = password, db = database, use_unicode=True) elif backend==self.PGSQL: import psycopg2 # If DB connection is made over TCP, then the variables From 0f811374000d0d1e99d0138a67c3dd4f878fdeae Mon Sep 17 00:00:00 2001 From: Worros Date: Mon, 30 Mar 2009 03:45:41 +0800 Subject: [PATCH 013/118] Add THANKS.txt to acknowledge contributors. Text is in wiki format for quick update of the wiki. --- THANKS.txt | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 THANKS.txt diff --git a/THANKS.txt b/THANKS.txt new file mode 100644 index 00000000..c4eb0ca4 --- /dev/null +++ b/THANKS.txt @@ -0,0 +1,35 @@ +=**Thanks - Contributors to FPDB**= + +This page is to acknowledge the people developing and contributing to FPDB, be it with code, documentation, testing, forum support and everything else that is required to make an open source project work. + +Most people have found the project though the software forum at [[http://forumserver.twoplustwo.com/45/software/|2+2]] so most references to 2+2 aliases + +Sincere apologies to those not listed. FPDB started life in August 2008, but we only got around to putting together a proper contributers list in April 2009. Private messages and emails have been deleted so we dont have a full history of non- + +==**Developers**== +Active: +Steffen123 - The initial code drop and project lead. +Eratosthenes - Initial HUD code drop, release manager +Ekdikeo - Contributor - contributed Everleaf support and much HUD work +Sorrow - Contributor - Graphing, import framework +mcturnbull (2+2?) - Contributor - import framework + +Inactive: +Sqlcoder - SQL query guru - Graphing filters, Player Stats, Positional Stats queries + general sql optimisation work +Bostik - Author of [[http://bostik.iki.fi/pokerstats/|PokerStats]] - Postgres linux work. PokerStats import code was the inspiration for the new framework. I'm sure well use more of his ideas in future. +FIXME - Original windows installer/packager + +==Documentation== +FIXME - Original web site guy +FIXME - Guy who wrote the Mac install page +FIXME - Any other wiki editors. + +==Other== +Eleatic Stranger - Super tester and contributor - Our #1 tester. Tracks the git repo and works the code harder than anyone and gives great feedback and bug reports. Thankyou. +Xaviax - Tester and honorary helpdesk - Another excellent tester tracking git, has responded to many queries in the fpdb thread on his own time. +KayosD - Hand History donation - Carbon Poker +freerollerjb - Hand History donation - Carbon Poker +puru - Hand History donation - Carbon Poker +freestailo - Hand History donation - Carbon Poker +MoDDe (Sourceforge) - Hand History Donation - Betfair +Jay10826 - Hand History donation - Ultimate Bet From a8c46e095d3626daa6604564d8fd471a8de7d9b2 Mon Sep 17 00:00:00 2001 From: Worros Date: Mon, 30 Mar 2009 06:12:19 +0800 Subject: [PATCH 014/118] Finish sentence in THANKS.txt --- THANKS.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/THANKS.txt b/THANKS.txt index c4eb0ca4..7adf9825 100644 --- a/THANKS.txt +++ b/THANKS.txt @@ -4,7 +4,7 @@ This page is to acknowledge the people developing and contributing to FPDB, be i Most people have found the project though the software forum at [[http://forumserver.twoplustwo.com/45/software/|2+2]] so most references to 2+2 aliases -Sincere apologies to those not listed. FPDB started life in August 2008, but we only got around to putting together a proper contributers list in April 2009. Private messages and emails have been deleted so we dont have a full history of non- +Sincere apologies to those not listed. FPDB started life in August 2008, but we only got around to putting together a proper contributers list in April 2009. Private messages and emails have been deleted so we dont have a full history of non-code based contributions ==**Developers**== Active: From 2129faa46b3bca5ae8031a7d5e22c42444188d95 Mon Sep 17 00:00:00 2001 From: Worros Date: Mon, 30 Mar 2009 06:39:21 +0800 Subject: [PATCH 015/118] Add UltimateBetToFpdb Copy from current PokerStarsToFpdb with minor regex updates. Still untested --- pyfpdb/UltimateBetToFpdb.py | 329 ++++++++++++++++++++++++++++++++++++ 1 file changed, 329 insertions(+) create mode 100755 pyfpdb/UltimateBetToFpdb.py diff --git a/pyfpdb/UltimateBetToFpdb.py b/pyfpdb/UltimateBetToFpdb.py new file mode 100755 index 00000000..6b11d8e6 --- /dev/null +++ b/pyfpdb/UltimateBetToFpdb.py @@ -0,0 +1,329 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2008, Carl Gherardi +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +######################################################################## + +import sys +from HandHistoryConverter import * + + +class UltimateBet(HandHistoryConverter): + + # Static regexes + re_GameInfo = re.compile("Stage #(?P[0-9]+):\s+\(?(?PHold\'em|Razz|Seven Card|Omaha|Omaha Hi/Lo|Badugi) (?PNo Limit|Normal|Pot Limit),? \(?(?P\$|)?(?P[.0-9]+)/\$?(?P[.0-9]+)\) - (?P.*$)", re.MULTILINE) + re_SplitHands = re.compile('(\n\n\n+)') + re_HandInfo = re.compile("^Table \'(?P[- a-zA-Z]+)\'(?P.+?$)?", re.MULTILINE) + re_Button = re.compile('Seat #(?P
[ a-zA-Z]+) - \$?(?P[.0-9]+)/\$?(?P[.0-9]+) - (?P.*) - (?P
[0-9]+):(?P[0-9]+) ET - (?P[0-9]+)/(?P[0-9]+)/(?P[0-9]+)Table (?P
[ a-zA-Z]+)\nSeat (?P
.+$)", re.MULTILINE) diff --git a/pyfpdb/FulltiltToFpdb.py b/pyfpdb/FulltiltToFpdb.py index f3d30229..79d606b3 100755 --- a/pyfpdb/FulltiltToFpdb.py +++ b/pyfpdb/FulltiltToFpdb.py @@ -28,7 +28,8 @@ class Fulltilt(HandHistoryConverter): # Static regexes re_GameInfo = re.compile('- (?P\$|)?(?P[.0-9]+)/\$?(?P[.0-9]+) (Ante \$(?P[.0-9]+) )?- (?P(No Limit|Pot Limit|Limit))? (?P(Hold\'em|Omaha Hi|Razz))') - re_SplitHands = re.compile(r"(\n\n+)") + re_SplitHands = re.compile(r"\n\n+") + re_TailSplitHands = re.compile(r"(\n\n+)") re_HandInfo = re.compile('.*#(?P[0-9]+): Table (?P
[- a-zA-Z]+) (\((?P.+)\) )?- \$?(?P[.0-9]+)/\$?(?P[.0-9]+) (Ante \$(?P[.0-9]+) )?- (?P[a-zA-Z\' ]+) - (?P.*)') re_Button = re.compile('^The button is in seat #(?P
[- a-zA-Z]+)\'(?P.+?$)?", re.MULTILINE) re_Button = re.compile('Seat #(?P