From 72dff55f6f38eb984181c7b55e122ad437f73b29 Mon Sep 17 00:00:00 2001 From: Worros Date: Mon, 9 Mar 2009 18:07:04 +0900 Subject: [PATCH 1/5] [KILLFTP 1/x] Tear out FTP specific hand import code --- pyfpdb/fpdb_import.py | 131 +++++++++++++----------------------------- 1 file changed, 41 insertions(+), 90 deletions(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index d86c4214..fde69125 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -293,12 +293,6 @@ class Importer: # print "import_fpdb_file", file, site, self.lines, "\n" return (0,0,0,1,0) - if firstline.find("Tournament Summary")!=-1: - print "TODO: implement importing tournament summaries" - #self.faobs = readfile(inputFile) - #self.parseTourneyHistory() - return 0 - site=fpdb_simple.recogniseSite(firstline) category=fpdb_simple.recogniseCategory(firstline) @@ -316,91 +310,53 @@ class Importer: if (len(hand[0])<2): hand=hand[1:] - cancelled=False - damaged=False - if (site=="ftp"): - for i in range (len(hand)): - if (hand[i].endswith(" has been canceled")): #this is their typo. this is a typo, right? - cancelled=True - - #FTP generates lines looking like: - #Seat 1: IOS Seat 2: kashman59 (big blind) showed [8c 9d] and won ($3.25) with a pair of Eights - #ie. Seat X multiple times on the same line in the summary section, when a new player sits down in the - #middle of the hand. - #TODO: Deal with this properly, either fix the file or make the parsing code work with this line. - if "Seat" in hand[i]: - mo = re.search(" Seat [0-9]+: ", hand[i]) - if mo: - print "mo=", mo, "\nmo.start=", mo.start(),"\nhand[i]=",hand[i] - hand.insert(i+1, hand[i][mo.start()+1:]) - hand[i] = hand[i][0:mo.start()] - - if (len(hand)<3): - pass - #todo: the above 2 lines are kind of a dirty hack, the mentioned circumstances should be handled elsewhere but that doesnt work with DOS/Win EOL. actually this doesnt work. - elif (hand[0].endswith(" (partial)")): #partial hand - do nothing - partial+=1 - elif (hand[1].find("Seat")==-1 and hand[2].find("Seat")==-1 and hand[3].find("Seat")==-1):#todo: should this be or instead of and? - partial+=1 - elif (cancelled or damaged): - partial+=1 - if damaged: - print """ - DEBUG: Partial hand triggered by a line containing 'Seat X:' twice. This is a - bug in the FTP software when a player sits down in the middle of a hand. - Adding a newline after the player name will fix the issue - """ - print "File: %s" %(file) - print "Line: %s" %(startpos) - else: #normal processing - isTourney=fpdb_simple.isTourney(hand[0]) - if not isTourney: - fpdb_simple.filterAnteBlindFold(site,hand) - hand=fpdb_simple.filterCrap(site, hand, isTourney) - self.hand=hand + isTourney=fpdb_simple.isTourney(hand[0]) + if not isTourney: + fpdb_simple.filterAnteBlindFold(site,hand) + hand=fpdb_simple.filterCrap(site, hand, isTourney) + self.hand=hand - try: - handsId=fpdb_parse_logic.mainParser(self.settings['db-backend'], self.fdb.db + try: + handsId=fpdb_parse_logic.mainParser(self.settings['db-backend'], self.fdb.db ,self.fdb.cursor, site, category, hand) - self.fdb.db.commit() + self.fdb.db.commit() - stored+=1 - if self.callHud: - #print "call to HUD here. handsId:",handsId - #pipe the Hands.id out to the HUD - self.caller.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep) - except fpdb_simple.DuplicateError: - duplicates+=1 - except (ValueError), fe: - errors+=1 - self.printEmailErrorMessage(errors, file, hand) - - if (self.settings['failOnError']): - self.fdb.db.commit() #dont remove this, in case hand processing was cancelled. - raise - except (fpdb_simple.FpdbError), fe: - errors+=1 - self.printEmailErrorMessage(errors, file, hand) + stored+=1 + if self.callHud: + #print "call to HUD here. handsId:",handsId + #pipe the Hands.id out to the HUD + self.caller.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep) + except fpdb_simple.DuplicateError: + duplicates+=1 + except (ValueError), fe: + errors+=1 + self.printEmailErrorMessage(errors, file, hand) - #fe.printStackTrace() #todo: get stacktrace - self.fdb.db.rollback() + if (self.settings['failOnError']): + self.fdb.db.commit() #dont remove this, in case hand processing was cancelled. + raise + except (fpdb_simple.FpdbError), fe: + errors+=1 + self.printEmailErrorMessage(errors, file, hand) + + self.fdb.db.rollback() - if (self.settings['failOnError']): - self.fdb.db.commit() #dont remove this, in case hand processing was cancelled. - raise - if (self.settings['minPrint']!=0): - if ((stored+duplicates+partial+errors)%self.settings['minPrint']==0): - print "stored:", stored, "duplicates:", duplicates, "partial:", partial, "errors:", errors + if (self.settings['failOnError']): + self.fdb.db.commit() #dont remove this, in case hand processing was cancelled. + raise + if (self.settings['minPrint']!=0): + if ((stored+duplicates+errors)%self.settings['minPrint']==0): + print "stored:", stored, "duplicates:", duplicates, "errors:", errors - if (self.settings['handCount']!=0): - if ((stored+duplicates+partial+errors)>=self.settings['handCount']): - if (not self.settings['quiet']): - print "quitting due to reaching the amount of hands to be imported" - print "Total stored:", stored, "duplicates:", duplicates, "partial/damaged:", partial, "errors:", errors, " time:", (time() - starttime) - sys.exit(0) - startpos=endpos + if (self.settings['handCount']!=0): + if ((stored+duplicates+errors)>=self.settings['handCount']): + if (not self.settings['quiet']): + print "quitting due to reaching the amount of hands to be imported" + print "Total stored:", stored, "duplicates:", duplicates, "errors:", errors, " time:", (time() - starttime) + sys.exit(0) + startpos=endpos ttime = time() - starttime - print "\rTotal stored:", stored, "duplicates:", duplicates, "partial:", partial, "errors:", errors, " time:", ttime + print "\rTotal stored:", stored, "duplicates:", duplicates, "errors:", errors, " time:", ttime if stored==0: if duplicates>0: @@ -411,16 +367,11 @@ class Importer: else: print "failed to read a single hand from file:", inputFile handsId=0 - #todo: this will cause return of an unstored hand number if the last hand was error or partial + #todo: this will cause return of an unstored hand number if the last hand was error self.fdb.db.commit() self.handsId=handsId return (stored, duplicates, partial, errors, ttime) - def parseTourneyHistory(self): - print "Tourney history parser stub" - #Find tournament boundaries. - #print self.foabs - def printEmailErrorMessage(self, errors, filename, line): traceback.print_exc(file=sys.stderr) print "Error No.",errors,", please send the hand causing this to steffen@sycamoretest.info so I can fix it." From 58445c0a1b72fb16f3db97bb13a798db1d896feb Mon Sep 17 00:00:00 2001 From: Worros Date: Mon, 9 Mar 2009 18:21:22 +0900 Subject: [PATCH 2/5] [KILLFTP 2/x] More ftp specific code removal --- pyfpdb/fpdb_parse_logic.py | 2 - pyfpdb/fpdb_simple.py | 140 ++++++++----------------------------- 2 files changed, 29 insertions(+), 113 deletions(-) diff --git a/pyfpdb/fpdb_parse_logic.py b/pyfpdb/fpdb_parse_logic.py index dab3bcc4..c5076283 100644 --- a/pyfpdb/fpdb_parse_logic.py +++ b/pyfpdb/fpdb_parse_logic.py @@ -111,8 +111,6 @@ def mainParser(backend, db, cursor, site, category, hand): tableResult=fpdb_simple.parseTableLine(site, base, hand[i]) else: raise fpdb_simple.FpdbError("unrecognised lineType:"+lineTypes[i]) - if site=="ftp": - tableResult=fpdb_simple.parseTableLine(site, base, hand[0]) maxSeats=tableResult['maxSeats'] tableName=tableResult['tableName'] #print "before part5, antes:", antes diff --git a/pyfpdb/fpdb_simple.py b/pyfpdb/fpdb_simple.py index dbf53e93..d3bb4097 100644 --- a/pyfpdb/fpdb_simple.py +++ b/pyfpdb/fpdb_simple.py @@ -476,8 +476,6 @@ def convert3B4B(site, category, limit_type, actionTypes, actionAmounts): #todo: run correction for below if (site=="ps" and category=="holdem" and limit_type=="nl" and len(bets)==3): fail=False - if (site=="ftp" and category=="omahahi" and limit_type=="pl" and len(bets)==3): - fail=False if fail: print "len(bets)>2 in convert3B4B, i didnt think this is possible. i:",i,"j:",j,"k:",k @@ -689,18 +687,8 @@ def filterCrap(site, hand, isTourney): toRemove.append(hand[i]) elif (hand[i].endswith("is disconnected")): toRemove.append(hand[i]) - elif (hand[i].endswith(" is feeling angry")): - toRemove.append(hand[i]) - elif (hand[i].endswith(" is feeling confused")): - toRemove.append(hand[i]) - elif (hand[i].endswith(" is feeling happy")): - toRemove.append(hand[i]) - elif (hand[i].endswith(" is feeling normal")): - toRemove.append(hand[i]) elif (hand[i].find(" is low with [")!=-1): toRemove.append(hand[i]) - #elif (hand[i].find("-max Seat #")!=-1 and hand[i].find(" is the button")!=-1): - # toRemove.append(hand[i]) elif (hand[i].endswith(" mucks")): toRemove.append(hand[i]) elif (hand[i].endswith(": mucks hand")): @@ -711,8 +699,6 @@ def filterCrap(site, hand, isTourney): toRemove.append(hand[i]) elif (hand[i].find(" shows ")!=-1 and hand[i].find("[")==-1): toRemove.append(hand[i]) - #elif (hand[i].startswith("Table '") and hand[i].endswith("-max")): - # toRemove.append(hand[i]) elif (hand[i].startswith("The button is in seat #")): toRemove.append(hand[i]) #above is alphabetic, reorder below if bored @@ -747,8 +733,6 @@ def filterCrap(site, hand, isTourney): #site specific variable position filter elif (hand[i].find(" said, \"")!=-1): toRemove.append(hand[i]) - elif (hand[i].find(": ")!=-1 and site=="ftp" and hand[i].find("Seat ")==-1 and hand[i].find(": Table")==-1): #filter ftp chat - toRemove.append(hand[i]) if isTourney: if (hand[i].endswith(" is sitting out") and (not hand[i].startswith("Seat "))): toRemove.append(hand[i]) @@ -857,14 +841,6 @@ def isWinLine(line): return True elif (line.find("ties for the low side pot")!=-1): return True - elif (line.find("ties for the main pot")!=-1): #for ftp tied main pot of split pot - return True - elif (line.find("ties for the pot")!=-1): #for ftp tie - return True - elif (line.find("ties for the side pot")!=-1): #for ftp tied split pots - return True - elif (line.find("wins side pot #")!=-1): #for ftp multi split pots - return True elif (line.find("wins the low main pot")!=-1): return True elif (line.find("wins the low pot")!=-1): @@ -879,8 +855,6 @@ def isWinLine(line): return True elif (line.find("wins the main pot")!=-1): return True - elif (line.find("wins the side pot")!=-1): #for ftp split pots - return True elif (line.find("collected")!=-1): return True else: @@ -904,10 +878,6 @@ def parseActionAmount(line, atype, site, isTourney): amount=0 elif (atype=="check"): amount=0 - elif (atype=="unbet" and site=="ftp"): - pos1=line.find("$")+1 - pos2=line.find(" returned to") - amount=float2int(line[pos1:pos2]) elif (atype=="unbet" and site=="ps"): #print "ps unbet, line:",line pos1=line.find("$")+1 @@ -1129,8 +1099,6 @@ def parseCashesAndSeatNos(lines, site): pos1=lines[i].rfind("($")+2 if pos1==1: #for tourneys - it's 1 instead of -1 due to adding 2 above pos1=lines[i].rfind("(")+1 - if (site=="ftp"): - pos2=lines[i].rfind(")") elif (site=="ps"): pos2=lines[i].find(" in chips") cashes.append(float2int(lines[i][pos1:pos2])) @@ -1157,41 +1125,25 @@ def parseHandStartTime(topline, site): if counter==10: break isUTC=False - if site=="ftp": - # Full Tilt Sit'n'Go - # Full Tilt Poker Game #10311865543: $1 + $0.25 Sit & Go (78057629), Table 1 - 25/50 - No Limit Hold'em - 0:07:45 ET - 2009/01/29 - # Cash Game: - # Full Tilt Poker Game #9403951181: Table CR - tay - $0.05/$0.10 - No Limit Hold'em - 9:40:20 ET - 2008/12/09 - # Full Tilt Poker Game #9468383505: Table Bike (deep 6) - $0.05/$0.10 - No Limit Hold'em - 5:09:36 ET - 2008/12/13 - pos = topline.find(" ", len(topline)-26)+1 - tmp = topline[pos:] - - rexx = '(?P
[0-9]+):(?P[0-9]+):(?P[0-9]+) ET [\- ]+(?P[0-9]{4})\/(?P[0-9]{2})\/(?P[0-9]{2})' - m = re.search(rexx,tmp) - result = datetime.datetime(int(m.group('YEAR')), int(m.group('MON')), int(m.group('DAY')), int(m.group('HR')), int(m.group('MIN')), int(m.group('SEC'))) - elif site=="ps": - if topline.find("UTC")!=-1: - pos1 = topline.find("-")+2 - pos2 = topline.find("UTC") - tmp=topline[pos1:pos2] - isUTC=True - else: - tmp=topline - #print "parsehandStartTime, tmp:", tmp - pos = tmp.find("-")+2 - tmp = tmp[pos:] - #Need to match either - # 2008/09/07 06:23:14 ET or - # 2008/08/17 - 01:14:43 (ET) or - # 2008/11/12 9:33:31 CET [2008/11/12 3:33:31 ET] - rexx = '(?P[0-9]{4})\/(?P[0-9]{2})\/(?P[0-9]{2})[\- ]+(?P
[0-9]+):(?P[0-9]+):(?P[0-9]+)' - m = re.search(rexx,tmp) - #print "year:", int(m.group('YEAR')), "month", int(m.group('MON')), "day", int(m.group('DAY')), "hour", int(m.group('HR')), "minute", int(m.group('MIN')), "second", int(m.group('SEC')) - result = datetime.datetime(int(m.group('YEAR')), int(m.group('MON')), int(m.group('DAY')), int(m.group('HR')), int(m.group('MIN')), int(m.group('SEC'))) + if topline.find("UTC")!=-1: + pos1 = topline.find("-")+2 + pos2 = topline.find("UTC") + tmp=topline[pos1:pos2] + isUTC=True else: - raise FpdbError("invalid site in parseHandStartTime") + tmp=topline + #print "parsehandStartTime, tmp:", tmp + pos = tmp.find("-")+2 + tmp = tmp[pos:] + #Need to match either + # 2008/09/07 06:23:14 ET or + # 2008/08/17 - 01:14:43 (ET) or + # 2008/11/12 9:33:31 CET [2008/11/12 3:33:31 ET] + rexx = '(?P[0-9]{4})\/(?P[0-9]{2})\/(?P[0-9]{2})[\- ]+(?P
[0-9]+):(?P[0-9]+):(?P[0-9]+)' + m = re.search(rexx,tmp) + result = datetime.datetime(int(m.group('YEAR')), int(m.group('MON')), int(m.group('DAY')), int(m.group('HR')), int(m.group('MIN')), int(m.group('SEC'))) - if (site=="ftp" or site=="ps") and not isUTC: #these use US ET + if not isUTC: #these use US ET result+=datetime.timedelta(hours=5) return result @@ -1308,40 +1260,13 @@ def parseSiteHandNo(topline): def parseTableLine(site, base, line): """returns a dictionary with maxSeats and tableName""" - if site=="ps": - pos1=line.find('\'')+1 - pos2=line.find('\'', pos1) - #print "table:",line[pos1:pos2] - pos3=pos2+2 - pos4=line.find("-max") - #print "seats:",line[pos3:pos4] - return {'maxSeats':int(line[pos3:pos4]), 'tableName':line[pos1:pos2]} - elif site=="ftp": - pos1=line.find("Table ")+6 - pos2=line.find("-")-1 - if base=="hold": - maxSeats=9 - elif base=="stud": - maxSeats=8 - - if line.find("6 max")!=-1: - maxSeats=6 - elif line.find("4 max")!=-1: - maxSeats=4 - elif line.find("heads up")!=-1: - maxSeats=2 - - tableName = line[pos1:pos2] - for pattern in [' \(6 max\)', ' \(heads up\)', ' \(deep\)', - ' \(deep hu\)', ' \(deep 6\)', ' \(2\)', - ' \(edu\)', ' \(edu, 6 max\)', ' \(6\)', - ' \(speed\)', - ' no all-in', ' fast', ',', ' 50BB min', '\s+$']: - tableName = re.sub(pattern, '', tableName) - tableName = tableName.rstrip() - return {'maxSeats':maxSeats, 'tableName':tableName} - else: - raise FpdbError("invalid site ID") + pos1=line.find('\'')+1 + pos2=line.find('\'', pos1) + #print "table:",line[pos1:pos2] + pos3=pos2+2 + pos4=line.find("-max") + #print "seats:",line[pos3:pos4] + return {'maxSeats':int(line[pos3:pos4]), 'tableName':line[pos1:pos2]} #end def parseTableLine #returns the hand no assigned by the poker site @@ -1359,17 +1284,11 @@ def parseWinLine(line, site, names, winnings, isTourney): if (line.startswith(names[i].encode("latin-1"))): #found a winner if isTourney: pos1=line.rfind("collected ")+10 - if (site=="ftp"): - pos2=line.find(")", pos1) - elif (site=="ps"): - pos2=line.find(" ", pos1) + pos2=line.find(" ", pos1) winnings[i]+=int(line[pos1:pos2]) else: pos1=line.rfind("$")+1 - if (site=="ftp"): - pos2=line.find(")", pos1) - elif (site=="ps"): - pos2=line.find(" ", pos1) + pos2=line.find(" ", pos1) winnings[i]+=float2int(line[pos1:pos2]) #end def parseWinLine @@ -1413,10 +1332,8 @@ def recogniseGametypeID(backend, db, cursor, topline, smallBlindLine, site_id, c pos1=pos2+2 if isTourney: pos1-=1 - if (site_id==1): #ftp - pos2=topline.find(" ", pos1) - elif (site_id==2): #ps - pos2=topline.find(")") + #ps + pos2=topline.find(")") if pos2<=pos1: pos2=topline.find(")", pos1) @@ -1558,6 +1475,7 @@ def recognisePlayerNo(line, names, atype): #returns the site abbreviation for the given site def recogniseSite(line): + # TODO: Make site agnostic if (line.startswith("Full Tilt Poker")): return "ftp" elif (line.startswith("PokerStars")): From 14b58d6ad538dfb7e930e07b84176d12b1cb8d47 Mon Sep 17 00:00:00 2001 From: Worros Date: Tue, 10 Mar 2009 21:36:52 +0900 Subject: [PATCH 3/5] [KILLFTP 3/x] Fix screwup from patch 1 --- pyfpdb/fpdb_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index fde69125..03de8554 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -354,7 +354,7 @@ class Importer: print "quitting due to reaching the amount of hands to be imported" print "Total stored:", stored, "duplicates:", duplicates, "errors:", errors, " time:", (time() - starttime) sys.exit(0) - startpos=endpos + startpos=endpos ttime = time() - starttime print "\rTotal stored:", stored, "duplicates:", duplicates, "errors:", errors, " time:", ttime From 161485e44e5886d21d5ea3e67ee673e4ce6f7bb6 Mon Sep 17 00:00:00 2001 From: Worros Date: Tue, 10 Mar 2009 22:00:37 +0900 Subject: [PATCH 4/5] [KILLFTP 4/x] paper bag fix caused iun patch 1. This code needs replacing --- pyfpdb/fpdb_import.py | 98 +++++++++++++++++++++++-------------------- 1 file changed, 52 insertions(+), 46 deletions(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 03de8554..c4c4e261 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -208,7 +208,7 @@ class Importer: #if os.path.isdir(file): #self.import_file_dict(file, self.filelist[file][0], self.filelist[file][1]) - self.addToDirList = filter(lambda x: self.addImportDirectory(x, True, self.addToDirList[x][0], self.addToDirList[x][1]), self.addToDirList) + self.addToDirList = filter(lambda x: self.addImportDirectory(x, True, self.addToDirList[x][0], self.addToDirList[x][1]), self.addToDirList) for file in self.removeFromFileList: if file in self.filelist: @@ -302,58 +302,64 @@ class Importer: partial=0 #counter errors=0 #counter - for i in xrange (len(self.lines)): #main loop, iterates through the lines of a file and calls the appropriate parser method - if (len(self.lines[i])<2): + for i in xrange (len(self.lines)): + if (len(self.lines[i])<2): #Wierd way to detect for '\r\n' or '\n' endpos=i hand=self.lines[startpos:endpos] if (len(hand[0])<2): hand=hand[1:] + - isTourney=fpdb_simple.isTourney(hand[0]) - if not isTourney: - fpdb_simple.filterAnteBlindFold(site,hand) - hand=fpdb_simple.filterCrap(site, hand, isTourney) - self.hand=hand + if (len(hand)<3): + pass + #TODO: This is ugly - we didn't actually find the start of the + # hand with the outer loop so we test again... + else: + isTourney=fpdb_simple.isTourney(hand[0]) + if not isTourney: + fpdb_simple.filterAnteBlindFold(site,hand) + hand=fpdb_simple.filterCrap(site, hand, isTourney) + self.hand=hand + + try: + handsId=fpdb_parse_logic.mainParser(self.settings['db-backend'], self.fdb.db + ,self.fdb.cursor, site, category, hand) + self.fdb.db.commit() + + stored+=1 + if self.callHud: + #print "call to HUD here. handsId:",handsId + #pipe the Hands.id out to the HUD + self.caller.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep) + except fpdb_simple.DuplicateError: + duplicates+=1 + except (ValueError), fe: + errors+=1 + self.printEmailErrorMessage(errors, file, hand) + + if (self.settings['failOnError']): + self.fdb.db.commit() #dont remove this, in case hand processing was cancelled. + raise + except (fpdb_simple.FpdbError), fe: + errors+=1 + self.printEmailErrorMessage(errors, file, hand) + + self.fdb.db.rollback() + + if (self.settings['failOnError']): + self.fdb.db.commit() #dont remove this, in case hand processing was cancelled. + raise + if (self.settings['minPrint']!=0): + if ((stored+duplicates+errors)%self.settings['minPrint']==0): + print "stored:", stored, "duplicates:", duplicates, "errors:", errors - try: - handsId=fpdb_parse_logic.mainParser(self.settings['db-backend'], self.fdb.db - ,self.fdb.cursor, site, category, hand) - self.fdb.db.commit() - - stored+=1 - if self.callHud: - #print "call to HUD here. handsId:",handsId - #pipe the Hands.id out to the HUD - self.caller.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep) - except fpdb_simple.DuplicateError: - duplicates+=1 - except (ValueError), fe: - errors+=1 - self.printEmailErrorMessage(errors, file, hand) - - if (self.settings['failOnError']): - self.fdb.db.commit() #dont remove this, in case hand processing was cancelled. - raise - except (fpdb_simple.FpdbError), fe: - errors+=1 - self.printEmailErrorMessage(errors, file, hand) - - self.fdb.db.rollback() - - if (self.settings['failOnError']): - self.fdb.db.commit() #dont remove this, in case hand processing was cancelled. - raise - if (self.settings['minPrint']!=0): - if ((stored+duplicates+errors)%self.settings['minPrint']==0): - print "stored:", stored, "duplicates:", duplicates, "errors:", errors - - if (self.settings['handCount']!=0): - if ((stored+duplicates+errors)>=self.settings['handCount']): - if (not self.settings['quiet']): - print "quitting due to reaching the amount of hands to be imported" - print "Total stored:", stored, "duplicates:", duplicates, "errors:", errors, " time:", (time() - starttime) - sys.exit(0) + if (self.settings['handCount']!=0): + if ((stored+duplicates+errors)>=self.settings['handCount']): + if (not self.settings['quiet']): + print "quitting due to reaching the amount of hands to be imported" + print "Total stored:", stored, "duplicates:", duplicates, "errors:", errors, " time:", (time() - starttime) + sys.exit(0) startpos=endpos ttime = time() - starttime print "\rTotal stored:", stored, "duplicates:", duplicates, "errors:", errors, " time:", ttime From 264b18863339013ee65956a58c490650b05e7496 Mon Sep 17 00:00:00 2001 From: Worros Date: Tue, 10 Mar 2009 23:03:02 +0900 Subject: [PATCH 5/5] [KILLFTP 5/x] Remove references to the named site "PokerStars" or "Full Tilt Poker" --- pyfpdb/fpdb_import.py | 8 ++--- pyfpdb/fpdb_parse_logic.py | 22 +++++++------- pyfpdb/fpdb_simple.py | 61 +++++++++++--------------------------- 3 files changed, 32 insertions(+), 59 deletions(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index c4c4e261..dde5ec6f 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -293,7 +293,7 @@ class Importer: # print "import_fpdb_file", file, site, self.lines, "\n" return (0,0,0,1,0) - site=fpdb_simple.recogniseSite(firstline) + # site=fpdb_simple.recogniseSite(firstline) category=fpdb_simple.recogniseCategory(firstline) startpos=0 @@ -318,13 +318,13 @@ class Importer: else: isTourney=fpdb_simple.isTourney(hand[0]) if not isTourney: - fpdb_simple.filterAnteBlindFold(site,hand) - hand=fpdb_simple.filterCrap(site, hand, isTourney) + fpdb_simple.filterAnteBlindFold(hand) + hand=fpdb_simple.filterCrap(hand, isTourney) self.hand=hand try: handsId=fpdb_parse_logic.mainParser(self.settings['db-backend'], self.fdb.db - ,self.fdb.cursor, site, category, hand) + ,self.fdb.cursor, category, hand) self.fdb.db.commit() stored+=1 diff --git a/pyfpdb/fpdb_parse_logic.py b/pyfpdb/fpdb_parse_logic.py index c5076283..a1f2540f 100644 --- a/pyfpdb/fpdb_parse_logic.py +++ b/pyfpdb/fpdb_parse_logic.py @@ -21,7 +21,7 @@ import fpdb_simple import fpdb_save_to_db #parses a holdem hand -def mainParser(backend, db, cursor, site, category, hand): +def mainParser(backend, db, cursor, category, hand): category=fpdb_simple.recogniseCategory(hand[0]) if (category=="holdem" or category=="omahahi" or category=="omahahilo"): base="hold" @@ -35,8 +35,8 @@ def mainParser(backend, db, cursor, site, category, hand): #part 1: read hand no and check for duplicate siteHandNo=fpdb_simple.parseSiteHandNo(hand[0]) - handStartTime=fpdb_simple.parseHandStartTime(hand[0], site) - siteID=fpdb_simple.recogniseSiteID(cursor, site) + handStartTime=fpdb_simple.parseHandStartTime(hand[0]) + siteID=fpdb_simple.recogniseSiteID() #print "parse logic, siteID:",siteID,"site:",site isTourney=fpdb_simple.isTourney(hand[0]) @@ -51,8 +51,6 @@ def mainParser(backend, db, cursor, site, category, hand): #print "small blind line:",smallBlindLine gametypeID=fpdb_simple.recogniseGametypeID(backend, db, cursor, hand[0], hand[smallBlindLine], siteID, category, isTourney) if isTourney: - if site!="ps": - raise fpdb_simple.FpdbError("tourneys are only supported on PS right now") siteTourneyNo=fpdb_simple.parseTourneyNo(hand[0]) buyin=fpdb_simple.parseBuyin(hand[0]) fee=fpdb_simple.parseFee(hand[0]) @@ -75,7 +73,7 @@ def mainParser(backend, db, cursor, site, category, hand): seatLines.append(hand[i]) names=fpdb_simple.parseNames(seatLines) playerIDs = fpdb_simple.recognisePlayerIDs(cursor, names, siteID) - tmp=fpdb_simple.parseCashesAndSeatNos(seatLines, site) + tmp=fpdb_simple.parseCashesAndSeatNos(seatLines) startCashes=tmp['startCashes'] seatNos=tmp['seatNos'] @@ -88,15 +86,15 @@ def mainParser(backend, db, cursor, site, category, hand): #part 4: take appropriate action for each line based on linetype for i in range(len(hand)): if (lineTypes[i]=="cards"): - fpdb_simple.parseCardLine (site, category, lineStreets[i], hand[i], names, cardValues, cardSuits, boardValues, boardSuits) + fpdb_simple.parseCardLine (category, lineStreets[i], hand[i], names, cardValues, cardSuits, boardValues, boardSuits) #if category=="studhilo": # print "hand[i]:", hand[i] # print "cardValues:", cardValues # print "cardSuits:", cardSuits elif (lineTypes[i]=="action"): - fpdb_simple.parseActionLine (site, base, isTourney, hand[i], lineStreets[i], playerIDs, names, actionTypes, allIns, actionAmounts, actionNos, actionTypeByNo) + fpdb_simple.parseActionLine (base, isTourney, hand[i], lineStreets[i], playerIDs, names, actionTypes, allIns, actionAmounts, actionNos, actionTypeByNo) elif (lineTypes[i]=="win"): - fpdb_simple.parseWinLine (hand[i], site, names, winnings, isTourney) + fpdb_simple.parseWinLine (hand[i], names, winnings, isTourney) elif (lineTypes[i]=="rake"): if isTourney: totalRake=0 @@ -106,9 +104,9 @@ def mainParser(backend, db, cursor, site, category, hand): elif (lineTypes[i]=="header" or lineTypes[i]=="rake" or lineTypes[i]=="name" or lineTypes[i]=="ignore"): pass elif (lineTypes[i]=="ante"): - fpdb_simple.parseAnteLine(hand[i], site, isTourney, names, antes) + fpdb_simple.parseAnteLine(hand[i], isTourney, names, antes) elif (lineTypes[i]=="table"): - tableResult=fpdb_simple.parseTableLine(site, base, hand[i]) + tableResult=fpdb_simple.parseTableLine(base, hand[i]) else: raise fpdb_simple.FpdbError("unrecognised lineType:"+lineTypes[i]) maxSeats=tableResult['maxSeats'] @@ -125,7 +123,7 @@ def mainParser(backend, db, cursor, site, category, hand): cursor.execute("SELECT limitType FROM Gametypes WHERE id=%s",(gametypeID, )) limit_type=cursor.fetchone()[0] - fpdb_simple.convert3B4B(site, category, limit_type, actionTypes, actionAmounts) + fpdb_simple.convert3B4B(category, limit_type, actionTypes, actionAmounts) totalWinnings=0 for i in range(len(winnings)): diff --git a/pyfpdb/fpdb_simple.py b/pyfpdb/fpdb_simple.py index d3bb4097..cad7e046 100644 --- a/pyfpdb/fpdb_simple.py +++ b/pyfpdb/fpdb_simple.py @@ -458,7 +458,7 @@ def classifyLines(hand, category, lineTypes, lineStreets): lineStreets.append(currentStreet) #end def classifyLines -def convert3B4B(site, category, limit_type, actionTypes, actionAmounts): +def convert3B4B(category, limit_type, actionTypes, actionAmounts): """calculates the actual bet amounts in the given amount array and changes it accordingly.""" for i in range (len(actionTypes)): for j in range (len(actionTypes[i])): @@ -474,7 +474,7 @@ def convert3B4B(site, category, limit_type, actionTypes, actionAmounts): elif (len(bets)>2): fail=True #todo: run correction for below - if (site=="ps" and category=="holdem" and limit_type=="nl" and len(bets)==3): + if (limit_type=="nl" or limit_type == "pl"): fail=False if fail: @@ -611,7 +611,7 @@ def fillCardArrays(player_count, base, category, card_values, card_suits): #filters out a player that folded before paying ante or blinds. This should be called #before calling the actual hand parser. manipulates hand, no return. -def filterAnteBlindFold(site,hand): +def filterAnteBlindFold(hand): #todo: this'll only get rid of one ante folder, not multiple ones #todo: in tourneys this should not be removed but #print "start of filterAnteBlindFold" @@ -649,7 +649,7 @@ def filterAnteBlindFold(site,hand): #end def filterAnteFold #removes useless lines as well as trailing spaces -def filterCrap(site, hand, isTourney): +def filterCrap(hand, isTourney): #remove two trailing spaces at end of line for i in range (len(hand)): if (hand[i][-1]==' '): @@ -714,12 +714,8 @@ def filterCrap(site, hand, isTourney): toRemove.append(hand[i]) elif (hand[i].find("joins the table at seat ")!=-1): toRemove.append(hand[i]) - elif (hand[i].endswith(" sits down")): - toRemove.append(hand[i]) elif (hand[i].endswith("leaves the table")): toRemove.append(hand[i]) - elif (hand[i].endswith(" stands up")): - toRemove.append(hand[i]) elif (hand[i].find("is high with ")!=-1): toRemove.append(hand[i]) elif (hand[i].endswith("doesn't show hand")): @@ -862,7 +858,7 @@ def isWinLine(line): #end def isWinLine #returns the amount of cash/chips put into the put in the given action line -def parseActionAmount(line, atype, site, isTourney): +def parseActionAmount(line, atype, isTourney): #if (line.endswith(" and is all-in")): # line=line[:-14] #elif (line.endswith(", and is all in")): @@ -878,14 +874,14 @@ def parseActionAmount(line, atype, site, isTourney): amount=0 elif (atype=="check"): amount=0 - elif (atype=="unbet" and site=="ps"): + elif (atype=="unbet"): #print "ps unbet, line:",line pos1=line.find("$")+1 if pos1==0: pos1=line.find("(")+1 pos2=line.find(")") amount=float2int(line[pos1:pos2]) - elif (atype=="bet" and site=="ps" and line.find(": raises $")!=-1 and line.find("to $")!=-1): + elif (atype=="bet" and line.find(": raises $")!=-1 and line.find("to $")!=-1): pos=line.find("to $")+4 amount=float2int(line[pos:]) else: @@ -908,7 +904,7 @@ def parseActionAmount(line, atype, site, isTourney): #doesnt return anything, simply changes the passed arrays action_types and # action_amounts. For stud this expects numeric streets (3-7), for # holdem/omaha it expects predeal, preflop, flop, turn or river -def parseActionLine(site, base, isTourney, line, street, playerIDs, names, action_types, allIns, action_amounts, actionNos, actionTypeByNo): +def parseActionLine(base, isTourney, line, street, playerIDs, names, action_types, allIns, action_amounts, actionNos, actionTypeByNo): if (street=="predeal" or street=="preflop"): street=0 elif (street=="flop"): @@ -927,7 +923,7 @@ def parseActionLine(site, base, isTourney, line, street, playerIDs, names, actio line, allIn=goesAllInOnThisLine(line) atype=parseActionType(line) playerno=recognisePlayerNo(line, names, atype) - amount=parseActionAmount(line, atype, site, isTourney) + amount=parseActionAmount(line, atype, isTourney) action_types[street][playerno].append(atype) allIns[street][playerno].append(allIn) @@ -988,7 +984,7 @@ def parseActionType(line): #end def parseActionType #parses the ante out of the given line and checks which player paid it, updates antes accordingly. -def parseAnteLine(line, site, isTourney, names, antes): +def parseAnteLine(line, isTourney, names, antes): for i in range(len(names)): if (line.startswith(names[i].encode("latin-1"))): #found the ante'er pos=line.rfind("$")+1 @@ -1014,7 +1010,7 @@ def parseBuyin(topline): #parses a card line and changes the passed arrays accordingly #todo: reorganise this messy method -def parseCardLine(site, category, street, line, names, cardValues, cardSuits, boardValues, boardSuits): +def parseCardLine(category, street, line, names, cardValues, cardSuits, boardValues, boardSuits): if (line.startswith("Dealt to ") or line.find(" shows [")!=-1 or line.find("mucked [")!=-1): playerNo=recognisePlayerNo(line, names, "card") #anything but unbet will be ok for that string @@ -1088,7 +1084,7 @@ def parseCardLine(site, category, street, line, names, cardValues, cardSuits, bo raise FpdbError ("unrecognised line:"+line) #end def parseCardLine -def parseCashesAndSeatNos(lines, site): +def parseCashesAndSeatNos(lines): """parses the startCashes and seatNos of each player out of the given lines and returns them as a dictionary of two arrays""" cashes = [] seatNos = [] @@ -1099,8 +1095,7 @@ def parseCashesAndSeatNos(lines, site): pos1=lines[i].rfind("($")+2 if pos1==1: #for tourneys - it's 1 instead of -1 due to adding 2 above pos1=lines[i].rfind("(")+1 - elif (site=="ps"): - pos2=lines[i].find(" in chips") + pos2=lines[i].find(" in chips") cashes.append(float2int(lines[i][pos1:pos2])) return {'startCashes':cashes, 'seatNos':seatNos} #end def parseCashesAndSeatNos @@ -1114,7 +1109,7 @@ def parseFee(topline): #end def parsefee #returns a datetime object with the starttime indicated in the given topline -def parseHandStartTime(topline, site): +def parseHandStartTime(topline): #convert x:13:35 to 0x:13:35 counter=0 while (True): @@ -1258,7 +1253,7 @@ def parseSiteHandNo(topline): return topline[pos1:pos2] #end def parseSiteHandNo -def parseTableLine(site, base, line): +def parseTableLine(base, line): """returns a dictionary with maxSeats and tableName""" pos1=line.find('\'')+1 pos2=line.find('\'', pos1) @@ -1278,7 +1273,7 @@ def parseTourneyNo(topline): #end def parseTourneyNo #parses a win/collect line. manipulates the passed array winnings, no explicit return -def parseWinLine(line, site, names, winnings, isTourney): +def parseWinLine(line, names, winnings, isTourney): #print "parseWinLine: line:",line for i in range(len(names)): if (line.startswith(names[i].encode("latin-1"))): #found a winner @@ -1473,28 +1468,8 @@ def recognisePlayerNo(line, names, atype): raise FpdbError ("failed to recognise player in: "+line+" atype:"+atype) #end def recognisePlayerNo -#returns the site abbreviation for the given site -def recogniseSite(line): - # TODO: Make site agnostic - if (line.startswith("Full Tilt Poker")): - return "ftp" - elif (line.startswith("PokerStars")): - return "ps" - else: - raise FpdbError("failed to recognise site, line:"+line) -#end def recogniseSite - -#returns the ID of the given site -def recogniseSiteID(cursor, site): - if (site=="ftp"): - return 1 - #cursor.execute("SELECT id FROM Sites WHERE name = ('Full Tilt Poker')") - elif (site=="ps"): - return 2 - #cursor.execute("SELECT id FROM Sites WHERE name = ('PokerStars')") - else: - raise FpdbError("invalid site in recogniseSiteID: "+site) - return cursor.fetchall()[0][0] +def recogniseSiteID(): + return 2 #end def recogniseSiteID #removes trailing \n from the given array