From 5611f115cddeae2948a2e538860a7d15baf114b1 Mon Sep 17 00:00:00 2001 From: Worros Date: Tue, 30 Jun 2009 18:32:30 +0800 Subject: [PATCH 01/11] Remove DEBUG line that wasn't supposed to make the last commit --- pyfpdb/Hand.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyfpdb/Hand.py b/pyfpdb/Hand.py index 38c3609b..61e06930 100644 --- a/pyfpdb/Hand.py +++ b/pyfpdb/Hand.py @@ -202,7 +202,6 @@ If a player has None chips he won't be added.""" def setCommunityCards(self, street, cards): logging.debug("setCommunityCards %s %s" %(street, cards)) self.board[street] = [self.card(c) for c in cards] - print "DEBUG: self.board: %s" % self.board def card(self,c): """upper case the ranks but not suits, 'atjqk' => 'ATJQK'""" From 3c438a92985881430a5705d2705237f021a11c83 Mon Sep 17 00:00:00 2001 From: Worros Date: Tue, 30 Jun 2009 18:32:58 +0800 Subject: [PATCH 02/11] Make site selection behave like other options --- pyfpdb/GuiGraphViewer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyfpdb/GuiGraphViewer.py b/pyfpdb/GuiGraphViewer.py index cec91e61..2dfb0173 100644 --- a/pyfpdb/GuiGraphViewer.py +++ b/pyfpdb/GuiGraphViewer.py @@ -154,7 +154,7 @@ class GuiGraphViewer (threading.Thread): if not sitenos: #Should probably pop up here. print "No sites selected - defaulting to PokerStars" - sitenos = [2] + return if not playerids: print "No player ids found" From 79d0ba35b0294782567dfa3648bd7f5d4bde616c Mon Sep 17 00:00:00 2001 From: Worros Date: Tue, 14 Jul 2009 06:39:49 +0800 Subject: [PATCH 03/11] Initial commit of BossMedia importer. Contributed by Thomas Wigele (twigele gmail com - star-fox on sourceforge) Sample HH - http://www.pastebin.ca/1490652 --- pyfpdb/Win2dayToFpdb.py | 384 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 384 insertions(+) create mode 100644 pyfpdb/Win2dayToFpdb.py diff --git a/pyfpdb/Win2dayToFpdb.py b/pyfpdb/Win2dayToFpdb.py new file mode 100644 index 00000000..0417dcb1 --- /dev/null +++ b/pyfpdb/Win2dayToFpdb.py @@ -0,0 +1,384 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2008, Carl Gherardi +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +######################################################################## + +import sys +from HandHistoryConverter import * + +# Win2day HH Format + +class Win2day(HandHistoryConverter): + + # Static regexes + # + + #'^$' + re_GameInfo = re.compile('^', re.MULTILINE) + re_SplitHands = re.compile('') + re_HandInfo = re.compile("^Table \'(?P[- a-zA-Z]+)\'(?P.+?$)?", re.MULTILINE) + re_Button = re.compile('\n\n\n + re_PlayerInfo = re.compile('^', re.MULTILINE) + re_Card = re.compile('^', re.MULTILINE) + re_BoardLast = re.compile('^', re.MULTILINE) + + def __init__(self, in_path = '-', out_path = '-', follow = False, autostart=True): + HandHistoryConverter.__init__(self, in_path, out_path, sitename="Win2day", follow=follow) + logging.info("Initialising Win2day converter class") + self.filetype = "text" + self.codepage = "cp1252" + self.sideID = 4 + if autostart: + self.start() + + + def compilePlayerRegexs(self, hand): + players = set([player[1] for player in hand.players]) + if not players <= self.compiledPlayers: # x <= y means 'x is subset of y' + # we need to recompile the player regexs. + self.compiledPlayers = players + player_re = "(?P" + "|".join(map(re.escape, players)) + ")" + logging.debug("player_re: " + player_re) + # + + self.re_PostSB = re.compile(r'^' % player_re, re.MULTILINE) + self.re_PostBB = re.compile(r'^' % player_re, re.MULTILINE) + self.re_Antes = re.compile(r"^%s: posts the ante \$?(?P[.0-9]+)" % player_re, re.MULTILINE) + self.re_BringIn = re.compile(r"^%s: brings[- ]in( low|) for \$?(?P[.0-9]+)" % player_re, re.MULTILINE) + self.re_PostBoth = re.compile(r'^' % player_re, re.MULTILINE) + + #r'\n\n' + self.re_HeroCards = re.compile(r'\n(?P\n)' % player_re, re.MULTILINE) + + #'^' + self.re_Action = re.compile(r'^' % player_re, re.MULTILINE) + + self.re_ShowdownAction = re.compile(r'\n(?P\n)' % player_re, re.MULTILINE) + # + # + self.re_CollectPot = re.compile(r'' % player_re, re.MULTILINE) + self.re_sitsOut = re.compile("^%s sits out" % player_re, re.MULTILINE) + self.re_ShownCards = re.compile("^Seat (?P[0-9]+): %s \(.*\) showed \[(?P.*)\].*" % player_re, re.MULTILINE) + + + def readSupportedGames(self): + return [["ring", "hold", "nl"], + ["ring", "hold", "pl"], + ["ring", "hold", "fl"], + ["ring", "stud", "fl"], + ["ring", "draw", "fl"], + ["ring", "omaha", "pl"] + ] + + def determineGameType(self, handText): + info = {'type':'ring'} + + m = self.re_GameInfo.search(handText) + if not m: + print "determineGameType:", handText + return None + + mg = m.groupdict() + + # translations from captured groups to our info strings + #limits = { 'NL':'nl', 'PL':'pl', 'Limit':'fl' } + limits = { 'NL':'nl', 'PL':'pl'} + games = { # base, category + "GAME_THM" : ('hold','holdem'), + # 'Omaha' : ('hold','omahahi'), + #'Omaha Hi/Lo' : ('hold','omahahilo'), + # 'Razz' : ('stud','razz'), + #'7 Card Stud' : ('stud','studhi'), + # 'Badugi' : ('draw','badugi') + } + if 'LIMIT' in mg: + info['limitType'] = limits[mg['LIMIT']] + if 'GAME' in mg: + (info['base'], info['category']) = games[mg['GAME']] + if 'SB' in mg: + info['sb'] = mg['SB'] + if 'BB' in mg: + info['bb'] = mg['BB'] + if 'CURRENCY' in mg: + info['currency'] = mg['CURRENCY'] + # NB: SB, BB must be interpreted as blinds or bets depending on limit type. + + return info + + + def readHandInfo(self, hand): + info = {} + m = self.re_HandInfo.search(hand.handText,re.DOTALL) + if m: + info.update(m.groupdict()) + # TODO: Be less lazy and parse maxseats from the HandInfo regex + if m.group('TABLEATTRIBUTES'): + m2 = re.search("\s*(\d+)-max", m.group('TABLEATTRIBUTES')) + hand.maxseats = int(m2.group(1)) + m = self.re_GameInfo.search(hand.handText) + if m: info.update(m.groupdict()) + m = self.re_Button.search(hand.handText) + if m: info.update(m.groupdict()) + # TODO : I rather like the idea of just having this dict as hand.info + logging.debug("readHandInfo: %s" % info) + for key in info: + if key == 'DATETIME': + # Win2day uses UTC timestamp + # m2 = re.search("(?P[0-9]{4})\/(?P[0-9]{2})\/(?P[0-9]{2})[\- ]+(?P[0-9]+):(?P[0-9]+):(?P[0-9]+)", info[key]) + # datetime = "%s/%s/%s %s:%s:%s" % (m2.group('Y'), m2.group('M'),m2.group('D'),m2.group('H'),m2.group('MIN'),m2.group('S')) + # hand.starttime = time.strptime(time.gmtime(info[key])) + # hand.starttime = time.gmtime(int(info[key])) + hand.starttime = time.gmtime(int(info[key])) + if key == 'HID': + hand.handid = info[key] + if key == 'TABLE': + hand.tablename = info[key] + if key == 'BUTTON': + hand.buttonpos = info[key] + + def readButton(self, hand): + m = self.re_Button.search(hand.handText) + if m: + for player in hand.players: + if player[1] == m.group('BUTTON'): + hand.buttonpos = player[0] + break + else: + logging.info('readButton: not found') + + def readPlayerStacks(self, hand): + logging.debug("readPlayerStacks") + m = self.re_PlayerInfo.finditer(hand.handText) + players = [] + for a in m: + hand.addPlayer(int(a.group('SEAT')), a.group('PNAME'), a.group('CASH')) + + def markStreets(self, hand): + # PREFLOP = ** Dealing down cards ** + # This re fails if, say, river is missing; then we don't get the ** that starts the river. + if hand.gametype['base'] in ("hold"): + #m = re.search(r"\*\*\* HOLE CARDS \*\*\*(?P.+(?=\*\*\* FLOP \*\*\*)|.+)" + # r"(\*\*\* FLOP \*\*\*(?P \[\S\S \S\S \S\S\].+(?=\*\*\* TURN \*\*\*)|.+))?" + # r"(\*\*\* TURN \*\*\* \[\S\S \S\S \S\S] (?P\[\S\S\].+(?=\*\*\* RIVER \*\*\*)|.+))?" + # r"(\*\*\* RIVER \*\*\* \[\S\S \S\S \S\S \S\S] (?P\[\S\S\].+))?", hand.handText,re.DOTALL) + + m = re.search('(?P.+(?=.+(?=.+(?=.+(?= 38): + retCard += 's' + elif(card > 25): + retCard += 'h' + elif(card > 12): + retCard += 'c' + else: + retCard += 'd' + + return(retCard) + + def readDrawCards(self, hand, street): + logging.debug("readDrawCards") + m = self.re_HeroCards.finditer(hand.streets[street]) + if m == None: + hand.involved = False + else: + for player in m: + hand.hero = player.group('PNAME') # Only really need to do this once + newcards = player.group('NEWCARDS') + oldcards = player.group('OLDCARDS') + if newcards == None: + newcards = set() + else: + newcards = set(newcards.split(' ')) + if oldcards == None: + oldcards = set() + else: + oldcards = set(oldcards.split(' ')) + hand.addDrawHoleCards(newcards, oldcards, player.group('PNAME'), street) + + + def readStudPlayerCards(self, hand, street): + # See comments of reference implementation in FullTiltToFpdb.py + logging.debug("readStudPlayerCards") + m = self.re_HeroCards.finditer(hand.streets[street]) + for player in m: + #~ logging.debug(player.groupdict()) + (pname, oldcards, newcards) = (player.group('PNAME'), player.group('OLDCARDS'), player.group('NEWCARDS')) + if oldcards: + oldcards = [c.strip() for c in oldcards.split(' ')] + if newcards: + newcards = [c.strip() for c in newcards.split(' ')] + if street=='ANTES': + return + elif street=='THIRD': + # we'll have observed hero holecards in CARDS and thirdstreet open cards in 'NEWCARDS' + # hero: [xx][o] + # others: [o] + hand.addPlayerCards(player = player.group('PNAME'), street = street, closed = oldcards, open = newcards) + elif street in ('FOURTH', 'FIFTH', 'SIXTH'): + # 4th: + # hero: [xxo] [o] + # others: [o] [o] + # 5th: + # hero: [xxoo] [o] + # others: [oo] [o] + # 6th: + # hero: [xxooo] [o] + # others: [ooo] [o] + hand.addPlayerCards(player = player.group('PNAME'), street = street, open = newcards) + # we may additionally want to check the earlier streets tally with what we have but lets trust it for now. + elif street=='SEVENTH' and newcards: + # hero: [xxoooo] [x] + # others: not reported. + hand.addPlayerCards(player = player.group('PNAME'), street = street, closed = newcards) + + def readAction(self, hand, street): + m = self.re_Action.finditer(hand.streets[street]) + for action in m: + if action.group('ATYPE') == 'ACTION_RAISE': + hand.addRaiseBy( street, action.group('PNAME'), action.group('BET') ) + elif action.group('ATYPE') == 'ACTION_CALL': + hand.addCall( street, action.group('PNAME'), action.group('BET') ) + elif action.group('ATYPE') == 'ACTION_ALLIN': + hand.addRaiseBy( street, action.group('PNAME'), action.group('BET') ) + elif action.group('ATYPE') == 'ACTION_BET': + hand.addBet( street, action.group('PNAME'), action.group('BET') ) + elif action.group('ATYPE') == 'ACTION_FOLD': + hand.addFold( street, action.group('PNAME')) + elif action.group('ATYPE') == 'ACTION_CHECK': + hand.addCheck( street, action.group('PNAME')) + elif action.group('ATYPE') == 'ACTION_DISCARD': + hand.addDiscard(street, action.group('PNAME'), action.group('NODISCARDED'), action.group('DISCARDED')) + elif action.group('ATYPE') == 'ACTION_STAND': + hand.addStandsPat( street, action.group('PNAME')) + else: + print "DEBUG: unimplemented readAction: '%s' '%s'" %(action.group('PNAME'),action.group('ATYPE'),) + + + def readShowdownActions(self, hand): + for shows in self.re_ShowdownAction.finditer(hand.handText): + showdownCards = set([]) + for card in self.re_Card.finditer(shows.group('CARDS')): + print "DEBUG:", card, card.group('CARD'), self.convertWin2dayCards(card.group('CARD')) + showdownCards.add(self.convertWin2dayCards(card.group('CARD'))) + + hand.addShownCards(showdownCards, shows.group('PNAME')) + + def readCollectPot(self,hand): + for m in self.re_CollectPot.finditer(hand.handText): + potcoll = Decimal(m.group('POT')) + if potcoll > 0: + hand.addCollectPot(player=m.group('PNAME'),pot=potcoll) + + def readShownCards(self,hand): + for m in self.re_ShownCards.finditer(hand.handText): + if m.group('CARDS') is not None: + cards = m.group('CARDS') + cards = set(cards.split(' ')) + hand.addShownCards(cards=cards, player=m.group('PNAME')) + +if __name__ == "__main__": + parser = OptionParser() + parser.add_option("-i", "--input", dest="ipath", help="parse input hand history", default="-") + parser.add_option("-o", "--output", dest="opath", help="output translation to", default="-") + parser.add_option("-f", "--follow", dest="follow", help="follow (tail -f) the input", action="store_true", default=False) + parser.add_option("-q", "--quiet", + action="store_const", const=logging.CRITICAL, dest="verbosity", default=logging.INFO) + parser.add_option("-v", "--verbose", + action="store_const", const=logging.INFO, dest="verbosity") + parser.add_option("--vv", + action="store_const", const=logging.DEBUG, dest="verbosity") + + (options, args) = parser.parse_args() + + LOG_FILENAME = './logging.out' + logging.basicConfig(filename=LOG_FILENAME,level=options.verbosity) + + e = Win2day(in_path = options.ipath, out_path = options.opath, follow = options.follow) From dc6a1c45f9fae72155dde1dc97dd404dc459a4a8 Mon Sep 17 00:00:00 2001 From: Matt Turnbull Date: Tue, 14 Jul 2009 00:04:10 +0100 Subject: [PATCH 04/11] sqlite - connecting and (re)creating some tables --- pyfpdb/Configuration.py | 2 + pyfpdb/FpdbSQLQueries.py | 66 +++++++++++++++--- pyfpdb/fpdb.py | 8 ++- pyfpdb/fpdb_db.py | 144 ++++++++++++++++++++++++++++----------- pyfpdb/fpdb_import.py | 1 + 5 files changed, 170 insertions(+), 51 deletions(-) diff --git a/pyfpdb/Configuration.py b/pyfpdb/Configuration.py index 0ea4b371..27e06b3d 100755 --- a/pyfpdb/Configuration.py +++ b/pyfpdb/Configuration.py @@ -493,6 +493,8 @@ class Config: db['db-backend'] = 2 elif string.lower(self.supported_databases[name].db_server) == 'postgresql': db['db-backend'] = 3 + elif string.lower(self.supported_databases[name].db_server) == 'sqlite': + db['db-backend'] = 4 else: db['db-backend'] = None # this is big trouble return db diff --git a/pyfpdb/FpdbSQLQueries.py b/pyfpdb/FpdbSQLQueries.py index 598c0868..b7e5679e 100644 --- a/pyfpdb/FpdbSQLQueries.py +++ b/pyfpdb/FpdbSQLQueries.py @@ -38,7 +38,9 @@ class FpdbSQLQueries: elif(self.dbname == 'PostgreSQL'): self.query['list_tables'] = """SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'""" elif(self.dbname == 'SQLite'): - self.query['list_tables'] = """ """ + self.query['list_tables'] = """SELECT name FROM sqlite_master + WHERE type='table' + ORDER BY name;""" ################################################################## # Drop Tables - MySQL, PostgreSQL and SQLite all share same syntax @@ -63,8 +65,8 @@ class FpdbSQLQueries: self.query['createSettingsTable'] = """CREATE TABLE Settings (version SMALLINT)""" elif(self.dbname == 'SQLite'): - #Probably doesn't work. - self.query['createSettingsTable'] = """ """ + self.query['createSettingsTable'] = """CREATE TABLE Settings + (version INTEGER) """ ################################ @@ -83,7 +85,10 @@ class FpdbSQLQueries: name varchar(32), currency char(3))""" elif(self.dbname == 'SQLite'): - self.query['createSitesTable'] = """ """ + self.query['createSitesTable'] = """CREATE TABLE Sites ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + currency TEXT NOT NULL)""" ################################ @@ -118,7 +123,19 @@ class FpdbSQLQueries: smallBet int, bigBet int)""" elif(self.dbname == 'SQLite'): - self.query['createGametypesTable'] = """ """ + self.query['createGametypesTable'] = """CREATE TABLE GameTypes ( + id INTEGER PRIMARY KEY, + siteId INTEGER, + type TEXT, + base TEXT, + category TEXT, + limitType TEXT, + hiLo TEXT, + smallBlind INTEGER, + bigBlind INTEGER, + smallBet INTEGER, + bigBet INTEGER, + FOREIGN KEY(siteId) REFERENCES Sites(id) ON DELETE CASCADE)""" ################################ @@ -141,7 +158,13 @@ class FpdbSQLQueries: comment text, commentTs timestamp without time zone)""" elif(self.dbname == 'SQLite'): - self.query['createPlayersTable'] = """ """ + self.query['createPlayersTable'] = """CREATE TABLE Players ( + id INTEGER PRIMARY KEY, + name TEXT, + siteId INTEGER, + comment TEXT, + commentTs BLOB, + FOREIGN KEY(siteId) REFERENCES Sites(id) ON DELETE CASCADE)""" ################################ @@ -245,11 +268,22 @@ class FpdbSQLQueries: comment TEXT, commentTs timestamp without time zone)""" elif(self.dbname == 'SQLite'): - self.query['createHandsTable'] = """ """ + self.query['createHandsTable'] = """CREATE TABLE Hands ( + id INTEGER PRIMARY KEY, + tableName TEXT(20), + siteHandNo INTEGER, + gametypeId INTEGER, + handStart BLOB, + importTime BLOB, + seats INTEGER, + maxSeats INTEGER, + comment TEXT, + commentTs BLOB, + FOREIGN KEY(gametypeId) REFERENCES Gametypes(id) ON DELETE CASCADE)""" ################################ - # Create Gametypes + # Create BoardCards ################################ if(self.dbname == 'MySQL InnoDB'): @@ -336,7 +370,14 @@ class FpdbSQLQueries: comment TEXT, commentTs timestamp without time zone)""" elif(self.dbname == 'SQLite'): - self.query['createTourneysTable'] = """ """ + self.query['createTourneysTable'] = """CREATE TABLE TourneyTypes ( + id INTEGER PRIMARY KEY, + siteId INTEGER, + buyin INTEGER, + fee INTEGER, + knockout INTEGER, + rebuyOrAddon BOOL, + FOREIGN KEY(siteId) REFERENCES Sites(id) ON DELETE CASCADE)""" ################################ # Create HandsPlayers @@ -869,6 +910,13 @@ class FpdbSQLQueries: elif(self.dbname == 'SQLite'): self.query['addPlayersIndex'] = """ """ + + if(self.dbname == 'MySQL InnoDB' or self.dbname == 'PostgreSQL'): + self.query['set tx level'] = """SET SESSION TRANSACTION + ISOLATION LEVEL READ COMMITTED""" + elif(self.dbname == 'SQLite'): + self.query['set tx level'] = """ """ + ################################ # Queries used in GuiGraphViewer ################################ diff --git a/pyfpdb/fpdb.py b/pyfpdb/fpdb.py index b03e8424..61f5f111 100755 --- a/pyfpdb/fpdb.py +++ b/pyfpdb/fpdb.py @@ -19,7 +19,6 @@ import os import sys import Options import string - cl_options = string.join(sys.argv[1:]) (options, sys.argv) = Options.fpdb_options() @@ -28,6 +27,8 @@ if not options.errorsToConsole: errorFile = open('fpdb-error-log.txt', 'w', 0) sys.stderr = errorFile +import logging + import pygtk pygtk.require('2.0') import gtk @@ -481,6 +482,11 @@ This program is licensed under the AGPL3, see docs"""+os.sep+"agpl-3.0.txt") #end def tabGraphViewer def __init__(self): + LOG_FILENAME = './logging.out' + LOG_FORMAT = "%(asctime)-15s %(levelname)-8s %(message)s" + logging.basicConfig(filename=LOG_FILENAME,level=10,format=LOG_FORMAT) + logging.info("Fpdb started") + self.threads=[] self.db=None self.config = Configuration.Config(file=options.config, dbname=options.dbname) diff --git a/pyfpdb/fpdb_db.py b/pyfpdb/fpdb_db.py index 29817af9..c52a2e56 100644 --- a/pyfpdb/fpdb_db.py +++ b/pyfpdb/fpdb_db.py @@ -18,20 +18,21 @@ import os import re import sys +import logging from time import time, strftime import fpdb_simple import FpdbSQLQueries class fpdb_db: + MYSQL_INNODB = 2 + PGSQL = 3 + SQLITE = 4 def __init__(self): """Simple constructor, doesnt really do anything""" self.db = None self.cursor = None self.sql = {} - self.MYSQL_INNODB = 2 - self.PGSQL = 3 - self.SQLITE = 4 # Data Structures for index and foreign key creation # drop_code is an int with possible values: 0 - don't drop for bulk import @@ -72,6 +73,8 @@ class fpdb_db: , {'tab':'TourneysPlayers', 'col':'tourneyId', 'drop':0} , {'tab':'TourneyTypes', 'col':'siteId', 'drop':0} ] + , [ # indexes for sqlite (list index 4) + ] ] self.foreignKeys = [ @@ -146,12 +149,12 @@ class fpdb_db: self.settings = {} self.settings['os'] = "linuxmac" if os.name != "nt" else "windows" - self.settings.update(config.get_db_parameters()) - self.connect(self.settings['db-backend'], - self.settings['db-host'], - self.settings['db-databaseName'], - self.settings['db-user'], - self.settings['db-password']) + db = config.get_db_parameters() + self.connect(backend=db['db-backend'], + host=db['db-host'], + database=db['db-databaseName'], + user=db['db-user'], + password=db['db-password']) #end def do_connect def connect(self, backend=None, host=None, database=None, @@ -164,13 +167,13 @@ class fpdb_db: self.user=user self.password=password self.database=database - if backend==self.MYSQL_INNODB: + if backend==fpdb_db.MYSQL_INNODB: import MySQLdb try: self.db = MySQLdb.connect(host = host, user = user, passwd = password, db = database, use_unicode=True) except: raise fpdb_simple.FpdbError("MySQL connection failed") - elif backend==self.PGSQL: + elif backend==fpdb_db.PGSQL: import psycopg2 import psycopg2.extensions psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) @@ -201,12 +204,19 @@ class fpdb_db: msg = "PostgreSQL connection to database (%s) user (%s) failed." % (database, user) print msg raise fpdb_simple.FpdbError(msg) + elif backend==fpdb_db.SQLITE: + logging.info("Connecting to SQLite:(%s)") + import sqlite3 + self.db = sqlite3.connect(database,detect_types=sqlite3.PARSE_DECLTYPES) + sqlite3.register_converter("bool", lambda x: bool(int(x))) + sqlite3.register_adapter(bool, lambda x: "1" if x else "0") + else: raise fpdb_simple.FpdbError("unrecognised database backend:"+backend) self.cursor=self.db.cursor() - self.cursor.execute('SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED') # Set up query dictionary as early in the connection process as we can. self.sql = FpdbSQLQueries.FpdbSQLQueries(self.get_backend_name()) + self.cursor.execute(self.sql.query['set tx level']) self.wrongDbVersion=False try: self.cursor.execute("SELECT * FROM Settings") @@ -237,7 +247,9 @@ class fpdb_db: def create_tables(self): #todo: should detect and fail gracefully if tables already exist. + logging.debug(self.sql.query['createSettingsTable']) self.cursor.execute(self.sql.query['createSettingsTable']) + logging.debug(self.sql.query['createSitesTable']) self.cursor.execute(self.sql.query['createSitesTable']) self.cursor.execute(self.sql.query['createGametypesTable']) self.cursor.execute(self.sql.query['createPlayersTable']) @@ -275,10 +287,12 @@ class fpdb_db: for table in tables: self.cursor.execute(self.sql.query['drop_table'] + table[0] + ' cascade') elif(self.get_backend_name() == 'SQLite'): - #todo: sqlite version here - print "Empty function here" + self.cursor.execute(self.sql.query['list_tables']) + for table in self.cursor.fetchall(): + logging.debug(self.sql.query['drop_table'] + table[0]) + self.cursor.execute(self.sql.query['drop_table'] + table[0]) - self.db.commit() + self.db.commit() #end def drop_tables def drop_referential_integrity(self): @@ -307,6 +321,8 @@ class fpdb_db: return "MySQL InnoDB" elif self.backend==3: return "PostgreSQL" + elif self.backend==4: + return "SQLite" else: raise fpdb_simple.FpdbError("invalid backend") #end def get_backend_name @@ -316,11 +332,14 @@ class fpdb_db: #end def get_db_info def fillDefaultData(self): - self.cursor.execute("INSERT INTO Settings VALUES (118);") - self.cursor.execute("INSERT INTO Sites VALUES (DEFAULT, 'Full Tilt Poker', 'USD');") - self.cursor.execute("INSERT INTO Sites VALUES (DEFAULT, 'PokerStars', 'USD');") - self.cursor.execute("INSERT INTO Sites VALUES (DEFAULT, 'Everleaf', 'USD');") - self.cursor.execute("INSERT INTO TourneyTypes VALUES (DEFAULT, 1, 0, 0, 0, False);") + logging.debug("INSERT INTO Settings (version) VALUES (118);") + self.cursor.execute("INSERT INTO Settings (version) VALUES (118);") + self.cursor.execute("INSERT INTO Sites (name,currency) VALUES ('Full Tilt Poker', 'USD')") + self.cursor.execute("INSERT INTO Sites (name,currency) VALUES ('PokerStars', 'USD')") + self.cursor.execute("INSERT INTO Sites (name,currency) VALUES ('Everleaf', 'USD')") + self.cursor.execute("""INSERT INTO TourneyTypes + (siteId,buyin,fee,knockout,rebuyOrAddon) VALUES + (1,0,0,0,?)""",(False,) ) #end def fillDefaultData def recreate_tables(self): @@ -617,7 +636,7 @@ class fpdb_db: ret = -1 else: ret = row[0] - elif self.backend == self.SQLITE: + elif self.backend == fpdb_db.SQLITE: # don't know how to do this in sqlite print "getLastInsertId(): not coded for sqlite yet" ret = -1 @@ -628,27 +647,70 @@ class fpdb_db: def storeHand(self, p): #stores into table hands: - self.cursor.execute ("""INSERT INTO Hands - (siteHandNo, gametypeId, handStart, seats, tableName, importTime, maxSeats - ,boardcard1, boardcard2, boardcard3, boardcard4, boardcard5 - ,playersVpi, playersAtStreet1, playersAtStreet2 - ,playersAtStreet3, playersAtStreet4, playersAtShowdown - ,street0Raises, street1Raises, street2Raises - ,street3Raises, street4Raises, street1Pot - ,street2Pot, street3Pot, street4Pot - ,showdownPot + self.cursor.execute ("""INSERT INTO Hands ( + tablename, + sitehandno, + gametypeid, + handstart, + importtime, + seats, + maxseats, + boardcard1, + boardcard2, + boardcard3, + boardcard4, + boardcard5, +-- texture, + playersVpi, + playersAtStreet1, + playersAtStreet2, + playersAtStreet3, + playersAtStreet4, + playersAtShowdown, + street0Raises, + street1Raises, + street2Raises, + street3Raises, + street4Raises, +-- street1Pot, +-- street2Pot, +-- street3Pot, +-- street4Pot, +-- showdownPot ) VALUES - (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""" - ,(p['siteHandNo'], gametype_id, p['handStart'], len(names), p['tableName'], datetime.datetime.today(), p['maxSeats'] - ,p['boardcard1'], ['boardcard2'], p['boardcard3'], ['boardcard4'], ['boardcard5'] - ,hudCache['playersVpi'], hudCache['playersAtStreet1'], hudCache['playersAtStreet2'] - ,hudCache['playersAtStreet3'], hudCache['playersAtStreet4'], hudCache['playersAtShowdown'] - ,hudCache['street0Raises'], hudCache['street1Raises'], hudCache['street2Raises'] - ,hudCache['street3Raises'], hudCache['street4Raises'], hudCache['street1Pot'] - ,hudCache['street2Pot'], hudCache['street3Pot'], hudCache['street4Pot'] - ,hudCache['showdownPot'] - ) - ) + (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, %s, %s)""", + ( + p['tablename'], + p['sitehandno'], + p['gametypeid'], + p['handStart'], + datetime.datetime.today(), + len(p['names']), + p['maxSeats'], + p['boardcard1'], + p['boardcard2'], + p['boardcard3'], + p['boardcard4'], + p['boardcard5'], + hudCache['playersVpi'], + hudCache['playersAtStreet1'], + hudCache['playersAtStreet2'], + hudCache['playersAtStreet3'], + hudCache['playersAtStreet4'], + hudCache['playersAtShowdown'], + hudCache['street0Raises'], + hudCache['street1Raises'], + hudCache['street2Raises'], + hudCache['street3Raises'], + hudCache['street4Raises'], + hudCache['street1Pot'], + hudCache['street2Pot'], + hudCache['street3Pot'], + hudCache['street4Pot'], + hudCache['showdownPot'] + ) + ) #return getLastInsertId(backend, conn, cursor) #end class fpdb_db diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 455c1e27..bcc5d02f 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -22,6 +22,7 @@ import os # todo: remove this once import_dir is in fpdb_import import sys from time import time, strftime +import logging import traceback import math import datetime From b252e350d619089fcbe5cd54e9f242689f236704 Mon Sep 17 00:00:00 2001 From: Matt Turnbull Date: Tue, 14 Jul 2009 23:34:47 +0100 Subject: [PATCH 05/11] fix my breakage.. --- pyfpdb/HandHistoryConverter.py | 79 ++++++++-------------------------- 1 file changed, 19 insertions(+), 60 deletions(-) diff --git a/pyfpdb/HandHistoryConverter.py b/pyfpdb/HandHistoryConverter.py index f5c4c2a5..cd3a1db0 100644 --- a/pyfpdb/HandHistoryConverter.py +++ b/pyfpdb/HandHistoryConverter.py @@ -28,91 +28,51 @@ import codecs from decimal import Decimal import operator from xml.dom.minidom import Node -# from pokereval import PokerEval import time import datetime -import gettext -#from pokerengine.pokercards import * -# provides letter2name{}, letter2names{}, visible_card(), not_visible_card(), is_visible(), card_value(), class PokerCards -# but it's probably not installed so here are the ones we may want: -letter2name = { - 'A': 'Ace', - 'K': 'King', - 'Q': 'Queen', - 'J': 'Jack', - 'T': 'Ten', - '9': 'Nine', - '8': 'Eight', - '7': 'Seven', - '6': 'Six', - '5': 'Five', - '4': 'Four', - '3': 'Trey', - '2': 'Deuce' - } - -letter2names = { - 'A': 'Aces', - 'K': 'Kings', - 'Q': 'Queens', - 'J': 'Jacks', - 'T': 'Tens', - '9': 'Nines', - '8': 'Eights', - '7': 'Sevens', - '6': 'Sixes', - '5': 'Fives', - '4': 'Fours', - '3': 'Treys', - '2': 'Deuces' - } - -import gettext -gettext.install('myapplication') +#import gettext +#gettext.install('fpdb') class HandHistoryConverter(): READ_CHUNK_SIZE = 10000 # bytes to read at a time from file (in tail mode) def __init__(self, in_path = '-', out_path = '-', sitename = None, follow=False): - logging.info("HandHistory init called") + logging.info("HandHistory init") # default filetype and codepage. Subclasses should set these properly. self.filetype = "text" self.codepage = "utf8" - + self.in_path = in_path self.out_path = out_path - if self.out_path == '-': - # write to stdout + + if in_path == '-': + self.in_fh = sys.stdin + + if out_path == '-': self.out_fh = sys.stdout else: - # TODO: out_path should be sanity checked before opening. Perhaps in fpdb_import? - # I'm not sure what we're looking for, although we don't want out_path==in_path!='-' - self.out_fh = open(self.out_path, 'w') # doomswitch is now on :| + # TODO: out_path should be sanity checked. + self.out_fh = open(self.out_path, 'w') + self.sitename = sitename self.follow = follow self.compiledPlayers = set() self.maxseats = 10 def __str__(self): - #TODO : I got rid of most of the hhdir stuff. - tmp = "HandHistoryConverter: '%s'\n" % (self.sitename) - #tmp = tmp + "\thhbase: '%s'\n" % (self.hhbase) - #tmp = tmp + "\thhdir: '%s'\n" % (self.hhdir) - tmp = tmp + "\tfiletype: '%s'\n" % (self.filetype) - tmp = tmp + "\tinfile: '%s'\n" % (self.in_path) - tmp = tmp + "\toutfile: '%s'\n" % (self.out_path) - #tmp = tmp + "\tgametype: '%s'\n" % (self.gametype[0]) - #tmp = tmp + "\tgamebase: '%s'\n" % (self.gametype[1]) - #tmp = tmp + "\tlimit: '%s'\n" % (self.gametype[2]) - #tmp = tmp + "\tsb/bb: '%s/%s'\n" % (self.gametype[3], self.gametype[4]) - return tmp + return """ +HandHistoryConverter: '%(sitename)s' + filetype: '%(filetype)s' + in_path: '%(in_path)s' + out_path: '%(out_path)s' + """ % { 'sitename':self.sitename, 'filetype':self.filetype, 'in_path':self.in_path, 'out_path':self.out_path } def start(self): """process a hand at a time from the input specified by in_path. If in follow mode, wait for more data to turn up. -Otherwise, finish at eof... +Otherwise, finish at eof. """ starttime = time.time() @@ -224,7 +184,6 @@ which it expects to find at self.re_TailSplitHands -- see for e.g. Everleaf.py. base = gametype['base'] limit = gametype['limitType'] l = [type] + [base] + [limit] - hand = None if l in self.readSupportedGames(): hand = None if gametype['base'] == 'hold': From 806d61ae2368315bc6f9d18e86a89e9066cb06cf Mon Sep 17 00:00:00 2001 From: Matt Turnbull Date: Wed, 15 Jul 2009 00:14:25 +0100 Subject: [PATCH 06/11] log line --- pyfpdb/fpdb_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyfpdb/fpdb_db.py b/pyfpdb/fpdb_db.py index c52a2e56..a0f6ca92 100644 --- a/pyfpdb/fpdb_db.py +++ b/pyfpdb/fpdb_db.py @@ -205,7 +205,7 @@ class fpdb_db: print msg raise fpdb_simple.FpdbError(msg) elif backend==fpdb_db.SQLITE: - logging.info("Connecting to SQLite:(%s)") + logging.info("Connecting to SQLite:%(database)s" % {'database':database}) import sqlite3 self.db = sqlite3.connect(database,detect_types=sqlite3.PARSE_DECLTYPES) sqlite3.register_converter("bool", lambda x: bool(int(x))) From 70f271ef78bd8e4f722540718f426b6cbdb3bc12 Mon Sep 17 00:00:00 2001 From: Matt Turnbull Date: Wed, 15 Jul 2009 00:15:04 +0100 Subject: [PATCH 07/11] location of test PS horse hh --- pyfpdb/PokerStarsToFpdb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyfpdb/PokerStarsToFpdb.py b/pyfpdb/PokerStarsToFpdb.py index 5350713c..4b66e899 100755 --- a/pyfpdb/PokerStarsToFpdb.py +++ b/pyfpdb/PokerStarsToFpdb.py @@ -1,4 +1,4 @@ - #!/usr/bin/env python +#!/usr/bin/env python # -*- coding: utf-8 -*- # # Copyright 2008, Carl Gherardi @@ -320,7 +320,7 @@ follow : whether to tail -f the input""" if __name__ == "__main__": parser = OptionParser() - parser.add_option("-i", "--input", dest="ipath", help="parse input hand history", default="regression-test-files/pokerstars/HH20090226 Natalie V - $0.10-$0.20 - HORSE.txt") + parser.add_option("-i", "--input", dest="ipath", help="parse input hand history", default="regression-test-files/stars/horse/HH20090226 Natalie V - $0.10-$0.20 - HORSE.txt") parser.add_option("-o", "--output", dest="opath", help="output translation to", default="-") parser.add_option("-f", "--follow", dest="follow", help="follow (tail -f) the input", action="store_true", default=False) parser.add_option("-q", "--quiet", From 35a5cffb77b07399b4dec80cfbec9451e2b2bc51 Mon Sep 17 00:00:00 2001 From: Matt Turnbull Date: Wed, 15 Jul 2009 00:26:53 +0100 Subject: [PATCH 08/11] Working on PS HORSE file now.. --- pyfpdb/HandHistoryConverter.py | 4 ++-- pyfpdb/PokerStarsToFpdb.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyfpdb/HandHistoryConverter.py b/pyfpdb/HandHistoryConverter.py index 590ee065..47ddecf5 100644 --- a/pyfpdb/HandHistoryConverter.py +++ b/pyfpdb/HandHistoryConverter.py @@ -31,8 +31,8 @@ from xml.dom.minidom import Node import time import datetime -#import gettext -#gettext.install('fpdb') +import gettext +gettext.install('fpdb') class HandHistoryConverter(): diff --git a/pyfpdb/PokerStarsToFpdb.py b/pyfpdb/PokerStarsToFpdb.py index 41b3c67f..b855586b 100755 --- a/pyfpdb/PokerStarsToFpdb.py +++ b/pyfpdb/PokerStarsToFpdb.py @@ -298,7 +298,7 @@ follow : whether to tail -f the input""" hand.addHoleCards(street, hand.hero, closed=newcards, shown=False, mucked=False, dealt=True) for street, text in hand.streets.iteritems(): - if street in ('PREFLOP', 'DEAL'): continue # already done these + if not text or street in ('PREFLOP', 'DEAL'): continue # already done these m = self.re_HeroCards.finditer(hand.streets[street]) for found in m: player = found.group('PNAME') From 521e17f8568b1bcd21b5a0a4cb08f879f5d94381 Mon Sep 17 00:00:00 2001 From: Worros Date: Wed, 15 Jul 2009 12:22:46 +0800 Subject: [PATCH 09/11] Add some boilerplate stuff for Boss Media/Win2day plugin --- pyfpdb/HUD_config.xml.example | 46 +++++++++++++++++++++++++++++++++- pyfpdb/HandHistoryConverter.py | 2 +- pyfpdb/Win2dayToFpdb.py | 0 pyfpdb/fpdb_db.py | 1 + 4 files changed, 47 insertions(+), 2 deletions(-) mode change 100644 => 100755 pyfpdb/Win2dayToFpdb.py diff --git a/pyfpdb/HUD_config.xml.example b/pyfpdb/HUD_config.xml.example index ef30fb9b..3cd592cc 100644 --- a/pyfpdb/HUD_config.xml.example +++ b/pyfpdb/HUD_config.xml.example @@ -159,6 +159,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -292,6 +336,7 @@ + @@ -300,4 +345,3 @@ - diff --git a/pyfpdb/HandHistoryConverter.py b/pyfpdb/HandHistoryConverter.py index fc6567f0..7e08554b 100644 --- a/pyfpdb/HandHistoryConverter.py +++ b/pyfpdb/HandHistoryConverter.py @@ -114,7 +114,7 @@ class HandHistoryConverter(): If in follow mode, wait for more data to turn up. Otherwise, finish at eof... -""" +""" starttime = time.time() if not self.sanityCheck(): print "Cowardly refusing to continue after failed sanity check" diff --git a/pyfpdb/Win2dayToFpdb.py b/pyfpdb/Win2dayToFpdb.py old mode 100644 new mode 100755 diff --git a/pyfpdb/fpdb_db.py b/pyfpdb/fpdb_db.py index f89b5d6d..0af29aa1 100644 --- a/pyfpdb/fpdb_db.py +++ b/pyfpdb/fpdb_db.py @@ -319,6 +319,7 @@ class fpdb_db: self.cursor.execute("INSERT INTO Sites VALUES (DEFAULT, 'Full Tilt Poker', 'USD');") self.cursor.execute("INSERT INTO Sites VALUES (DEFAULT, 'PokerStars', 'USD');") self.cursor.execute("INSERT INTO Sites VALUES (DEFAULT, 'Everleaf', 'USD');") + self.cursor.execute("INSERT INTO Sites VALUES (DEFAULT, 'Win2day', 'USD');") self.cursor.execute("INSERT INTO TourneyTypes VALUES (DEFAULT, 1, 0, 0, 0, False);") #end def fillDefaultData From ec9ce544ca6cef56c7b1cc46003a3a4d09e7c85c Mon Sep 17 00:00:00 2001 From: Worros Date: Wed, 15 Jul 2009 23:32:17 +0800 Subject: [PATCH 10/11] Fix if database hasn't been created yet. Still broken on my setup, but at least fpdb now starts --- pyfpdb/Database.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index 0f354054..d4e667df 100755 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -71,20 +71,23 @@ class Database: # a new session) cur = self.connection.cursor() - self.hand_1day_ago = 0 - cur.execute(self.sql.query['get_hand_1day_ago']) - row = cur.fetchone() - if row and row[0]: - self.hand_1day_ago = row[0] - #print "hand 1day ago =", self.hand_1day_ago + if self.fdb.wrongDbVersion == False: + self.hand_1day_ago = 0 + cur.execute(self.sql.query['get_hand_1day_ago']) + row = cur.fetchone() + if row and row[0]: + self.hand_1day_ago = row[0] + #print "hand 1day ago =", self.hand_1day_ago - d = timedelta(days=self.hud_days) - now = datetime.utcnow() - d - self.date_ndays_ago = "d%02d%02d%02d" % (now.year-2000, now.month, now.day) + d = timedelta(days=self.hud_days) + now = datetime.utcnow() - d + self.date_ndays_ago = "d%02d%02d%02d" % (now.year-2000, now.month, now.day) - self.hand_nhands_ago = 0 # todo - #cur.execute(self.sql.query['get_table_name'], (hand_id, )) - #row = cur.fetchone() + self.hand_nhands_ago = 0 # todo + #cur.execute(self.sql.query['get_table_name'], (hand_id, )) + #row = cur.fetchone() + else: + print "Bailing on DB query, not sure it exists yet" self.saveActions = False if self.import_options['saveActions'] == False else True def do_connect(self, c): From b92feee021229886e5baabaa9a3d8c7d3732d9c6 Mon Sep 17 00:00:00 2001 From: Worros Date: Wed, 15 Jul 2009 23:58:34 +0800 Subject: [PATCH 11/11] Revert Matts change for sqlite until fixed. Previous syntax failed to insert the required line into TourneyTypes --- pyfpdb/fpdb_db.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyfpdb/fpdb_db.py b/pyfpdb/fpdb_db.py index 30101d0c..2b75ea12 100644 --- a/pyfpdb/fpdb_db.py +++ b/pyfpdb/fpdb_db.py @@ -336,9 +336,10 @@ class fpdb_db: self.cursor.execute("INSERT INTO Sites (name,currency) VALUES ('PokerStars', 'USD')") self.cursor.execute("INSERT INTO Sites (name,currency) VALUES ('Everleaf', 'USD')") self.cursor.execute("INSERT INTO Sites (name,currency) VALUES ('Win2day', 'USD')") - self.cursor.execute("""INSERT INTO TourneyTypes - (siteId,buyin,fee,knockout,rebuyOrAddon) VALUES - (1,0,0,0,?)""",(False,) ) + self.cursor.execute("INSERT INTO TourneyTypes VALUES (DEFAULT, 1, 0, 0, 0, False);") + #self.cursor.execute("""INSERT INTO TourneyTypes + # (siteId,buyin,fee,knockout,rebuyOrAddon) VALUES + # (1,0,0,0,?)""",(False,) ) #end def fillDefaultData def recreate_tables(self):