diff --git a/Makefile b/Makefile index 3d8ec313..078a3bd0 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Variable definitions -VERSION = 0.12 +VERSION = 0.20 DATE = $(shell date +%Y%m%d) all: diff --git a/docs/readme.txt b/docs/readme.txt index 03e071d3..e3dbac23 100644 --- a/docs/readme.txt +++ b/docs/readme.txt @@ -1,5 +1,5 @@ README.txt -updated 26 March 2009, REB +updated 22 February 2010, REB fpdb - Free Poker Database @@ -29,7 +29,7 @@ fpdb supports: Omaha (incl Hi/low) 7 Card Stud (incl Hi/low) Razz - Draw support is under development + Triple Draw and Badugi Mixed Games -- HUD under development Operating Systems: @@ -38,23 +38,38 @@ fpdb supports: Mac OS/X -- no support for HUD Databases: + SQLite configured by default MySQL PostgreSQL - SQLite under development Downloads: Releases: http://sourceforge.net/project/showfiles.php?group_id=226872 Development code via git: http://www.assembla.com/spaces/free_poker_tools/trac_git_tool Developers: - At least 7 people have contributed code or patches. Others are welcome. + At least 10 people have contributed code or patches. Others are welcome. +Source Code: + If you received fpdb as the Windows compressed exe, then you did not +receive souce code for fpdb or the included libraries. If you wish, you can +obtain the source code here: + + fpdb: see Downloads, above. + python: http://python.org/ + gtk: http://www.gtk.org/download.html + pygtk: http://www.pygtk.org/downloads.html + psycopg2: http://initd.org/pub/software/psycopg/ + mysqldb: http://sourceforge.net/projects/mysql-python/files/ + sqlalchemy: http://www.sqlalchemy.org/download.html + numpy: http://www.scipy.org/Download + matplotlib: http://sourceforge.net/projects/matplotlib/files/ + License ======= Trademarks of third parties have been used under Fair Use or similar laws. Copyright 2008 Steffen Jobbagy-Felso -Copyright 2009 Ray E. Barker +Copyright 2009,2010 Ray E. Barker Permission is granted to copy, distribute and/or modify this document under the terms of the GNU Free Documentation License, Version 1.2 as published by the Free Software Foundation; with diff --git a/pyfpdb/Configuration.py b/pyfpdb/Configuration.py index f7694e90..d0f81a1b 100755 --- a/pyfpdb/Configuration.py +++ b/pyfpdb/Configuration.py @@ -414,6 +414,7 @@ class Import: self.hhArchiveBase = node.getAttribute("hhArchiveBase") self.saveActions = string_to_bool(node.getAttribute("saveActions"), default=True) self.fastStoreHudCache = string_to_bool(node.getAttribute("fastStoreHudCache"), default=False) + self.saveStarsHH = string_to_bool(node.getAttribute("saveStarsHH"), default=False) def __str__(self): return " interval = %s\n callFpdbHud = %s\n hhArchiveBase = %s\n saveActions = %s\n fastStoreHudCache = %s\n" \ @@ -469,7 +470,8 @@ class Config: self.file = file self.dir_self = get_exec_path() - self.dir_config = os.path.dirname(self.file) +# self.dir_config = os.path.dirname(self.file) + self.dir_config = get_default_config_path() self.dir_log = os.path.join(self.dir_config, 'log') self.dir_database = os.path.join(self.dir_config, 'database') self.log_file = os.path.join(self.dir_log, 'fpdb-log.txt') @@ -481,12 +483,19 @@ class Config: print "\nReading configuration file %s\n" % file try: doc = xml.dom.minidom.parse(file) + self.file_error = None except: log.error("Error parsing %s. See error log file." % (file)) traceback.print_exc(file=sys.stderr) - print "press enter to continue" - sys.stdin.readline() - sys.exit() + self.file_error = sys.exc_info()[1] + # we could add a parameter to decide whether to return or read a line and exit? + return + #print "press enter to continue" + #sys.stdin.readline() + #sys.exit() +#ExpatError: not well-formed (invalid token): line 511, column 4 +#sys.exc_info = (, ExpatError('not well-formed (invalid token): line 511, +# column 4',), ) self.doc = doc self.supported_sites = {} @@ -688,18 +697,8 @@ class Config: try: db['db-server'] = self.supported_databases[name].db_server except: pass - if self.supported_databases[name].db_server== DATABASE_TYPE_MYSQL: - db['db-backend'] = 2 - elif self.supported_databases[name].db_server== DATABASE_TYPE_POSTGRESQL: - db['db-backend'] = 3 - elif self.supported_databases[name].db_server== DATABASE_TYPE_SQLITE: - db['db-backend'] = 4 - # sqlcoder: this assignment fixes unicode problems for me with sqlite (windows, cp1252) - # feel free to remove or improve this if you understand the problems - # better than me (not hard!) - Charset.not_needed1, Charset.not_needed2, Charset.not_needed3 = True, True, True - else: - raise ValueError('Unsupported database backend: %s' % self.supported_databases[name].db_server) + db['db-backend'] = self.get_backend(self.supported_databases[name].db_server) + return db def set_db_parameters(self, db_name = 'fpdb', db_ip = None, db_user = None, @@ -718,6 +717,23 @@ class Config: if db_server is not None: self.supported_databases[db_name].dp_server = db_server if db_type is not None: self.supported_databases[db_name].dp_type = db_type return + + def get_backend(self, name): + """Returns the number of the currently used backend""" + if name == DATABASE_TYPE_MYSQL: + ret = 2 + elif name == DATABASE_TYPE_POSTGRESQL: + ret = 3 + elif name == DATABASE_TYPE_SQLITE: + ret = 4 + # sqlcoder: this assignment fixes unicode problems for me with sqlite (windows, cp1252) + # feel free to remove or improve this if you understand the problems + # better than me (not hard!) + Charset.not_needed1, Charset.not_needed2, Charset.not_needed3 = True, True, True + else: + raise ValueError('Unsupported database backend: %s' % self.supported_databases[name].db_server) + + return ret def getDefaultSite(self): "Returns first enabled site or None" @@ -808,8 +824,12 @@ class Config: try: imp['saveActions'] = self.imp.saveActions except: imp['saveActions'] = True + try: imp['saveStarsHH'] = self.imp.saveStarsHH + except: imp['saveStarsHH'] = False + try: imp['fastStoreHudCache'] = self.imp.fastStoreHudCache except: imp['fastStoreHudCache'] = True + return imp def get_default_paths(self, site = None): diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index 0303aad2..8939d4b3 100644 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -142,14 +142,18 @@ class Database: , {'tab':'TourneyTypes', 'col':'siteId', 'drop':0} ] , [ # indexes for sqlite (list index 4) - # {'tab':'Players', 'col':'name', 'drop':0} unique indexes not dropped - # {'tab':'Hands', 'col':'siteHandNo', 'drop':0} unique indexes not dropped {'tab':'Hands', 'col':'gametypeId', 'drop':0} , {'tab':'HandsPlayers', 'col':'handId', 'drop':0} , {'tab':'HandsPlayers', 'col':'playerId', 'drop':0} , {'tab':'HandsPlayers', 'col':'tourneyTypeId', 'drop':0} , {'tab':'HandsPlayers', 'col':'tourneysPlayersId', 'drop':0} - #, {'tab':'Tourneys', 'col':'siteTourneyNo', 'drop':0} unique indexes not dropped + , {'tab':'HudCache', 'col':'gametypeId', 'drop':1} + , {'tab':'HudCache', 'col':'playerId', 'drop':0} + , {'tab':'HudCache', 'col':'tourneyTypeId', 'drop':0} + , {'tab':'Players', 'col':'siteId', 'drop':1} + , {'tab':'Tourneys', 'col':'tourneyTypeId', 'drop':1} + , {'tab':'TourneysPlayers', 'col':'playerId', 'drop':0} + , {'tab':'TourneyTypes', 'col':'siteId', 'drop':0} ] ] @@ -226,7 +230,7 @@ class Database: # create index indexname on tablename (col); - def __init__(self, c, sql = None): + def __init__(self, c, sql = None, autoconnect = True): #log = Configuration.get_logger("logging.conf", "db", log_dir=c.dir_log) log.debug("Creating Database instance, sql = %s" % sql) self.config = c @@ -247,41 +251,42 @@ class Database: else: self.sql = sql - # connect to db - self.do_connect(c) - - if self.backend == self.PGSQL: - from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT, ISOLATION_LEVEL_READ_COMMITTED, ISOLATION_LEVEL_SERIALIZABLE - #ISOLATION_LEVEL_AUTOCOMMIT = 0 - #ISOLATION_LEVEL_READ_COMMITTED = 1 - #ISOLATION_LEVEL_SERIALIZABLE = 2 + if autoconnect: + # connect to db + self.do_connect(c) + + if self.backend == self.PGSQL: + from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT, ISOLATION_LEVEL_READ_COMMITTED, ISOLATION_LEVEL_SERIALIZABLE + #ISOLATION_LEVEL_AUTOCOMMIT = 0 + #ISOLATION_LEVEL_READ_COMMITTED = 1 + #ISOLATION_LEVEL_SERIALIZABLE = 2 - if self.backend == self.SQLITE and self.database == ':memory:' and self.wrongDbVersion: - log.info("sqlite/:memory: - creating") - self.recreate_tables() - self.wrongDbVersion = False + if self.backend == self.SQLITE and self.database == ':memory:' and self.wrongDbVersion: + log.info("sqlite/:memory: - creating") + self.recreate_tables() + self.wrongDbVersion = False - self.pcache = None # PlayerId cache - self.cachemiss = 0 # Delete me later - using to count player cache misses - self.cachehit = 0 # Delete me later - using to count player cache hits + self.pcache = None # PlayerId cache + self.cachemiss = 0 # Delete me later - using to count player cache misses + self.cachehit = 0 # Delete me later - using to count player cache hits - # config while trying out new hudcache mechanism - self.use_date_in_hudcache = True + # config while trying out new hudcache mechanism + self.use_date_in_hudcache = True - #self.hud_hero_style = 'T' # Duplicate set of vars just for hero - not used yet. - #self.hud_hero_hands = 2000 # Idea is that you might want all-time stats for others - #self.hud_hero_days = 30 # but last T days or last H hands for yourself + #self.hud_hero_style = 'T' # Duplicate set of vars just for hero - not used yet. + #self.hud_hero_hands = 2000 # Idea is that you might want all-time stats for others + #self.hud_hero_days = 30 # but last T days or last H hands for yourself - # vars for hand ids or dates fetched according to above config: - self.hand_1day_ago = 0 # max hand id more than 24 hrs earlier than now - self.date_ndays_ago = 'd000000' # date N days ago ('d' + YYMMDD) - self.h_date_ndays_ago = 'd000000' # date N days ago ('d' + YYMMDD) for hero - self.date_nhands_ago = {} # dates N hands ago per player - not used yet + # vars for hand ids or dates fetched according to above config: + self.hand_1day_ago = 0 # max hand id more than 24 hrs earlier than now + self.date_ndays_ago = 'd000000' # date N days ago ('d' + YYMMDD) + self.h_date_ndays_ago = 'd000000' # date N days ago ('d' + YYMMDD) for hero + self.date_nhands_ago = {} # dates N hands ago per player - not used yet - self.saveActions = False if self.import_options['saveActions'] == False else True + self.saveActions = False if self.import_options['saveActions'] == False else True - self.connection.rollback() # make sure any locks taken so far are released + self.connection.rollback() # make sure any locks taken so far are released #end def __init__ # could be used by hud to change hud style @@ -313,7 +318,7 @@ class Database: self.__connected = True def connect(self, backend=None, host=None, database=None, - user=None, password=None): + user=None, password=None, create=False): """Connects a database with the given parameters""" if backend is None: raise FpdbError('Database backend not defined') @@ -377,6 +382,7 @@ class Database: print msg raise FpdbError(msg) elif backend == Database.SQLITE: + create = True import sqlite3 if use_pool: sqlite3 = pool.manage(sqlite3, pool_size=1) @@ -384,32 +390,35 @@ class Database: # log.warning("SQLite won't work well without 'sqlalchemy' installed.") if database != ":memory:": - if not os.path.isdir(self.config.dir_database): + if not os.path.isdir(self.config.dir_database) and create: print "Creating directory: '%s'" % (self.config.dir_database) log.info("Creating directory: '%s'" % (self.config.dir_database)) os.mkdir(self.config.dir_database) database = os.path.join(self.config.dir_database, database) self.db_path = database log.info("Connecting to SQLite: %(database)s" % {'database':self.db_path}) - self.connection = sqlite3.connect(self.db_path, detect_types=sqlite3.PARSE_DECLTYPES ) - sqlite3.register_converter("bool", lambda x: bool(int(x))) - sqlite3.register_adapter(bool, lambda x: "1" if x else "0") - self.connection.create_function("floor", 1, math.floor) - tmp = sqlitemath() - self.connection.create_function("mod", 2, tmp.mod) - if use_numpy: - self.connection.create_aggregate("variance", 1, VARIANCE) + if os.path.exists(database) or create: + self.connection = sqlite3.connect(self.db_path, detect_types=sqlite3.PARSE_DECLTYPES ) + sqlite3.register_converter("bool", lambda x: bool(int(x))) + sqlite3.register_adapter(bool, lambda x: "1" if x else "0") + self.connection.create_function("floor", 1, math.floor) + tmp = sqlitemath() + self.connection.create_function("mod", 2, tmp.mod) + if use_numpy: + self.connection.create_aggregate("variance", 1, VARIANCE) + else: + log.warning("Some database functions will not work without NumPy support") + self.cursor = self.connection.cursor() + self.cursor.execute('PRAGMA temp_store=2') # use memory for temp tables/indexes + self.cursor.execute('PRAGMA synchronous=0') # don't wait for file writes to finish else: - log.warning("Some database functions will not work without NumPy support") - self.cursor = self.connection.cursor() - self.cursor.execute('PRAGMA temp_store=2') # use memory for temp tables/indexes - self.cursor.execute('PRAGMA synchronous=0') # don't wait for file writes to finish + raise FpdbError("sqlite database "+database+" does not exist") else: - raise FpdbError("unrecognised database backend:"+backend) + raise FpdbError("unrecognised database backend:"+str(backend)) self.cursor = self.connection.cursor() self.cursor.execute(self.sql.query['set tx level']) - self.check_version(database=database, create=True) + self.check_version(database=database, create=create) def check_version(self, database, create): @@ -721,6 +730,8 @@ class Database: # now get the stats c.execute(self.sql.query[query], subs) + #for row in c.fetchall(): # needs "explain query plan" in sql statement + # print "query plan: ", row colnames = [desc[0] for desc in c.description] for row in c.fetchall(): playerid = row[0] @@ -907,7 +918,6 @@ class Database: print "warning: constraint %s_%s_fkey not dropped: %s, continuing ..." \ % (fk['fktab'],fk['fkcol'], str(sys.exc_value).rstrip('\n')) else: - print "Only MySQL and Postgres supported so far" return -1 for idx in self.indexes[self.backend]: @@ -942,7 +952,6 @@ class Database: print "warning: index %s_%s_idx not dropped %s, continuing ..." \ % (idx['tab'],idx['col'], str(sys.exc_value).rstrip('\n')) else: - print "Error: Only MySQL and Postgres supported so far" return -1 if self.backend == self.PGSQL: @@ -997,7 +1006,6 @@ class Database: except: print " create fk failed: " + str(sys.exc_info()) else: - print "Only MySQL and Postgres supported so far" return -1 for idx in self.indexes[self.backend]: @@ -1019,7 +1027,6 @@ class Database: except: print " create index failed: " + str(sys.exc_info()) else: - print "Only MySQL and Postgres supported so far" return -1 if self.backend == self.PGSQL: @@ -2096,6 +2103,7 @@ class HandToWrite: if __name__=="__main__": c = Configuration.Config() + sql = SQL.Sql(db_server = 'sqlite') db_connection = Database(c) # mysql fpdb holdem # db_connection = Database(c, 'fpdb-p', 'test') # mysql fpdb holdem @@ -2113,12 +2121,25 @@ if __name__=="__main__": if hero: print "nutOmatic is id_player = %d" % hero + # example of displaying query plan in sqlite: + if db_connection.backend == 4: + print + c = db_connection.get_cursor() + c.execute('explain query plan '+sql.query['get_table_name'], (h, )) + for row in c.fetchall(): + print "query plan: ", row + print + + t0 = time() stat_dict = db_connection.get_stats_from_hand(h, "ring") + t1 = time() for p in stat_dict.keys(): print p, " ", stat_dict[p] print "cards =", db_connection.get_cards(u'1') db_connection.close_connection + + print "get_stats took: %4.3f seconds" % (t1-t0) print "press enter to continue" sys.stdin.readline() diff --git a/pyfpdb/DerivedStats.py b/pyfpdb/DerivedStats.py index d7ded83b..327b8de2 100644 --- a/pyfpdb/DerivedStats.py +++ b/pyfpdb/DerivedStats.py @@ -162,7 +162,7 @@ class DerivedStats(): self.handsplayers[player]['wonAtSD'] = 1.0 for player in hand.pot.committed: - self.handsplayers[player]['totalProfit'] = int(self.handsplayers[player]['winnings'] - (100*hand.pot.committed[player])) + self.handsplayers[player]['totalProfit'] = int(self.handsplayers[player]['winnings'] - (100*hand.pot.committed[player])- (100*hand.pot.common[player])) self.calcCBets(hand) @@ -227,7 +227,7 @@ class DerivedStats(): #print "bb =", bb, "sb =", sb, "players =", players for i,player in enumerate(reversed(players)): - self.handsplayers[player]['position'] = str(i) + self.handsplayers[player]['position'] = i def assembleHudCache(self, hand): # No real work to be done - HandsPlayers data already contains the correct info @@ -305,12 +305,13 @@ class DerivedStats(): """Fills stealAttempt(Chance|ed, fold(Bb|Sb)ToSteal(Chance|) Steal attempt - open raise on positions 1 0 S - i.e. MP3, CO, BU, SB + (note: I don't think PT2 counts SB steals in HU hands, maybe we shouldn't?) Fold to steal - folding blind after steal attemp wo any other callers or raisers """ steal_attempt = False - steal_positions = ('1', '0', 'S') + steal_positions = (1, 0, 'S') if hand.gametype['base'] == 'stud': - steal_positions = ('2', '1', '0') + steal_positions = (2, 1, 0) for action in hand.actions[hand.actionStreets[1]]: pname, act = action[0], action[1] posn = self.handsplayers[pname]['position'] @@ -344,9 +345,9 @@ class DerivedStats(): for action in hand.actions[hand.actionStreets[1]]: # FIXME: fill other(3|4)BStreet0 - i have no idea what does it mean pname, aggr = action[0], action[1] in ('raises', 'bets') - self.handsplayers[pname]['street0_3BChance'] = bet_level == 2 + self.handsplayers[pname]['street0_3BChance'] = self.handsplayers[pname]['street0_3BChance'] or bet_level == 2 self.handsplayers[pname]['street0_4BChance'] = bet_level == 3 - self.handsplayers[pname]['street0_3BDone'] = aggr and (self.handsplayers[pname]['street0_3BChance']) + self.handsplayers[pname]['street0_3BDone'] = self.handsplayers[pname]['street0_3BDone'] or (aggr and self.handsplayers[pname]['street0_3BChance']) self.handsplayers[pname]['street0_4BDone'] = aggr and (self.handsplayers[pname]['street0_4BChance']) if aggr: bet_level += 1 diff --git a/pyfpdb/Filters.py b/pyfpdb/Filters.py index 27e5a49d..c8e8e219 100644 --- a/pyfpdb/Filters.py +++ b/pyfpdb/Filters.py @@ -211,6 +211,9 @@ class Filters(threading.Thread): self.Button2.connect("clicked", self.callback['button2'], "clicked") self.Button2.set_sensitive(True) + # make sure any locks on db are released: + self.db.rollback() + def get_vbox(self): """returns the vbox of this thread""" return self.mainVBox diff --git a/pyfpdb/FulltiltToFpdb.py b/pyfpdb/FulltiltToFpdb.py index af55fd41..10ecb8dd 100755 --- a/pyfpdb/FulltiltToFpdb.py +++ b/pyfpdb/FulltiltToFpdb.py @@ -65,8 +65,8 @@ class Fulltilt(HandHistoryConverter): (\s\((?PTurbo)\))?)|(?P.+)) ''', re.VERBOSE) re_Button = re.compile('^The button is in seat #(?P