From 1df418c8e4cf761007c708f4092fa5dbea5c2d16 Mon Sep 17 00:00:00 2001 From: sqlcoder Date: Wed, 27 Jan 2010 23:48:39 +0000 Subject: [PATCH] create fpdb dir if required, improve sqlite performance :-) --- pyfpdb/Configuration.py | 22 ++++++++--- pyfpdb/Database.py | 85 +++++++++++++++++++++++++---------------- pyfpdb/HUD_main.py | 5 ++- 3 files changed, 72 insertions(+), 40 deletions(-) diff --git a/pyfpdb/Configuration.py b/pyfpdb/Configuration.py index 11d38bf4..09f7ca2e 100755 --- a/pyfpdb/Configuration.py +++ b/pyfpdb/Configuration.py @@ -60,12 +60,14 @@ def get_exec_path(): def get_config(file_name, fallback = True): """Looks in cwd and in self.default_config_path for a config file.""" - config_path = os.path.join(get_exec_path(), file_name) + exec_dir = get_exec_path() + config_path = os.path.join(exec_dir, file_name) # print "config_path=", config_path if os.path.exists(config_path): # there is a file in the cwd return config_path # so we use it else: # no file in the cwd, look where it should be in the first place - config_path = os.path.join(get_default_config_path(), file_name) + default_dir = get_default_config_path() + config_path = os.path.join(default_dir, file_name) # print "config path 2=", config_path if os.path.exists(config_path): return config_path @@ -77,14 +79,24 @@ def get_config(file_name, fallback = True): # OK, fall back to the .example file, should be in the start dir if os.path.exists(file_name + ".example"): try: + print "" + if not os.path.isdir(default_dir): + msg = "Creating directory: '%s'" % (default_dir) + print msg + logging.info(msg) + os.mkdir(default_dir) shutil.copyfile(file_name + ".example", config_path) - print "No %s found, using %s.example.\n" % (file_name, file_name) - print "Config file has been created at %s.\nYou will probably have to edit it." % config_path - sys.stderr.write("No %s found, copying %s.example.\n" % (file_name, file_name) ) + msg = "No %s found in %s or %s\n" % (file_name, exec_dir, default_dir) \ + + "Config file has been created at %s.\n" % config_path \ + + "Edit your screen_name and hand history path in the supported_sites "\ + + "section of the \nPreferences window (Main menu) before trying to import hands." + print msg + logging.info(msg) file_name = config_path except: print "Error copying .example file, cannot fall back. Exiting.\n" sys.stderr.write("Error copying .example file, cannot fall back. Exiting.\n") + sys.stderr.write( str(sys.exc_info()) ) sys.exit() else: print "No %s found, cannot fall back. Exiting.\n" % file_name diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index f17f1d33..54690979 100644 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -39,40 +39,38 @@ import string import re import Queue import codecs -import logging import math # pyGTK modules +# FreePokerTools modules +import SQL +import Card +import Tourney +import Charset +from Exceptions import * +import Configuration +log = Configuration.get_logger("logging.conf","db") + + # Other library modules try: import sqlalchemy.pool as pool use_pool = True except ImportError: - logging.info("Not using sqlalchemy connection pool.") + log.info("Not using sqlalchemy connection pool.") use_pool = False try: from numpy import var use_numpy = True except ImportError: - logging.info("Not using numpy to define variance in sqlite.") + log.info("Not using numpy to define variance in sqlite.") use_numpy = False -# FreePokerTools modules -import Configuration -import SQL -import Card -import Tourney -import Charset -from Exceptions import * - -log = Configuration.get_logger("logging.conf") - - DB_VERSION = 119 @@ -374,20 +372,20 @@ class Database: print msg raise FpdbError(msg) elif backend == Database.SQLITE: - logging.info("Connecting to SQLite: %(database)s" % {'database':database}) + log.info("Connecting to SQLite: %(database)s" % {'database':database}) import sqlite3 if use_pool: sqlite3 = pool.manage(sqlite3, pool_size=1) else: - logging.warning("SQLite won't work well without 'sqlalchemy' installed.") + log.warning("SQLite won't work well without 'sqlalchemy' installed.") if database != ":memory:": if not os.path.isdir(self.config.dir_databases): print "Creating directory: '%s'" % (self.config.dir_databases) - logging.info("Creating directory: '%s'" % (self.config.dir_databases)) + log.info("Creating directory: '%s'" % (self.config.dir_databases)) os.mkdir(self.config.dir_databases) database = os.path.join(self.config.dir_databases, database) - logging.info(" sqlite db: " + database) + log.info(" sqlite db: " + database) self.connection = sqlite3.connect(database, detect_types=sqlite3.PARSE_DECLTYPES ) sqlite3.register_converter("bool", lambda x: bool(int(x))) sqlite3.register_adapter(bool, lambda x: "1" if x else "0") @@ -397,7 +395,10 @@ class Database: if use_numpy: self.connection.create_aggregate("variance", 1, VARIANCE) else: - logging.warning("Some database functions will not work without NumPy support") + log.warning("Some database functions will not work without NumPy support") + self.cursor = self.connection.cursor() + self.cursor.execute('PRAGMA temp_store=2') # use memory for temp tables/indexes + self.cursor.execute('PRAGMA synchronous=0') # don't wait for file writes to finish else: raise FpdbError("unrecognised database backend:"+backend) @@ -412,7 +413,7 @@ class Database: self.cursor.execute("SELECT * FROM Settings") settings = self.cursor.fetchone() if settings[0] != DB_VERSION: - logging.error("outdated or too new database version (%s) - please recreate tables" + log.error("outdated or too new database version (%s) - please recreate tables" % (settings[0])) self.wrongDbVersion = True except:# _mysql_exceptions.ProgrammingError: @@ -422,11 +423,6 @@ class Database: log.info("failed to read settings table - recreating tables") self.recreate_tables() self.check_version(database=database, create=False) - if not self.wrongDbVersion: - msg = "Edit your screen_name and hand history path in the supported_sites "\ - +"section of the \nPreferences window (Main menu) before trying to import hands" - print "\n%s" % msg - log.warning(msg) else: print "Failed to read settings table - please recreate tables" log.info("failed to read settings table - please recreate tables") @@ -436,7 +432,27 @@ class Database: #end def connect def commit(self): - self.connection.commit() + if self.backend != self.SQLITE: + self.connection.commit() + else: + # sqlite commits can fail because of shared locks on the database (SQLITE_BUSY) + # re-try commit if it fails in case this happened + maxtimes = 5 + pause = 1 + ok = False + for i in xrange(maxtimes): + try: + ret = self.connection.commit() + log.debug("commit finished ok, i = "+str(i)) + ok = True + except: + log.debug("commit "+str(i)+" failed: info=" + str(sys.exc_info()) + + " value=" + str(sys.exc_value)) + sleep(pause) + if ok: break + if not ok: + log.debug("commit failed") + raise FpdbError('sqlite commit failed') def rollback(self): self.connection.rollback() @@ -1750,7 +1766,10 @@ class Database: def insertPlayer(self, name, site_id): result = None - _name = Charset.to_db_utf8(name) + if self.backend == self.SQLITE: + _name = name + else: + _name = Charset.to_db_utf8(name) c = self.get_cursor() q = "SELECT name, id FROM Players WHERE siteid=%s and name=%s" q = q.replace('%s', self.sql.query['placeholder']) @@ -1906,7 +1925,7 @@ class Database: # end def send_finish_msg(): def tRecogniseTourneyType(self, tourney): - logging.debug("Database.tRecogniseTourneyType") + log.debug("Database.tRecogniseTourneyType") typeId = 1 # Check if Tourney exists, and if so retrieve TTypeId : in that case, check values of the ttype cursor = self.get_cursor() @@ -1922,10 +1941,10 @@ class Database: try: len(result) typeId = result[0] - logging.debug("Tourney found in db with Tourney_Type_ID = %d" % typeId) + log.debug("Tourney found in db with Tourney_Type_ID = %d" % typeId) for ev in expectedValues : if ( getattr( tourney, expectedValues.get(ev) ) <> result[ev] ): - logging.debug("TypeId mismatch : wrong %s : Tourney=%s / db=%s" % (expectedValues.get(ev), getattr( tourney, expectedValues.get(ev)), result[ev]) ) + log.debug("TypeId mismatch : wrong %s : Tourney=%s / db=%s" % (expectedValues.get(ev), getattr( tourney, expectedValues.get(ev)), result[ev]) ) typeIdMatch = False #break except: @@ -1935,7 +1954,7 @@ class Database: if typeIdMatch == False : # Check for an existing TTypeId that matches tourney info (buyin/fee, knockout, rebuy, speed, matrix, shootout) # if not found create it - logging.debug("Searching for a TourneyTypeId matching TourneyType data") + log.debug("Searching for a TourneyTypeId matching TourneyType data") cursor.execute (self.sql.query['getTourneyTypeId'].replace('%s', self.sql.query['placeholder']), (tourney.siteId, tourney.buyin, tourney.fee, tourney.isKO, tourney.isRebuy, tourney.speed, tourney.isHU, tourney.isShootout, tourney.isMatrix) @@ -1945,9 +1964,9 @@ class Database: try: len(result) typeId = result[0] - logging.debug("Existing Tourney Type Id found : %d" % typeId) + log.debug("Existing Tourney Type Id found : %d" % typeId) except TypeError: #this means we need to create a new entry - logging.debug("Tourney Type Id not found : create one") + log.debug("Tourney Type Id not found : create one") cursor.execute (self.sql.query['insertTourneyTypes'].replace('%s', self.sql.query['placeholder']), (tourney.siteId, tourney.buyin, tourney.fee, tourney.isKO, tourney.isRebuy, tourney.speed, tourney.isHU, tourney.isShootout, tourney.isMatrix) diff --git a/pyfpdb/HUD_main.py b/pyfpdb/HUD_main.py index 62025018..3fcd0fca 100755 --- a/pyfpdb/HUD_main.py +++ b/pyfpdb/HUD_main.py @@ -204,6 +204,7 @@ class HUD_main(object): # get basic info about the new hand from the db # if there is a db error, complain, skip hand, and proceed + log.info("HUD_main.read_stdin: hand processing starting ...") try: (table_name, max, poker_game, type, site_id, site_name, num_seats, tour_number, tab_number) = \ self.db_connection.get_table_info(new_hand_id) @@ -230,6 +231,7 @@ class HUD_main(object): try: self.db_connection.init_hud_stat_vars( self.hud_dict[temp_key].hud_params['hud_days'] , self.hud_dict[temp_key].hud_params['h_hud_days']) + t4 = time.time() stat_dict = self.db_connection.get_stats_from_hand(new_hand_id, type, self.hud_dict[temp_key].hud_params, self.hero_ids[site_id]) self.hud_dict[temp_key].stat_dict = stat_dict except KeyError: # HUD instance has been killed off, key is stale @@ -238,6 +240,7 @@ class HUD_main(object): # Unlocks table, copied from end of function self.db_connection.connection.rollback() return + t5 = time.time() cards = self.db_connection.get_cards(new_hand_id) comm_cards = self.db_connection.get_common_cards(new_hand_id) if comm_cards != {}: # stud! @@ -250,10 +253,8 @@ class HUD_main(object): else: # get stats using default params--also get cards self.db_connection.init_hud_stat_vars( self.hud_params['hud_days'], self.hud_params['h_hud_days'] ) - t4 = time.time() stat_dict = self.db_connection.get_stats_from_hand(new_hand_id, type, self.hud_params ,self.hero_ids[site_id], num_seats) - t5 = time.time() cards = self.db_connection.get_cards(new_hand_id) comm_cards = self.db_connection.get_common_cards(new_hand_id) if comm_cards != {}: # stud!