evilny0/fpdb
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

create fpdb dir if required, improve sqlite performance :-)

Browse Source
This commit is contained in:
sqlcoder 2010-01-27 23:48:39 +00:00
parent e82d276cf9
commit 1df418c8e4
3 changed files with 72 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
pyfpdb/Configuration.py
View File

@ -60,12 +60,14 @@ def get_exec_path():
def get_config(file_name, fallback = True): def get_config(file_name, fallback = True):
"""Looks in cwd and in self.default_config_path for a config file.""" """Looks in cwd and in self.default_config_path for a config file."""
config_path = os.path.join(get_exec_path(), file_name) exec_dir = get_exec_path()
config_path = os.path.join(exec_dir, file_name)
# print "config_path=", config_path # print "config_path=", config_path
if os.path.exists(config_path): # there is a file in the cwd if os.path.exists(config_path): # there is a file in the cwd
return config_path # so we use it return config_path # so we use it
else: # no file in the cwd, look where it should be in the first place else: # no file in the cwd, look where it should be in the first place
config_path = os.path.join(get_default_config_path(), file_name) default_dir = get_default_config_path()
config_path = os.path.join(default_dir, file_name)
# print "config path 2=", config_path # print "config path 2=", config_path
if os.path.exists(config_path): if os.path.exists(config_path):
return config_path return config_path
@ -77,14 +79,24 @@ def get_config(file_name, fallback = True):
# OK, fall back to the .example file, should be in the start dir # OK, fall back to the .example file, should be in the start dir
if os.path.exists(file_name + ".example"): if os.path.exists(file_name + ".example"):
try: try:
print ""
if not os.path.isdir(default_dir):
msg = "Creating directory: '%s'" % (default_dir)
print msg
logging.info(msg)
os.mkdir(default_dir)
shutil.copyfile(file_name + ".example", config_path) shutil.copyfile(file_name + ".example", config_path)
print "No %s found, using %s.example.\n" % (file_name, file_name) msg = "No %s found in %s or %s\n" % (file_name, exec_dir, default_dir) \
print "Config file has been created at %s.\nYou will probably have to edit it." % config_path + "Config file has been created at %s.\n" % config_path \
sys.stderr.write("No %s found, copying %s.example.\n" % (file_name, file_name) ) + "Edit your screen_name and hand history path in the supported_sites "\
+ "section of the \nPreferences window (Main menu) before trying to import hands."
print msg
logging.info(msg)
file_name = config_path file_name = config_path
except: except:
print "Error copying .example file, cannot fall back. Exiting.\n" print "Error copying .example file, cannot fall back. Exiting.\n"
sys.stderr.write("Error copying .example file, cannot fall back. Exiting.\n") sys.stderr.write("Error copying .example file, cannot fall back. Exiting.\n")
sys.stderr.write( str(sys.exc_info()) )
sys.exit() sys.exit()
else: else:
print "No %s found, cannot fall back. Exiting.\n" % file_name print "No %s found, cannot fall back. Exiting.\n" % file_name

81
pyfpdb/Database.py
View File

@ -39,40 +39,38 @@ import string
import re import re
import Queue import Queue
import codecs import codecs
import logging
import math import math
# pyGTK modules # pyGTK modules
# FreePokerTools modules
import SQL
import Card
import Tourney
import Charset
from Exceptions import *
import Configuration
log = Configuration.get_logger("logging.conf","db")
# Other library modules # Other library modules
try: try:
import sqlalchemy.pool as pool import sqlalchemy.pool as pool
use_pool = True use_pool = True
except ImportError: except ImportError:
logging.info("Not using sqlalchemy connection pool.") log.info("Not using sqlalchemy connection pool.")
use_pool = False use_pool = False
try: try:
from numpy import var from numpy import var
use_numpy = True use_numpy = True
except ImportError: except ImportError:
logging.info("Not using numpy to define variance in sqlite.") log.info("Not using numpy to define variance in sqlite.")
use_numpy = False use_numpy = False
# FreePokerTools modules
import Configuration
import SQL
import Card
import Tourney
import Charset
from Exceptions import *
log = Configuration.get_logger("logging.conf")
DB_VERSION = 119 DB_VERSION = 119
@ -374,20 +372,20 @@ class Database:
print msg print msg
raise FpdbError(msg) raise FpdbError(msg)
elif backend == Database.SQLITE: elif backend == Database.SQLITE:
logging.info("Connecting to SQLite: %(database)s" % {'database':database}) log.info("Connecting to SQLite: %(database)s" % {'database':database})
import sqlite3 import sqlite3
if use_pool: if use_pool:
sqlite3 = pool.manage(sqlite3, pool_size=1) sqlite3 = pool.manage(sqlite3, pool_size=1)
else: else:
logging.warning("SQLite won't work well without 'sqlalchemy' installed.") log.warning("SQLite won't work well without 'sqlalchemy' installed.")
if database != ":memory:": if database != ":memory:":
if not os.path.isdir(self.config.dir_databases): if not os.path.isdir(self.config.dir_databases):
print "Creating directory: '%s'" % (self.config.dir_databases) print "Creating directory: '%s'" % (self.config.dir_databases)
logging.info("Creating directory: '%s'" % (self.config.dir_databases)) log.info("Creating directory: '%s'" % (self.config.dir_databases))
os.mkdir(self.config.dir_databases) os.mkdir(self.config.dir_databases)
database = os.path.join(self.config.dir_databases, database) database = os.path.join(self.config.dir_databases, database)
logging.info(" sqlite db: " + database) log.info(" sqlite db: " + database)
self.connection = sqlite3.connect(database, detect_types=sqlite3.PARSE_DECLTYPES ) self.connection = sqlite3.connect(database, detect_types=sqlite3.PARSE_DECLTYPES )
sqlite3.register_converter("bool", lambda x: bool(int(x))) sqlite3.register_converter("bool", lambda x: bool(int(x)))
sqlite3.register_adapter(bool, lambda x: "1" if x else "0") sqlite3.register_adapter(bool, lambda x: "1" if x else "0")
@ -397,7 +395,10 @@ class Database:
if use_numpy: if use_numpy:
self.connection.create_aggregate("variance", 1, VARIANCE) self.connection.create_aggregate("variance", 1, VARIANCE)
else: else:
logging.warning("Some database functions will not work without NumPy support") log.warning("Some database functions will not work without NumPy support")
self.cursor = self.connection.cursor()
self.cursor.execute('PRAGMA temp_store=2') # use memory for temp tables/indexes
self.cursor.execute('PRAGMA synchronous=0') # don't wait for file writes to finish
else: else:
raise FpdbError("unrecognised database backend:"+backend) raise FpdbError("unrecognised database backend:"+backend)
@ -412,7 +413,7 @@ class Database:
self.cursor.execute("SELECT * FROM Settings") self.cursor.execute("SELECT * FROM Settings")
settings = self.cursor.fetchone() settings = self.cursor.fetchone()
if settings[0] != DB_VERSION: if settings[0] != DB_VERSION:
logging.error("outdated or too new database version (%s) - please recreate tables" log.error("outdated or too new database version (%s) - please recreate tables"
% (settings[0])) % (settings[0]))
self.wrongDbVersion = True self.wrongDbVersion = True
except:# _mysql_exceptions.ProgrammingError: except:# _mysql_exceptions.ProgrammingError:
@ -422,11 +423,6 @@ class Database:
log.info("failed to read settings table - recreating tables") log.info("failed to read settings table - recreating tables")
self.recreate_tables() self.recreate_tables()
self.check_version(database=database, create=False) self.check_version(database=database, create=False)
if not self.wrongDbVersion:
msg = "Edit your screen_name and hand history path in the supported_sites "\
+"section of the \nPreferences window (Main menu) before trying to import hands"
print "\n%s" % msg
log.warning(msg)
else: else:
print "Failed to read settings table - please recreate tables" print "Failed to read settings table - please recreate tables"
log.info("failed to read settings table - please recreate tables") log.info("failed to read settings table - please recreate tables")
@ -436,7 +432,27 @@ class Database:
#end def connect #end def connect
def commit(self): def commit(self):
if self.backend != self.SQLITE:
self.connection.commit() self.connection.commit()
else:
# sqlite commits can fail because of shared locks on the database (SQLITE_BUSY)
# re-try commit if it fails in case this happened
maxtimes = 5
pause = 1
ok = False
for i in xrange(maxtimes):
try:
ret = self.connection.commit()
log.debug("commit finished ok, i = "+str(i))
ok = True
except:
log.debug("commit "+str(i)+" failed: info=" + str(sys.exc_info())
+ " value=" + str(sys.exc_value))
sleep(pause)
if ok: break
if not ok:
log.debug("commit failed")
raise FpdbError('sqlite commit failed')
def rollback(self): def rollback(self):
self.connection.rollback() self.connection.rollback()
@ -1750,6 +1766,9 @@ class Database:
def insertPlayer(self, name, site_id): def insertPlayer(self, name, site_id):
result = None result = None
if self.backend == self.SQLITE:
_name = name
else:
_name = Charset.to_db_utf8(name) _name = Charset.to_db_utf8(name)
c = self.get_cursor() c = self.get_cursor()
q = "SELECT name, id FROM Players WHERE siteid=%s and name=%s" q = "SELECT name, id FROM Players WHERE siteid=%s and name=%s"
@ -1906,7 +1925,7 @@ class Database:
# end def send_finish_msg(): # end def send_finish_msg():
def tRecogniseTourneyType(self, tourney): def tRecogniseTourneyType(self, tourney):
logging.debug("Database.tRecogniseTourneyType") log.debug("Database.tRecogniseTourneyType")
typeId = 1 typeId = 1
# Check if Tourney exists, and if so retrieve TTypeId : in that case, check values of the ttype # Check if Tourney exists, and if so retrieve TTypeId : in that case, check values of the ttype
cursor = self.get_cursor() cursor = self.get_cursor()
@ -1922,10 +1941,10 @@ class Database:
try: try:
len(result) len(result)
typeId = result[0] typeId = result[0]
logging.debug("Tourney found in db with Tourney_Type_ID = %d" % typeId) log.debug("Tourney found in db with Tourney_Type_ID = %d" % typeId)
for ev in expectedValues : for ev in expectedValues :
if ( getattr( tourney, expectedValues.get(ev) ) <> result[ev] ): if ( getattr( tourney, expectedValues.get(ev) ) <> result[ev] ):
logging.debug("TypeId mismatch : wrong %s : Tourney=%s / db=%s" % (expectedValues.get(ev), getattr( tourney, expectedValues.get(ev)), result[ev]) ) log.debug("TypeId mismatch : wrong %s : Tourney=%s / db=%s" % (expectedValues.get(ev), getattr( tourney, expectedValues.get(ev)), result[ev]) )
typeIdMatch = False typeIdMatch = False
#break #break
except: except:
@ -1935,7 +1954,7 @@ class Database:
if typeIdMatch == False : if typeIdMatch == False :
# Check for an existing TTypeId that matches tourney info (buyin/fee, knockout, rebuy, speed, matrix, shootout) # Check for an existing TTypeId that matches tourney info (buyin/fee, knockout, rebuy, speed, matrix, shootout)
# if not found create it # if not found create it
logging.debug("Searching for a TourneyTypeId matching TourneyType data") log.debug("Searching for a TourneyTypeId matching TourneyType data")
cursor.execute (self.sql.query['getTourneyTypeId'].replace('%s', self.sql.query['placeholder']), cursor.execute (self.sql.query['getTourneyTypeId'].replace('%s', self.sql.query['placeholder']),
(tourney.siteId, tourney.buyin, tourney.fee, tourney.isKO, (tourney.siteId, tourney.buyin, tourney.fee, tourney.isKO,
tourney.isRebuy, tourney.speed, tourney.isHU, tourney.isShootout, tourney.isMatrix) tourney.isRebuy, tourney.speed, tourney.isHU, tourney.isShootout, tourney.isMatrix)
@ -1945,9 +1964,9 @@ class Database:
try: try:
len(result) len(result)
typeId = result[0] typeId = result[0]
logging.debug("Existing Tourney Type Id found : %d" % typeId) log.debug("Existing Tourney Type Id found : %d" % typeId)
except TypeError: #this means we need to create a new entry except TypeError: #this means we need to create a new entry
logging.debug("Tourney Type Id not found : create one") log.debug("Tourney Type Id not found : create one")
cursor.execute (self.sql.query['insertTourneyTypes'].replace('%s', self.sql.query['placeholder']), cursor.execute (self.sql.query['insertTourneyTypes'].replace('%s', self.sql.query['placeholder']),
(tourney.siteId, tourney.buyin, tourney.fee, tourney.isKO, tourney.isRebuy, (tourney.siteId, tourney.buyin, tourney.fee, tourney.isKO, tourney.isRebuy,
tourney.speed, tourney.isHU, tourney.isShootout, tourney.isMatrix) tourney.speed, tourney.isHU, tourney.isShootout, tourney.isMatrix)

5
pyfpdb/HUD_main.py
View File

@ -204,6 +204,7 @@ class HUD_main(object):
# get basic info about the new hand from the db # get basic info about the new hand from the db
# if there is a db error, complain, skip hand, and proceed # if there is a db error, complain, skip hand, and proceed
log.info("HUD_main.read_stdin: hand processing starting ...")
try: try:
(table_name, max, poker_game, type, site_id, site_name, num_seats, tour_number, tab_number) = \ (table_name, max, poker_game, type, site_id, site_name, num_seats, tour_number, tab_number) = \
self.db_connection.get_table_info(new_hand_id) self.db_connection.get_table_info(new_hand_id)
@ -230,6 +231,7 @@ class HUD_main(object):
try: try:
self.db_connection.init_hud_stat_vars( self.hud_dict[temp_key].hud_params['hud_days'] self.db_connection.init_hud_stat_vars( self.hud_dict[temp_key].hud_params['hud_days']
, self.hud_dict[temp_key].hud_params['h_hud_days']) , self.hud_dict[temp_key].hud_params['h_hud_days'])
t4 = time.time()
stat_dict = self.db_connection.get_stats_from_hand(new_hand_id, type, self.hud_dict[temp_key].hud_params, self.hero_ids[site_id]) stat_dict = self.db_connection.get_stats_from_hand(new_hand_id, type, self.hud_dict[temp_key].hud_params, self.hero_ids[site_id])
self.hud_dict[temp_key].stat_dict = stat_dict self.hud_dict[temp_key].stat_dict = stat_dict
except KeyError: # HUD instance has been killed off, key is stale except KeyError: # HUD instance has been killed off, key is stale
@ -238,6 +240,7 @@ class HUD_main(object):
# Unlocks table, copied from end of function # Unlocks table, copied from end of function
self.db_connection.connection.rollback() self.db_connection.connection.rollback()
return return
t5 = time.time()
cards = self.db_connection.get_cards(new_hand_id) cards = self.db_connection.get_cards(new_hand_id)
comm_cards = self.db_connection.get_common_cards(new_hand_id) comm_cards = self.db_connection.get_common_cards(new_hand_id)
if comm_cards != {}: # stud! if comm_cards != {}: # stud!
@ -250,10 +253,8 @@ class HUD_main(object):
else: else:
# get stats using default params--also get cards # get stats using default params--also get cards
self.db_connection.init_hud_stat_vars( self.hud_params['hud_days'], self.hud_params['h_hud_days'] ) self.db_connection.init_hud_stat_vars( self.hud_params['hud_days'], self.hud_params['h_hud_days'] )
t4 = time.time()
stat_dict = self.db_connection.get_stats_from_hand(new_hand_id, type, self.hud_params stat_dict = self.db_connection.get_stats_from_hand(new_hand_id, type, self.hud_params
,self.hero_ids[site_id], num_seats) ,self.hero_ids[site_id], num_seats)
t5 = time.time()
cards = self.db_connection.get_cards(new_hand_id) cards = self.db_connection.get_cards(new_hand_id)
comm_cards = self.db_connection.get_common_cards(new_hand_id) comm_cards = self.db_connection.get_common_cards(new_hand_id)
if comm_cards != {}: # stud! if comm_cards != {}: # stud!