diff --git a/pyfpdb/Configuration.py b/pyfpdb/Configuration.py index abd5b016..9093c22f 100755 --- a/pyfpdb/Configuration.py +++ b/pyfpdb/Configuration.py @@ -56,36 +56,36 @@ def get_exec_path(): if hasattr(sys, "frozen"): # compiled by py2exe return os.path.dirname(sys.executable) else: - pathname = os.path.dirname(sys.argv[0]) - return os.path.abspath(pathname) + return sys.path[0] def get_config(file_name, fallback = True): - """Looks in cwd and in self.default_config_path for a config file.""" - config_path = os.path.join(get_exec_path(), file_name) -# print "config_path=", config_path - if os.path.exists(config_path): # there is a file in the cwd - return config_path # so we use it - else: # no file in the cwd, look where it should be in the first place - config_path = os.path.join(get_default_config_path(), file_name) -# print "config path 2=", config_path - if os.path.exists(config_path): + """Looks in exec dir and in self.default_config_path for a config file.""" + config_path = os.path.join(DIR_SELF, file_name) # look in exec dir + if os.path.exists(config_path) and os.path.isfile(config_path): + return config_path # there is a file in the exec dir so we use it + else: + config_path = os.path.join(DIR_CONFIG, file_name) # look in config dir + if os.path.exists(config_path) and os.path.isfile(config_path): return config_path # No file found if not fallback: return False -# OK, fall back to the .example file, should be in the start dir - if os.path.exists(file_name + ".example"): +# OK, fall back to the .example file, should be in the exec dir + if os.path.exists(os.path.join(DIR_SELF, file_name + ".example")): try: - shutil.copyfile(file_name + ".example", file_name) + shutil.copyfile(os.path.join(DIR_SELF, file_name + ".example"), os.path.join(DIR_CONFIG, file_name)) print "No %s found, using %s.example.\n" % (file_name, file_name) - print "A %s file has been created. You will probably have to edit it." % file_name - sys.stderr.write("No %s found, using %s.example.\n" % (file_name, file_name) ) + print "A %s file has been created. You will probably have to edit it." % os.path.join(DIR_CONFIG, file_name) + log.error("No %s found, using %s.example.\n" % (file_name, file_name) ) except: print "No %s found, cannot fall back. Exiting.\n" % file_name - sys.stderr.write("No %s found, cannot fall back. Exiting.\n" % file_name) sys.exit() + else: + print "No %s found, cannot fall back. Exiting.\n" % file_name + sys.stderr.write("No %s found, cannot fall back. Exiting.\n" % file_name) + sys.exit() return file_name def get_logger(file_name, config = "config", fallback = False): @@ -94,18 +94,26 @@ def get_logger(file_name, config = "config", fallback = False): try: logging.config.fileConfig(conf) log = logging.getLogger(config) - log.debug("%s logger initialised" % config) return log except: pass log = logging.basicConfig() log = logging.getLogger() - log.debug("config logger initialised") + log.error("basicConfig logger initialised") return log -# find a logging.conf file and set up logging -log = get_logger("logging.conf") +def check_dir(path, create = True): + """Check if a dir exists, optionally creates if not.""" + if os.path.exists(path): + if os.path.isdir(path): + return path + else: + return False + if create: + print "creating directory %s" % path + else: + return False ######################################################################## # application wide consts @@ -113,19 +121,31 @@ log = get_logger("logging.conf") APPLICATION_NAME_SHORT = 'fpdb' APPLICATION_VERSION = 'xx.xx.xx' -DIR_SELF = os.path.dirname(get_exec_path()) -#TODO: imo no good idea to place 'database' in parent dir -DIR_DATABASES = os.path.join(os.path.dirname(DIR_SELF), 'database') +DIR_SELF = get_exec_path() +DIR_CONFIG = check_dir(get_default_config_path()) +DIR_DATABASE = check_dir(os.path.join(DIR_CONFIG, 'database')) +DIR_LOG = check_dir(os.path.join(DIR_CONFIG, 'log')) DATABASE_TYPE_POSTGRESQL = 'postgresql' DATABASE_TYPE_SQLITE = 'sqlite' DATABASE_TYPE_MYSQL = 'mysql' +#TODO: should this be a tuple or a dict DATABASE_TYPES = ( DATABASE_TYPE_POSTGRESQL, DATABASE_TYPE_SQLITE, DATABASE_TYPE_MYSQL, ) +# find a logging.conf file and set up logging +log = get_logger("logging.conf", config = "config") +log.debug("config logger initialised") + +# and then log our consts +log.info("DIR SELF = %s" % DIR_SELF) +log.info("DIR CONFIG = %s" % DIR_CONFIG) +log.info("DIR DATABASE = %s" % DIR_DATABASE) +log.info("DIR LOG = %s" % DIR_LOG) +NEWIMPORT = True LOCALE_ENCODING = locale.getdefaultlocale()[1] ######################################################################## @@ -408,11 +428,10 @@ class Config: if file is not None: # config file path passed in file = os.path.expanduser(file) if not os.path.exists(file): - print "Configuration file %s not found. Using defaults." % (file) - sys.stderr.write("Configuration file %s not found. Using defaults." % (file)) + log.error("Specified configuration file %s not found. Using defaults." % (file)) file = None - if file is None: file = get_config("HUD_config.xml") + if file is None: file = get_config("HUD_config.xml", True) # Parse even if there was no real config file found and we are using the example # If using the example, we'll edit it later @@ -429,6 +448,8 @@ class Config: self.doc = doc self.file = file + self.dir = os.path.dirname(self.file) + self.dir_databases = os.path.join(self.dir, 'database') self.supported_sites = {} self.supported_games = {} self.supported_databases = {} # databaseName --> Database instance diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index c340079c..20807583 100644 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -38,11 +38,31 @@ from decimal import Decimal import string import re import Queue +import codecs +import logging +import math + # pyGTK modules + +# Other library modules +try: + import sqlalchemy.pool as pool + use_pool = True +except ImportError: + logging.info("Not using sqlalchemy connection pool.") + use_pool = False + +try: + from numpy import var + use_numpy = True +except ImportError: + logging.info("Not using numpy to define variance in sqlite.") + use_numpy = False + + # FreePokerTools modules -import fpdb_db import Configuration import SQL import Card @@ -50,7 +70,29 @@ import Tourney import Charset from Exceptions import * -log = Configuration.get_logger("logging.conf") +log = Configuration.get_logger("logging.conf", config = "db") +log.debug("db logger initialized.") +encoder = codecs.lookup('utf-8') + +DB_VERSION = 119 + + +# Variance created as sqlite has a bunch of undefined aggregate functions. + +class VARIANCE: + def __init__(self): + self.store = [] + + def step(self, value): + self.store.append(value) + + def finalize(self): + return float(var(self.store)) + +class sqlitemath: + def mod(self, a, b): + return a%b + class Database: @@ -188,15 +230,14 @@ class Database: log.info("Creating Database instance, sql = %s" % sql) self.config = c self.__connected = False - self.fdb = fpdb_db.fpdb_db() # sets self.fdb.db self.fdb.cursor and self.fdb.sql - self.do_connect(c) - - if self.backend == self.PGSQL: - from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT, ISOLATION_LEVEL_READ_COMMITTED, ISOLATION_LEVEL_SERIALIZABLE - #ISOLATION_LEVEL_AUTOCOMMIT = 0 - #ISOLATION_LEVEL_READ_COMMITTED = 1 - #ISOLATION_LEVEL_SERIALIZABLE = 2 - + self.settings = {} + self.settings['os'] = "linuxmac" if os.name != "nt" else "windows" + db_params = c.get_db_parameters() + self.import_options = c.get_import_parameters() + self.backend = db_params['db-backend'] + self.db_server = db_params['db-server'] + self.database = db_params['db-databaseName'] + self.host = db_params['db-host'] # where possible avoid creating new SQL instance by using the global one passed in if sql is None: @@ -204,6 +245,15 @@ class Database: else: self.sql = sql + # connect to db + self.do_connect(c) + print "connection =", self.connection + if self.backend == self.PGSQL: + from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT, ISOLATION_LEVEL_READ_COMMITTED, ISOLATION_LEVEL_SERIALIZABLE + #ISOLATION_LEVEL_AUTOCOMMIT = 0 + #ISOLATION_LEVEL_READ_COMMITTED = 1 + #ISOLATION_LEVEL_SERIALIZABLE = 2 + if self.backend == self.SQLITE and self.database == ':memory:' and self.wrongDbVersion: log.info("sqlite/:memory: - creating") self.recreate_tables() @@ -226,8 +276,6 @@ class Database: self.h_date_ndays_ago = 'd000000' # date N days ago ('d' + YYMMDD) for hero self.date_nhands_ago = {} # dates N hands ago per player - not used yet - self.cursor = self.fdb.cursor - self.saveActions = False if self.import_options['saveActions'] == False else True self.connection.rollback() # make sure any locks taken so far are released @@ -238,14 +286,20 @@ class Database: self.hud_style = style def do_connect(self, c): + if c is None: + raise FpdbError('Configuration not defined') + + db = c.get_db_parameters() try: - self.fdb.do_connect(c) + self.connect(backend=db['db-backend'], + host=db['db-host'], + database=db['db-databaseName'], + user=db['db-user'], + password=db['db-password']) except: # error during connect self.__connected = False raise - self.connection = self.fdb.db - self.wrongDbVersion = self.fdb.wrongDbVersion db_params = c.get_db_parameters() self.import_options = c.get_import_parameters() @@ -255,11 +309,137 @@ class Database: self.host = db_params['db-host'] self.__connected = True + def connect(self, backend=None, host=None, database=None, + user=None, password=None): + """Connects a database with the given parameters""" + if backend is None: + raise FpdbError('Database backend not defined') + self.backend = backend + self.host = host + self.user = user + self.password = password + self.database = database + self.connection = None + self.cursor = None + + if backend == Database.MYSQL_INNODB: + import MySQLdb + if use_pool: + MySQLdb = pool.manage(MySQLdb, pool_size=5) + try: + self.connection = MySQLdb.connect(host=host, user=user, passwd=password, db=database, use_unicode=True) + #TODO: Add port option + except MySQLdb.Error, ex: + if ex.args[0] == 1045: + raise FpdbMySQLAccessDenied(ex.args[0], ex.args[1]) + elif ex.args[0] == 2002 or ex.args[0] == 2003: # 2002 is no unix socket, 2003 is no tcp socket + raise FpdbMySQLNoDatabase(ex.args[0], ex.args[1]) + else: + print "*** WARNING UNKNOWN MYSQL ERROR", ex + elif backend == Database.PGSQL: + import psycopg2 + import psycopg2.extensions + if use_pool: + psycopg2 = pool.manage(psycopg2, pool_size=5) + psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) + # If DB connection is made over TCP, then the variables + # host, user and password are required + # For local domain-socket connections, only DB name is + # needed, and everything else is in fact undefined and/or + # flat out wrong + # sqlcoder: This database only connect failed in my windows setup?? + # Modifed it to try the 4 parameter style if the first connect fails - does this work everywhere? + connected = False + if self.host == "localhost" or self.host == "127.0.0.1": + try: + self.connection = psycopg2.connect(database = database) + connected = True + except: + # direct connection failed so try user/pass/... version + pass + if not connected: + try: + self.connection = psycopg2.connect(host = host, + user = user, + password = password, + database = database) + except Exception, ex: + if 'Connection refused' in ex.args[0]: + # meaning eg. db not running + raise FpdbPostgresqlNoDatabase(errmsg = ex.args[0]) + elif 'password authentication' in ex.args[0]: + raise FpdbPostgresqlAccessDenied(errmsg = ex.args[0]) + else: + msg = ex.args[0] + print msg + raise FpdbError(msg) + elif backend == Database.SQLITE: + logging.info("Connecting to SQLite: %(database)s" % {'database':database}) + import sqlite3 + if use_pool: + sqlite3 = pool.manage(sqlite3, pool_size=1) + else: + logging.warning("SQLite won't work well without 'sqlalchemy' installed.") + + if database != ":memory:": + if not os.path.isdir(self.config.dir_databases): + print "Creating directory: '%s'" % (self.config.dir_databases) + logging.info("Creating directory: '%s'" % (self.config.dir_databases)) + os.mkdir(self.config.dir_databases) + database = os.path.join(self.config.dir_databases, database) + logging.info(" sqlite db: " + database) + self.connection = sqlite3.connect(database, detect_types=sqlite3.PARSE_DECLTYPES ) + sqlite3.register_converter("bool", lambda x: bool(int(x))) + sqlite3.register_adapter(bool, lambda x: "1" if x else "0") + self.connection.create_function("floor", 1, math.floor) + tmp = sqlitemath() + self.connection.create_function("mod", 2, tmp.mod) + if use_numpy: + self.connection.create_aggregate("variance", 1, VARIANCE) + else: + logging.warning("Some database functions will not work without NumPy support") + else: + raise FpdbError("unrecognised database backend:"+backend) + + self.cursor = self.connection.cursor() + self.cursor.execute(self.sql.query['set tx level']) + self.check_version(database=database, create=True) + + + def check_version(self, database, create): + self.wrongDbVersion = False + try: + self.cursor.execute("SELECT * FROM Settings") + settings = self.cursor.fetchone() + if settings[0] != DB_VERSION: + logging.error("outdated or too new database version (%s) - please recreate tables" + % (settings[0])) + self.wrongDbVersion = True + except:# _mysql_exceptions.ProgrammingError: + if database != ":memory:": + if create: + print "Failed to read settings table - recreating tables" + log.info("failed to read settings table - recreating tables") + self.recreate_tables() + self.check_version(database=database, create=False) + if not self.wrongDbVersion: + msg = "Edit your screen_name and hand history path in the supported_sites "\ + +"section of the \nPreferences window (Main menu) before trying to import hands" + print "\n%s" % msg + log.warning(msg) + else: + print "Failed to read settings table - please recreate tables" + log.info("failed to read settings table - please recreate tables") + self.wrongDbVersion = True + else: + self.wrongDbVersion = True + #end def connect + def commit(self): - self.fdb.db.commit() + self.connection.commit() def rollback(self): - self.fdb.db.rollback() + self.connection.rollback() def connected(self): return self.__connected @@ -272,11 +452,18 @@ class Database: def disconnect(self, due_to_error=False): """Disconnects the DB (rolls back if param is true, otherwise commits""" - self.fdb.disconnect(due_to_error) + if due_to_error: + self.connection.rollback() + else: + self.connection.commit() + self.cursor.close() + self.connection.close() def reconnect(self, due_to_error=False): """Reconnects the DB""" - self.fdb.reconnect(due_to_error=False) + #print "started reconnect" + self.disconnect(due_to_error) + self.connect(self.backend, self.host, self.database, self.user, self.password) def get_backend_name(self): """Returns the name of the currently used backend""" @@ -289,6 +476,9 @@ class Database: else: raise FpdbError("invalid backend") + def get_db_info(self): + return (self.host, self.database, self.user, self.password) + def get_table_name(self, hand_id): c = self.connection.cursor() c.execute(self.sql.query['get_table_name'], (hand_id, )) @@ -844,6 +1034,7 @@ class Database: self.create_tables() self.createAllIndexes() self.commit() + print "Finished recreating tables" log.info("Finished recreating tables") #end def recreate_tables @@ -1109,7 +1300,7 @@ class Database: def fillDefaultData(self): c = self.get_cursor() - c.execute("INSERT INTO Settings (version) VALUES (118);") + c.execute("INSERT INTO Settings (version) VALUES (%s);" % (DB_VERSION)) c.execute("INSERT INTO Sites (name,currency) VALUES ('Full Tilt Poker', 'USD')") c.execute("INSERT INTO Sites (name,currency) VALUES ('PokerStars', 'USD')") c.execute("INSERT INTO Sites (name,currency) VALUES ('Everleaf', 'USD')") @@ -1265,7 +1456,7 @@ class Database: try: self.get_cursor().execute(self.sql.query['lockForInsert']) except: - print "Error during fdb.lock_for_insert:", str(sys.exc_value) + print "Error during lock_for_insert:", str(sys.exc_value) #end def lock_for_insert ########################### @@ -1284,6 +1475,7 @@ class Database: p['tableName'], p['gameTypeId'], p['siteHandNo'], + 0, # tourneyId: 0 means not a tourney hand p['handStart'], datetime.today(), #importtime p['seats'], diff --git a/pyfpdb/Filters.py b/pyfpdb/Filters.py index 37a31c9e..fa6d2400 100644 --- a/pyfpdb/Filters.py +++ b/pyfpdb/Filters.py @@ -27,8 +27,8 @@ import gobject #import pokereval import Configuration -import fpdb_db -import FpdbSQLQueries +import Database +import SQL import Charset class Filters(threading.Thread): @@ -800,10 +800,10 @@ def main(argv=None): config = Configuration.Config() db = None - db = fpdb_db.fpdb_db() + db = Database.Database() db.do_connect(config) - qdict = FpdbSQLQueries.FpdbSQLQueries(db.get_backend_name()) + qdict = SQL.SQL(db.get_backend_name()) i = Filters(db, config, qdict) main_window = gtk.Window() diff --git a/pyfpdb/FulltiltToFpdb.py b/pyfpdb/FulltiltToFpdb.py index 19f834e6..2802f465 100755 --- a/pyfpdb/FulltiltToFpdb.py +++ b/pyfpdb/FulltiltToFpdb.py @@ -18,12 +18,10 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ######################################################################## -import sys import logging from HandHistoryConverter import * # Fulltilt HH Format converter -# TODO: cat tourno and table to make table name for tournaments class Fulltilt(HandHistoryConverter): @@ -67,8 +65,8 @@ class Fulltilt(HandHistoryConverter): (\s\((?PTurbo)\))?)|(?P.+)) ''', re.VERBOSE) re_Button = re.compile('^The button is in seat #(?P