From 11bc48e126dd961fe58dece3ebd519d35edf0056 Mon Sep 17 00:00:00 2001 From: sqlcoder Date: Tue, 26 Jan 2010 23:54:04 +0000 Subject: [PATCH] merge fpdb_db.py into Database.py --- pyfpdb/Database.py | 206 +++++++++++++++++++++++++++++++++---- pyfpdb/Filters.py | 8 +- pyfpdb/GuiAutoImport.py | 1 - pyfpdb/GuiPlayerStats.py | 1 - pyfpdb/Hand.py | 2 +- pyfpdb/SQL.py | 21 ++++ pyfpdb/Tourney.py | 2 +- pyfpdb/fpdb.py | 1 - pyfpdb/fpdb_db.py | 213 --------------------------------------- pyfpdb/fpdb_import.py | 1 - 10 files changed, 215 insertions(+), 241 deletions(-) diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index c24d13fb..2c649dc4 100644 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -39,11 +39,30 @@ import string import re import Queue import codecs +import logging +import math + # pyGTK modules + +# Other library modules +try: + import sqlalchemy.pool as pool + use_pool = True +except ImportError: + logging.info("Not using sqlalchemy connection pool.") + use_pool = False + +try: + from numpy import var + use_numpy = True +except ImportError: + logging.info("Not using numpy to define variance in sqlite.") + use_numpy = False + + # FreePokerTools modules -import fpdb_db import Configuration import SQL import Card @@ -54,6 +73,27 @@ from Exceptions import * log = Configuration.get_logger("logging.conf") encoder = codecs.lookup('utf-8') + +DB_VERSION = 118 + + +# Variance created as sqlite has a bunch of undefined aggregate functions. + +class VARIANCE: + def __init__(self): + self.store = [] + + def step(self, value): + self.store.append(value) + + def finalize(self): + return float(var(self.store)) + +class sqlitemath: + def mod(self, a, b): + return a%b + + class Database: MYSQL_INNODB = 2 @@ -190,7 +230,22 @@ class Database: log.info("Creating Database instance, sql = %s" % sql) self.config = c self.__connected = False - self.fdb = fpdb_db.fpdb_db() # sets self.fdb.db self.fdb.cursor and self.fdb.sql + self.settings = {} + self.settings['os'] = "linuxmac" if os.name != "nt" else "windows" + db_params = c.get_db_parameters() + self.import_options = c.get_import_parameters() + self.backend = db_params['db-backend'] + self.db_server = db_params['db-server'] + self.database = db_params['db-databaseName'] + self.host = db_params['db-host'] + + # where possible avoid creating new SQL instance by using the global one passed in + if sql is None: + self.sql = SQL.Sql(db_server = self.db_server) + else: + self.sql = sql + + # connect to db self.do_connect(c) if self.backend == self.PGSQL: @@ -200,12 +255,6 @@ class Database: #ISOLATION_LEVEL_SERIALIZABLE = 2 - # where possible avoid creating new SQL instance by using the global one passed in - if sql is None: - self.sql = SQL.Sql(db_server = self.db_server) - else: - self.sql = sql - if self.backend == self.SQLITE and self.database == ':memory:' and self.wrongDbVersion: log.info("sqlite/:memory: - creating") self.recreate_tables() @@ -228,8 +277,6 @@ class Database: self.h_date_ndays_ago = 'd000000' # date N days ago ('d' + YYMMDD) for hero self.date_nhands_ago = {} # dates N hands ago per player - not used yet - self.cursor = self.fdb.cursor - self.saveActions = False if self.import_options['saveActions'] == False else True self.connection.rollback() # make sure any locks taken so far are released @@ -240,14 +287,20 @@ class Database: self.hud_style = style def do_connect(self, c): + if c is None: + raise FpdbError('Configuration not defined') + + db = c.get_db_parameters() try: - self.fdb.do_connect(c) + self.connect(backend=db['db-backend'], + host=db['db-host'], + database=db['db-databaseName'], + user=db['db-user'], + password=db['db-password']) except: # error during connect self.__connected = False raise - self.connection = self.fdb.db - self.wrongDbVersion = self.fdb.wrongDbVersion db_params = c.get_db_parameters() self.import_options = c.get_import_parameters() @@ -257,11 +310,118 @@ class Database: self.host = db_params['db-host'] self.__connected = True + def connect(self, backend=None, host=None, database=None, + user=None, password=None): + """Connects a database with the given parameters""" + if backend is None: + raise FpdbError('Database backend not defined') + self.backend = backend + self.host = host + self.user = user + self.password = password + self.database = database + self.connection = None + self.cursor = None + + if backend == Database.MYSQL_INNODB: + import MySQLdb + if use_pool: + MySQLdb = pool.manage(MySQLdb, pool_size=5) + try: + self.connection = MySQLdb.connect(host=host, user=user, passwd=password, db=database, use_unicode=True) + #TODO: Add port option + except MySQLdb.Error, ex: + if ex.args[0] == 1045: + raise FpdbMySQLAccessDenied(ex.args[0], ex.args[1]) + elif ex.args[0] == 2002 or ex.args[0] == 2003: # 2002 is no unix socket, 2003 is no tcp socket + raise FpdbMySQLNoDatabase(ex.args[0], ex.args[1]) + else: + print "*** WARNING UNKNOWN MYSQL ERROR", ex + elif backend == Database.PGSQL: + import psycopg2 + import psycopg2.extensions + if use_pool: + psycopg2 = pool.manage(psycopg2, pool_size=5) + psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) + # If DB connection is made over TCP, then the variables + # host, user and password are required + # For local domain-socket connections, only DB name is + # needed, and everything else is in fact undefined and/or + # flat out wrong + # sqlcoder: This database only connect failed in my windows setup?? + # Modifed it to try the 4 parameter style if the first connect fails - does this work everywhere? + connected = False + if self.host == "localhost" or self.host == "127.0.0.1": + try: + self.connection = psycopg2.connect(database = database) + connected = True + except: + # direct connection failed so try user/pass/... version + pass + if not connected: + try: + self.connection = psycopg2.connect(host = host, + user = user, + password = password, + database = database) + except Exception, ex: + if 'Connection refused' in ex.args[0]: + # meaning eg. db not running + raise FpdbPostgresqlNoDatabase(errmsg = ex.args[0]) + elif 'password authentication' in ex.args[0]: + raise FpdbPostgresqlAccessDenied(errmsg = ex.args[0]) + else: + msg = ex.args[0] + print msg + raise FpdbError(msg) + elif backend == Database.SQLITE: + logging.info("Connecting to SQLite: %(database)s" % {'database':database}) + import sqlite3 + if use_pool: + sqlite3 = pool.manage(sqlite3, pool_size=1) + else: + logging.warning("SQLite won't work well without 'sqlalchemy' installed.") + + if database != ":memory:": + if not os.path.isdir(self.config.dir_databases): + print "Creating directory: '%s'" % (self.config.dir_databases) + logging.info("Creating directory: '%s'" % (self.config.dir_databases)) + os.mkdir(self.config.dir_databases) + database = os.path.join(self.config.dir_databases, database) + logging.info(" sqlite db: " + database) + self.connection = sqlite3.connect(database, detect_types=sqlite3.PARSE_DECLTYPES ) + sqlite3.register_converter("bool", lambda x: bool(int(x))) + sqlite3.register_adapter(bool, lambda x: "1" if x else "0") + self.connection.create_function("floor", 1, math.floor) + tmp = sqlitemath() + self.connection.create_function("mod", 2, tmp.mod) + if use_numpy: + self.connection.create_aggregate("variance", 1, VARIANCE) + else: + logging.warning("Some database functions will not work without NumPy support") + else: + raise FpdbError("unrecognised database backend:"+backend) + + self.cursor = self.connection.cursor() + self.cursor.execute(self.sql.query['set tx level']) + self.wrongDbVersion = False + try: + self.cursor.execute("SELECT * FROM Settings") + settings = self.cursor.fetchone() + if settings[0] != DB_VERSION: + logging.error("outdated or too new database version (%s) - please recreate tables" + % (settings[0])) + self.wrongDbVersion = True + except:# _mysql_exceptions.ProgrammingError: + if database != ":memory:": print "failed to read settings table - please recreate tables" + self.wrongDbVersion = True + #end def connect + def commit(self): - self.fdb.db.commit() + self.connection.commit() def rollback(self): - self.fdb.db.rollback() + self.connection.rollback() def connected(self): return self.__connected @@ -274,11 +434,18 @@ class Database: def disconnect(self, due_to_error=False): """Disconnects the DB (rolls back if param is true, otherwise commits""" - self.fdb.disconnect(due_to_error) + if due_to_error: + self.connection.rollback() + else: + self.connection.commit() + self.cursor.close() + self.connection.close() def reconnect(self, due_to_error=False): """Reconnects the DB""" - self.fdb.reconnect(due_to_error=False) + #print "started reconnect" + self.disconnect(due_to_error) + self.connect(self.backend, self.host, self.database, self.user, self.password) def get_backend_name(self): """Returns the name of the currently used backend""" @@ -291,6 +458,9 @@ class Database: else: raise FpdbError("invalid backend") + def get_db_info(self): + return (self.host, self.database, self.user, self.password) + def get_table_name(self, hand_id): c = self.connection.cursor() c.execute(self.sql.query['get_table_name'], (hand_id, )) @@ -1267,7 +1437,7 @@ class Database: try: self.get_cursor().execute(self.sql.query['lockForInsert']) except: - print "Error during fdb.lock_for_insert:", str(sys.exc_value) + print "Error during lock_for_insert:", str(sys.exc_value) #end def lock_for_insert ########################### diff --git a/pyfpdb/Filters.py b/pyfpdb/Filters.py index b0da336b..3a331bfd 100644 --- a/pyfpdb/Filters.py +++ b/pyfpdb/Filters.py @@ -27,8 +27,8 @@ import gobject #import pokereval import Configuration -import fpdb_db -import FpdbSQLQueries +import Database +import SQL import Charset class Filters(threading.Thread): @@ -790,10 +790,10 @@ def main(argv=None): config = Configuration.Config() db = None - db = fpdb_db.fpdb_db() + db = Database.Database() db.do_connect(config) - qdict = FpdbSQLQueries.FpdbSQLQueries(db.get_backend_name()) + qdict = SQL.SQL(db.get_backend_name()) i = Filters(db, config, qdict) main_window = gtk.Window() diff --git a/pyfpdb/GuiAutoImport.py b/pyfpdb/GuiAutoImport.py index 517293ec..f9f1c335 100755 --- a/pyfpdb/GuiAutoImport.py +++ b/pyfpdb/GuiAutoImport.py @@ -300,7 +300,6 @@ if __name__== "__main__": (options, argv) = parser.parse_args() config = Configuration.Config() -# db = fpdb_db.fpdb_db() settings = {} settings['minPrint'] = options.minPrint diff --git a/pyfpdb/GuiPlayerStats.py b/pyfpdb/GuiPlayerStats.py index 7a3213e8..c634538e 100644 --- a/pyfpdb/GuiPlayerStats.py +++ b/pyfpdb/GuiPlayerStats.py @@ -27,7 +27,6 @@ from time import time, strftime import Card import fpdb_import import Database -import fpdb_db import Filters import Charset diff --git a/pyfpdb/Hand.py b/pyfpdb/Hand.py index d2b795e6..3906b00c 100644 --- a/pyfpdb/Hand.py +++ b/pyfpdb/Hand.py @@ -205,7 +205,7 @@ dealt whether they were seen in a 'dealt to' line def insert(self, db): """ Function to insert Hand into database Should not commit, and do minimal selects. Callers may want to cache commits -db: a connected fpdb_db object""" +db: a connected Database object""" self.stats.getStats(self) diff --git a/pyfpdb/SQL.py b/pyfpdb/SQL.py index 5ff4c8e6..43925c01 100644 --- a/pyfpdb/SQL.py +++ b/pyfpdb/SQL.py @@ -58,6 +58,27 @@ class Sql: self.query['drop_table'] = """DROP TABLE IF EXISTS """ + ################################################################## + # Set transaction isolation level + ################################################################## + + if db_server == 'mysql' or db_server == 'postgresql': + self.query['set tx level'] = """SET SESSION TRANSACTION + ISOLATION LEVEL READ COMMITTED""" + elif db_server == 'sqlite': + self.query['set tx level'] = """ """ + + + ################################ + # Select basic info + ################################ + + self.query['getSiteId'] = """SELECT id from Sites where name = %s""" + + self.query['getGames'] = """SELECT DISTINCT category from Gametypes""" + + self.query['getLimits'] = """SELECT DISTINCT bigBlind from Gametypes ORDER by bigBlind DESC""" + ################################ # Create Settings ################################ diff --git a/pyfpdb/Tourney.py b/pyfpdb/Tourney.py index 8f7e59c5..281a6710 100644 --- a/pyfpdb/Tourney.py +++ b/pyfpdb/Tourney.py @@ -185,7 +185,7 @@ class Tourney(object): def old_insert_from_Hand(self, db): """ Function to insert Hand into database Should not commit, and do minimal selects. Callers may want to cache commits -db: a connected fpdb_db object""" +db: a connected Database object""" # TODO: # Players - base playerid and siteid tuple sqlids = db.getSqlPlayerIDs([p[1] for p in self.players], self.siteId) diff --git a/pyfpdb/fpdb.py b/pyfpdb/fpdb.py index bff044d5..5136d56b 100755 --- a/pyfpdb/fpdb.py +++ b/pyfpdb/fpdb.py @@ -112,7 +112,6 @@ import GuiGraphViewer import GuiSessionViewer import SQL import Database -import FpdbSQLQueries import Configuration import Exceptions diff --git a/pyfpdb/fpdb_db.py b/pyfpdb/fpdb_db.py index bdbf430c..2c70dd55 100644 --- a/pyfpdb/fpdb_db.py +++ b/pyfpdb/fpdb_db.py @@ -16,219 +16,6 @@ #In the "official" distribution you can find the license in #agpl-3.0.txt in the docs folder of the package. -import os -import re -import sys -import logging -import math -from time import time, strftime -from Exceptions import * -try: - import sqlalchemy.pool as pool - use_pool = True -except ImportError: - logging.info("Not using sqlalchemy connection pool.") - use_pool = False - -try: - from numpy import var - use_numpy = True -except ImportError: - logging.info("Not using numpy to define variance in sqlite.") - use_numpy = False - -import FpdbSQLQueries -import Configuration - - -DB_VERSION = 118 - - -# Variance created as sqlite has a bunch of undefined aggregate functions. - -class VARIANCE: - def __init__(self): - self.store = [] - - def step(self, value): - self.store.append(value) - - def finalize(self): - return float(var(self.store)) - -class sqlitemath: - def mod(self, a, b): - return a%b - -class fpdb_db: - MYSQL_INNODB = 2 - PGSQL = 3 - SQLITE = 4 - - def __init__(self): - """Simple constructor, doesnt really do anything""" - self.db = None - self.cursor = None - self.sql = {} - #end def __init__ - - def do_connect(self, config=None): - """Connects a database using information in config""" - if config is None: - raise FpdbError('Configuration not defined') - - self.config = config - self.settings = {} - self.settings['os'] = "linuxmac" if os.name != "nt" else "windows" - - db = config.get_db_parameters() - self.connect(backend=db['db-backend'], - host=db['db-host'], - database=db['db-databaseName'], - user=db['db-user'], - password=db['db-password']) - #end def do_connect - - def connect(self, backend=None, host=None, database=None, - user=None, password=None): - """Connects a database with the given parameters""" - if backend is None: - raise FpdbError('Database backend not defined') - self.backend = backend - self.host = host - self.user = user - self.password = password - self.database = database - createTables = False - - if backend == fpdb_db.MYSQL_INNODB: - import MySQLdb - if use_pool: - MySQLdb = pool.manage(MySQLdb, pool_size=5) - try: - self.db = MySQLdb.connect(host=host, user=user, passwd=password, db=database, use_unicode=True) - #TODO: Add port option - except MySQLdb.Error, ex: - if ex.args[0] == 1045: - raise FpdbMySQLAccessDenied(ex.args[0], ex.args[1]) - elif ex.args[0] == 2002 or ex.args[0] == 2003: # 2002 is no unix socket, 2003 is no tcp socket - raise FpdbMySQLNoDatabase(ex.args[0], ex.args[1]) - else: - print "*** WARNING UNKNOWN MYSQL ERROR", ex - elif backend == fpdb_db.PGSQL: - import psycopg2 - import psycopg2.extensions - if use_pool: - psycopg2 = pool.manage(psycopg2, pool_size=5) - psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) - # If DB connection is made over TCP, then the variables - # host, user and password are required - # For local domain-socket connections, only DB name is - # needed, and everything else is in fact undefined and/or - # flat out wrong - # sqlcoder: This database only connect failed in my windows setup?? - # Modifed it to try the 4 parameter style if the first connect fails - does this work everywhere? - connected = False - if self.host == "localhost" or self.host == "127.0.0.1": - try: - self.db = psycopg2.connect(database = database) - connected = True - except: - # direct connection failed so try user/pass/... version - pass - if not connected: - try: - self.db = psycopg2.connect(host = host, - user = user, - password = password, - database = database) - except Exception, ex: - if 'Connection refused' in ex.args[0]: - # meaning eg. db not running - raise FpdbPostgresqlNoDatabase(errmsg = ex.args[0]) - elif 'password authentication' in ex.args[0]: - raise FpdbPostgresqlAccessDenied(errmsg = ex.args[0]) - else: - msg = ex.args[0] - print msg - raise FpdbError(msg) - elif backend == fpdb_db.SQLITE: - logging.info("Connecting to SQLite: %(database)s" % {'database':database}) - import sqlite3 - if use_pool: - sqlite3 = pool.manage(sqlite3, pool_size=1) - else: - logging.warning("SQLite won't work well without 'sqlalchemy' installed.") - - if database != ":memory:": - if not os.path.isdir(self.config.dir_databases): - print "Creating directory: '%s'" % (self.config.dir_databases) - logging.info("Creating directory: '%s'" % (self.config.dir_databases)) - os.mkdir(self.config.dir_databases) - createTables = True # not needed? just test for no settings table - database = os.path.join(self.config.dir_databases, database) - logging.info(" sqlite db: " + database) - self.db = sqlite3.connect(database, detect_types=sqlite3.PARSE_DECLTYPES ) - sqlite3.register_converter("bool", lambda x: bool(int(x))) - sqlite3.register_adapter(bool, lambda x: "1" if x else "0") - self.db.create_function("floor", 1, math.floor) - tmp = sqlitemath() - self.db.create_function("mod", 2, tmp.mod) - if use_numpy: - self.db.create_aggregate("variance", 1, VARIANCE) - else: - logging.warning("Some database functions will not work without NumPy support") - else: - raise FpdbError("unrecognised database backend:"+backend) - - self.cursor = self.db.cursor() - # Set up query dictionary as early in the connection process as we can. - self.sql = FpdbSQLQueries.FpdbSQLQueries(self.get_backend_name()) - self.cursor.execute(self.sql.query['set tx level']) - self.wrongDbVersion = False - try: - self.cursor.execute("SELECT * FROM Settings") - settings = self.cursor.fetchone() - if settings[0] != DB_VERSION: - logging.error("outdated or too new database version (%s) - please recreate tables" - % (settings[0])) - self.wrongDbVersion = True - except:# _mysql_exceptions.ProgrammingError: - if database != ":memory:": print "failed to read settings table - please recreate tables" - self.wrongDbVersion = True - #end def connect - - def disconnect(self, due_to_error=False): - """Disconnects the DB""" - if due_to_error: - self.db.rollback() - else: - self.db.commit() - self.cursor.close() - self.db.close() - #end def disconnect - - def reconnect(self, due_to_error=False): - """Reconnects the DB""" - #print "started fpdb_db.reconnect" - self.disconnect(due_to_error) - self.connect(self.backend, self.host, self.database, self.user, self.password) - - def get_backend_name(self): - """Returns the name of the currently used backend""" - if self.backend==2: - return "MySQL InnoDB" - elif self.backend==3: - return "PostgreSQL" - elif self.backend==4: - return "SQLite" - else: - raise FpdbError("invalid backend") - #end def get_backend_name - - def get_db_info(self): - return (self.host, self.database, self.user, self.password) - #end def get_db_info #end class fpdb_db diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 76705839..fe66fcda 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -35,7 +35,6 @@ import gtk # fpdb/FreePokerTools modules -import fpdb_db import Database import Configuration import Exceptions