evilny0/fpdb
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

merge fpdb_db.py into Database.py

Browse Source
This commit is contained in:
sqlcoder 2010-01-26 23:54:04 +00:00
parent 47baee65f9
commit 11bc48e126
10 changed files with 215 additions and 241 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

206
pyfpdb/Database.py
View File

@ -39,11 +39,30 @@ import string
import re
import Queue
import codecs
import logging
import math
# pyGTK modules
# Other library modules
try:
import sqlalchemy.pool as pool
use_pool = True
except ImportError:
logging.info("Not using sqlalchemy connection pool.")
use_pool = False
try:
from numpy import var
use_numpy = True
except ImportError:
logging.info("Not using numpy to define variance in sqlite.")
use_numpy = False
# FreePokerTools modules
import fpdb_db
import Configuration
import SQL
import Card
@ -54,6 +73,27 @@ from Exceptions import *
log = Configuration.get_logger("logging.conf")
encoder = codecs.lookup('utf-8')
DB_VERSION = 118
# Variance created as sqlite has a bunch of undefined aggregate functions.
class VARIANCE:
def __init__(self):
self.store = []
def step(self, value):
self.store.append(value)
def finalize(self):
return float(var(self.store))
class sqlitemath:
def mod(self, a, b):
return a%b
class Database:
MYSQL_INNODB = 2
@ -190,7 +230,22 @@ class Database:
log.info("Creating Database instance, sql = %s" % sql)
self.config = c
self.__connected = False
self.fdb = fpdb_db.fpdb_db() # sets self.fdb.db self.fdb.cursor and self.fdb.sql
self.settings = {}
self.settings['os'] = "linuxmac" if os.name != "nt" else "windows"
db_params = c.get_db_parameters()
self.import_options = c.get_import_parameters()
self.backend = db_params['db-backend']
self.db_server = db_params['db-server']
self.database = db_params['db-databaseName']
self.host = db_params['db-host']
# where possible avoid creating new SQL instance by using the global one passed in
if sql is None:
self.sql = SQL.Sql(db_server = self.db_server)
else:
self.sql = sql
# connect to db
self.do_connect(c)
if self.backend == self.PGSQL:
@ -200,12 +255,6 @@ class Database:
#ISOLATION_LEVEL_SERIALIZABLE = 2
# where possible avoid creating new SQL instance by using the global one passed in
if sql is None:
self.sql = SQL.Sql(db_server = self.db_server)
else:
self.sql = sql
if self.backend == self.SQLITE and self.database == ':memory:' and self.wrongDbVersion:
log.info("sqlite/:memory: - creating")
self.recreate_tables()
@ -228,8 +277,6 @@ class Database:
self.h_date_ndays_ago = 'd000000' # date N days ago ('d' + YYMMDD) for hero
self.date_nhands_ago = {} # dates N hands ago per player - not used yet
self.cursor = self.fdb.cursor
self.saveActions = False if self.import_options['saveActions'] == False else True
self.connection.rollback() # make sure any locks taken so far are released
@ -240,14 +287,20 @@ class Database:
self.hud_style = style
def do_connect(self, c):
if c is None:
raise FpdbError('Configuration not defined')
db = c.get_db_parameters()
try:
self.fdb.do_connect(c)
self.connect(backend=db['db-backend'],
host=db['db-host'],
database=db['db-databaseName'],
user=db['db-user'],
password=db['db-password'])
except:
# error during connect
self.__connected = False
raise
self.connection = self.fdb.db
self.wrongDbVersion = self.fdb.wrongDbVersion
db_params = c.get_db_parameters()
self.import_options = c.get_import_parameters()
@ -257,11 +310,118 @@ class Database:
self.host = db_params['db-host']
self.__connected = True
def connect(self, backend=None, host=None, database=None,
user=None, password=None):
"""Connects a database with the given parameters"""
if backend is None:
raise FpdbError('Database backend not defined')
self.backend = backend
self.host = host
self.user = user
self.password = password
self.database = database
self.connection = None
self.cursor = None
if backend == Database.MYSQL_INNODB:
import MySQLdb
if use_pool:
MySQLdb = pool.manage(MySQLdb, pool_size=5)
try:
self.connection = MySQLdb.connect(host=host, user=user, passwd=password, db=database, use_unicode=True)
#TODO: Add port option
except MySQLdb.Error, ex:
if ex.args[0] == 1045:
raise FpdbMySQLAccessDenied(ex.args[0], ex.args[1])
elif ex.args[0] == 2002 or ex.args[0] == 2003: # 2002 is no unix socket, 2003 is no tcp socket
raise FpdbMySQLNoDatabase(ex.args[0], ex.args[1])
else:
print "*** WARNING UNKNOWN MYSQL ERROR", ex
elif backend == Database.PGSQL:
import psycopg2
import psycopg2.extensions
if use_pool:
psycopg2 = pool.manage(psycopg2, pool_size=5)
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
# If DB connection is made over TCP, then the variables
# host, user and password are required
# For local domain-socket connections, only DB name is
# needed, and everything else is in fact undefined and/or
# flat out wrong
# sqlcoder: This database only connect failed in my windows setup??
# Modifed it to try the 4 parameter style if the first connect fails - does this work everywhere?
connected = False
if self.host == "localhost" or self.host == "127.0.0.1":
try:
self.connection = psycopg2.connect(database = database)
connected = True
except:
# direct connection failed so try user/pass/... version
pass
if not connected:
try:
self.connection = psycopg2.connect(host = host,
user = user,
password = password,
database = database)
except Exception, ex:
if 'Connection refused' in ex.args[0]:
# meaning eg. db not running
raise FpdbPostgresqlNoDatabase(errmsg = ex.args[0])
elif 'password authentication' in ex.args[0]:
raise FpdbPostgresqlAccessDenied(errmsg = ex.args[0])
else:
msg = ex.args[0]
print msg
raise FpdbError(msg)
elif backend == Database.SQLITE:
logging.info("Connecting to SQLite: %(database)s" % {'database':database})
import sqlite3
if use_pool:
sqlite3 = pool.manage(sqlite3, pool_size=1)
else:
logging.warning("SQLite won't work well without 'sqlalchemy' installed.")
if database != ":memory:":
if not os.path.isdir(self.config.dir_databases):
print "Creating directory: '%s'" % (self.config.dir_databases)
logging.info("Creating directory: '%s'" % (self.config.dir_databases))
os.mkdir(self.config.dir_databases)
database = os.path.join(self.config.dir_databases, database)
logging.info(" sqlite db: " + database)
self.connection = sqlite3.connect(database, detect_types=sqlite3.PARSE_DECLTYPES )
sqlite3.register_converter("bool", lambda x: bool(int(x)))
sqlite3.register_adapter(bool, lambda x: "1" if x else "0")
self.connection.create_function("floor", 1, math.floor)
tmp = sqlitemath()
self.connection.create_function("mod", 2, tmp.mod)
if use_numpy:
self.connection.create_aggregate("variance", 1, VARIANCE)
else:
logging.warning("Some database functions will not work without NumPy support")
else:
raise FpdbError("unrecognised database backend:"+backend)
self.cursor = self.connection.cursor()
self.cursor.execute(self.sql.query['set tx level'])
self.wrongDbVersion = False
try:
self.cursor.execute("SELECT * FROM Settings")
settings = self.cursor.fetchone()
if settings[0] != DB_VERSION:
logging.error("outdated or too new database version (%s) - please recreate tables"
% (settings[0]))
self.wrongDbVersion = True
except:# _mysql_exceptions.ProgrammingError:
if database != ":memory:": print "failed to read settings table - please recreate tables"
self.wrongDbVersion = True
#end def connect
def commit(self):
self.fdb.db.commit()
self.connection.commit()
def rollback(self):
self.fdb.db.rollback()
self.connection.rollback()
def connected(self):
return self.__connected
@ -274,11 +434,18 @@ class Database:
def disconnect(self, due_to_error=False):
"""Disconnects the DB (rolls back if param is true, otherwise commits"""
self.fdb.disconnect(due_to_error)
if due_to_error:
self.connection.rollback()
else:
self.connection.commit()
self.cursor.close()
self.connection.close()
def reconnect(self, due_to_error=False):
"""Reconnects the DB"""
self.fdb.reconnect(due_to_error=False)
#print "started reconnect"
self.disconnect(due_to_error)
self.connect(self.backend, self.host, self.database, self.user, self.password)
def get_backend_name(self):
"""Returns the name of the currently used backend"""
@ -291,6 +458,9 @@ class Database:
else:
raise FpdbError("invalid backend")
def get_db_info(self):
return (self.host, self.database, self.user, self.password)
def get_table_name(self, hand_id):
c = self.connection.cursor()
c.execute(self.sql.query['get_table_name'], (hand_id, ))
@ -1267,7 +1437,7 @@ class Database:
try:
self.get_cursor().execute(self.sql.query['lockForInsert'])
except:
print "Error during fdb.lock_for_insert:", str(sys.exc_value)
print "Error during lock_for_insert:", str(sys.exc_value)
#end def lock_for_insert
###########################

8
pyfpdb/Filters.py
View File

@ -27,8 +27,8 @@ import gobject
#import pokereval
import Configuration
import fpdb_db
import FpdbSQLQueries
import Database
import SQL
import Charset
class Filters(threading.Thread):
@ -790,10 +790,10 @@ def main(argv=None):
config = Configuration.Config()
db = None
db = fpdb_db.fpdb_db()
db = Database.Database()
db.do_connect(config)
qdict = FpdbSQLQueries.FpdbSQLQueries(db.get_backend_name())
qdict = SQL.SQL(db.get_backend_name())
i = Filters(db, config, qdict)
main_window = gtk.Window()

1
pyfpdb/GuiAutoImport.py
View File

@ -300,7 +300,6 @@ if __name__== "__main__":
(options, argv) = parser.parse_args()
config = Configuration.Config()
# db = fpdb_db.fpdb_db()
settings = {}
settings['minPrint'] = options.minPrint

1
pyfpdb/GuiPlayerStats.py
View File

@ -27,7 +27,6 @@ from time import time, strftime
import Card
import fpdb_import
import Database
import fpdb_db
import Filters
import Charset

2
pyfpdb/Hand.py
View File

@ -205,7 +205,7 @@ dealt whether they were seen in a 'dealt to' line
def insert(self, db):
""" Function to insert Hand into database
Should not commit, and do minimal selects. Callers may want to cache commits
db: a connected fpdb_db object"""
db: a connected Database object"""
self.stats.getStats(self)

21
pyfpdb/SQL.py
View File

@ -58,6 +58,27 @@ class Sql:
self.query['drop_table'] = """DROP TABLE IF EXISTS """
##################################################################
# Set transaction isolation level
##################################################################
if db_server == 'mysql' or db_server == 'postgresql':
self.query['set tx level'] = """SET SESSION TRANSACTION
ISOLATION LEVEL READ COMMITTED"""
elif db_server == 'sqlite':
self.query['set tx level'] = """ """
################################
# Select basic info
################################
self.query['getSiteId'] = """SELECT id from Sites where name = %s"""
self.query['getGames'] = """SELECT DISTINCT category from Gametypes"""
self.query['getLimits'] = """SELECT DISTINCT bigBlind from Gametypes ORDER by bigBlind DESC"""
################################
# Create Settings
################################

2
pyfpdb/Tourney.py
View File

@ -185,7 +185,7 @@ class Tourney(object):
def old_insert_from_Hand(self, db):
""" Function to insert Hand into database
Should not commit, and do minimal selects. Callers may want to cache commits
db: a connected fpdb_db object"""
db: a connected Database object"""
# TODO:
# Players - base playerid and siteid tuple
sqlids = db.getSqlPlayerIDs([p[1] for p in self.players], self.siteId)

1
pyfpdb/fpdb.py
View File

@ -112,7 +112,6 @@ import GuiGraphViewer
import GuiSessionViewer
import SQL
import Database
import FpdbSQLQueries
import Configuration
import Exceptions

213
pyfpdb/fpdb_db.py
View File

@ -16,219 +16,6 @@
#In the "official" distribution you can find the license in
#agpl-3.0.txt in the docs folder of the package.
import os
import re
import sys
import logging
import math
from time import time, strftime
from Exceptions import *
try:
import sqlalchemy.pool as pool
use_pool = True
except ImportError:
logging.info("Not using sqlalchemy connection pool.")
use_pool = False
try:
from numpy import var
use_numpy = True
except ImportError:
logging.info("Not using numpy to define variance in sqlite.")
use_numpy = False
import FpdbSQLQueries
import Configuration
DB_VERSION = 118
# Variance created as sqlite has a bunch of undefined aggregate functions.
class VARIANCE:
def __init__(self):
self.store = []
def step(self, value):
self.store.append(value)
def finalize(self):
return float(var(self.store))
class sqlitemath:
def mod(self, a, b):
return a%b
class fpdb_db:
MYSQL_INNODB = 2
PGSQL = 3
SQLITE = 4
def __init__(self):
"""Simple constructor, doesnt really do anything"""
self.db = None
self.cursor = None
self.sql = {}
#end def __init__
def do_connect(self, config=None):
"""Connects a database using information in config"""
if config is None:
raise FpdbError('Configuration not defined')
self.config = config
self.settings = {}
self.settings['os'] = "linuxmac" if os.name != "nt" else "windows"
db = config.get_db_parameters()
self.connect(backend=db['db-backend'],
host=db['db-host'],
database=db['db-databaseName'],
user=db['db-user'],
password=db['db-password'])
#end def do_connect
def connect(self, backend=None, host=None, database=None,
user=None, password=None):
"""Connects a database with the given parameters"""
if backend is None:
raise FpdbError('Database backend not defined')
self.backend = backend
self.host = host
self.user = user
self.password = password
self.database = database
createTables = False
if backend == fpdb_db.MYSQL_INNODB:
import MySQLdb
if use_pool:
MySQLdb = pool.manage(MySQLdb, pool_size=5)
try:
self.db = MySQLdb.connect(host=host, user=user, passwd=password, db=database, use_unicode=True)
#TODO: Add port option
except MySQLdb.Error, ex:
if ex.args[0] == 1045:
raise FpdbMySQLAccessDenied(ex.args[0], ex.args[1])
elif ex.args[0] == 2002 or ex.args[0] == 2003: # 2002 is no unix socket, 2003 is no tcp socket
raise FpdbMySQLNoDatabase(ex.args[0], ex.args[1])
else:
print "*** WARNING UNKNOWN MYSQL ERROR", ex
elif backend == fpdb_db.PGSQL:
import psycopg2
import psycopg2.extensions
if use_pool:
psycopg2 = pool.manage(psycopg2, pool_size=5)
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
# If DB connection is made over TCP, then the variables
# host, user and password are required
# For local domain-socket connections, only DB name is
# needed, and everything else is in fact undefined and/or
# flat out wrong
# sqlcoder: This database only connect failed in my windows setup??
# Modifed it to try the 4 parameter style if the first connect fails - does this work everywhere?
connected = False
if self.host == "localhost" or self.host == "127.0.0.1":
try:
self.db = psycopg2.connect(database = database)
connected = True
except:
# direct connection failed so try user/pass/... version
pass
if not connected:
try:
self.db = psycopg2.connect(host = host,
user = user,
password = password,
database = database)
except Exception, ex:
if 'Connection refused' in ex.args[0]:
# meaning eg. db not running
raise FpdbPostgresqlNoDatabase(errmsg = ex.args[0])
elif 'password authentication' in ex.args[0]:
raise FpdbPostgresqlAccessDenied(errmsg = ex.args[0])
else:
msg = ex.args[0]
print msg
raise FpdbError(msg)
elif backend == fpdb_db.SQLITE:
logging.info("Connecting to SQLite: %(database)s" % {'database':database})
import sqlite3
if use_pool:
sqlite3 = pool.manage(sqlite3, pool_size=1)
else:
logging.warning("SQLite won't work well without 'sqlalchemy' installed.")
if database != ":memory:":
if not os.path.isdir(self.config.dir_databases):
print "Creating directory: '%s'" % (self.config.dir_databases)
logging.info("Creating directory: '%s'" % (self.config.dir_databases))
os.mkdir(self.config.dir_databases)
createTables = True # not needed? just test for no settings table
database = os.path.join(self.config.dir_databases, database)
logging.info(" sqlite db: " + database)
self.db = sqlite3.connect(database, detect_types=sqlite3.PARSE_DECLTYPES )
sqlite3.register_converter("bool", lambda x: bool(int(x)))
sqlite3.register_adapter(bool, lambda x: "1" if x else "0")
self.db.create_function("floor", 1, math.floor)
tmp = sqlitemath()
self.db.create_function("mod", 2, tmp.mod)
if use_numpy:
self.db.create_aggregate("variance", 1, VARIANCE)
else:
logging.warning("Some database functions will not work without NumPy support")
else:
raise FpdbError("unrecognised database backend:"+backend)
self.cursor = self.db.cursor()
# Set up query dictionary as early in the connection process as we can.
self.sql = FpdbSQLQueries.FpdbSQLQueries(self.get_backend_name())
self.cursor.execute(self.sql.query['set tx level'])
self.wrongDbVersion = False
try:
self.cursor.execute("SELECT * FROM Settings")
settings = self.cursor.fetchone()
if settings[0] != DB_VERSION:
logging.error("outdated or too new database version (%s) - please recreate tables"
% (settings[0]))
self.wrongDbVersion = True
except:# _mysql_exceptions.ProgrammingError:
if database != ":memory:": print "failed to read settings table - please recreate tables"
self.wrongDbVersion = True
#end def connect
def disconnect(self, due_to_error=False):
"""Disconnects the DB"""
if due_to_error:
self.db.rollback()
else:
self.db.commit()
self.cursor.close()
self.db.close()
#end def disconnect
def reconnect(self, due_to_error=False):
"""Reconnects the DB"""
#print "started fpdb_db.reconnect"
self.disconnect(due_to_error)
self.connect(self.backend, self.host, self.database, self.user, self.password)
def get_backend_name(self):
"""Returns the name of the currently used backend"""
if self.backend==2:
return "MySQL InnoDB"
elif self.backend==3:
return "PostgreSQL"
elif self.backend==4:
return "SQLite"
else:
raise FpdbError("invalid backend")
#end def get_backend_name
def get_db_info(self):
return (self.host, self.database, self.user, self.password)
#end def get_db_info
#end class fpdb_db

1
pyfpdb/fpdb_import.py
View File

@ -35,7 +35,6 @@ import gtk
# fpdb/FreePokerTools modules
import fpdb_db
import Database
import Configuration
import Exceptions