Merge branch 'master' of git://git.assembla.com/fpdb-sql

Conflicts:
	pyfpdb/Configuration.py
	pyfpdb/Database.py
This commit is contained in:
Eratosthenes 2010-01-26 20:13:21 -05:00
commit 1ec6a36ece
13 changed files with 256 additions and 238 deletions

View File

@ -56,8 +56,7 @@ def get_exec_path():
if hasattr(sys, "frozen"): # compiled by py2exe
return os.path.dirname(sys.executable)
else:
pathname = os.path.dirname(sys.argv[0])
return os.path.abspath(pathname)
return sys.path[0]
def get_config(file_name, fallback = True):
"""Looks in exec dir and in self.default_config_path for a config file."""
@ -82,7 +81,10 @@ def get_config(file_name, fallback = True):
log.error("No %s found, using %s.example.\n" % (file_name, file_name) )
except:
print "No %s found, cannot fall back. Exiting.\n" % file_name
log.critical("No %s found, cannot fall back. Exiting.\n" % file_name)
sys.exit()
else:
print "No %s found, cannot fall back. Exiting.\n" % file_name
sys.stderr.write("No %s found, cannot fall back. Exiting.\n" % file_name)
sys.exit()
return file_name
@ -429,7 +431,7 @@ class Config:
log.error("Specified configuration file %s not found. Using defaults." % (file))
file = None
if file is None: file = get_config("HUD_config.xml")
if file is None: file = get_config("HUD_config.xml", True)
# Parse even if there was no real config file found and we are using the example
# If using the example, we'll edit it later
@ -446,6 +448,8 @@ class Config:
self.doc = doc
self.file = file
self.dir = os.path.dirname(self.file)
self.dir_databases = os.path.join(self.dir, 'database')
self.supported_sites = {}
self.supported_games = {}
self.supported_databases = {} # databaseName --> Database instance

View File

@ -39,11 +39,30 @@ import string
import re
import Queue
import codecs
import logging
import math
# pyGTK modules
# Other library modules
try:
import sqlalchemy.pool as pool
use_pool = True
except ImportError:
logging.info("Not using sqlalchemy connection pool.")
use_pool = False
try:
from numpy import var
use_numpy = True
except ImportError:
logging.info("Not using numpy to define variance in sqlite.")
use_numpy = False
# FreePokerTools modules
import fpdb_db
import Configuration
import SQL
import Card
@ -55,6 +74,27 @@ log = Configuration.get_logger("logging.conf", config = "db")
log.debug("db logger initialized.")
encoder = codecs.lookup('utf-8')
DB_VERSION = 119
# Variance created as sqlite has a bunch of undefined aggregate functions.
class VARIANCE:
def __init__(self):
self.store = []
def step(self, value):
self.store.append(value)
def finalize(self):
return float(var(self.store))
class sqlitemath:
def mod(self, a, b):
return a%b
class Database:
MYSQL_INNODB = 2
@ -191,7 +231,22 @@ class Database:
log.info("Creating Database instance, sql = %s" % sql)
self.config = c
self.__connected = False
self.fdb = fpdb_db.fpdb_db() # sets self.fdb.db self.fdb.cursor and self.fdb.sql
self.settings = {}
self.settings['os'] = "linuxmac" if os.name != "nt" else "windows"
db_params = c.get_db_parameters()
self.import_options = c.get_import_parameters()
self.backend = db_params['db-backend']
self.db_server = db_params['db-server']
self.database = db_params['db-databaseName']
self.host = db_params['db-host']
# where possible avoid creating new SQL instance by using the global one passed in
if sql is None:
self.sql = SQL.Sql(db_server = self.db_server)
else:
self.sql = sql
# connect to db
self.do_connect(c)
print "connection =", self.connection
if self.backend == self.PGSQL:
@ -200,12 +255,6 @@ class Database:
#ISOLATION_LEVEL_READ_COMMITTED = 1
#ISOLATION_LEVEL_SERIALIZABLE = 2
# where possible avoid creating new SQL instance by using the global one passed in
if sql is None:
self.sql = SQL.Sql(db_server = self.db_server)
else:
self.sql = sql
if self.backend == self.SQLITE and self.database == ':memory:' and self.wrongDbVersion:
log.info("sqlite/:memory: - creating")
self.recreate_tables()
@ -228,8 +277,6 @@ class Database:
self.h_date_ndays_ago = 'd000000' # date N days ago ('d' + YYMMDD) for hero
self.date_nhands_ago = {} # dates N hands ago per player - not used yet
self.cursor = self.fdb.cursor
self.saveActions = False if self.import_options['saveActions'] == False else True
self.connection.rollback() # make sure any locks taken so far are released
@ -240,14 +287,20 @@ class Database:
self.hud_style = style
def do_connect(self, c):
if c is None:
raise FpdbError('Configuration not defined')
db = c.get_db_parameters()
try:
self.fdb.do_connect(c)
self.connect(backend=db['db-backend'],
host=db['db-host'],
database=db['db-databaseName'],
user=db['db-user'],
password=db['db-password'])
except:
# error during connect
self.__connected = False
raise
self.connection = self.fdb.db
self.wrongDbVersion = self.fdb.wrongDbVersion
db_params = c.get_db_parameters()
self.import_options = c.get_import_parameters()
@ -257,11 +310,137 @@ class Database:
self.host = db_params['db-host']
self.__connected = True
def connect(self, backend=None, host=None, database=None,
user=None, password=None):
"""Connects a database with the given parameters"""
if backend is None:
raise FpdbError('Database backend not defined')
self.backend = backend
self.host = host
self.user = user
self.password = password
self.database = database
self.connection = None
self.cursor = None
if backend == Database.MYSQL_INNODB:
import MySQLdb
if use_pool:
MySQLdb = pool.manage(MySQLdb, pool_size=5)
try:
self.connection = MySQLdb.connect(host=host, user=user, passwd=password, db=database, use_unicode=True)
#TODO: Add port option
except MySQLdb.Error, ex:
if ex.args[0] == 1045:
raise FpdbMySQLAccessDenied(ex.args[0], ex.args[1])
elif ex.args[0] == 2002 or ex.args[0] == 2003: # 2002 is no unix socket, 2003 is no tcp socket
raise FpdbMySQLNoDatabase(ex.args[0], ex.args[1])
else:
print "*** WARNING UNKNOWN MYSQL ERROR", ex
elif backend == Database.PGSQL:
import psycopg2
import psycopg2.extensions
if use_pool:
psycopg2 = pool.manage(psycopg2, pool_size=5)
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
# If DB connection is made over TCP, then the variables
# host, user and password are required
# For local domain-socket connections, only DB name is
# needed, and everything else is in fact undefined and/or
# flat out wrong
# sqlcoder: This database only connect failed in my windows setup??
# Modifed it to try the 4 parameter style if the first connect fails - does this work everywhere?
connected = False
if self.host == "localhost" or self.host == "127.0.0.1":
try:
self.connection = psycopg2.connect(database = database)
connected = True
except:
# direct connection failed so try user/pass/... version
pass
if not connected:
try:
self.connection = psycopg2.connect(host = host,
user = user,
password = password,
database = database)
except Exception, ex:
if 'Connection refused' in ex.args[0]:
# meaning eg. db not running
raise FpdbPostgresqlNoDatabase(errmsg = ex.args[0])
elif 'password authentication' in ex.args[0]:
raise FpdbPostgresqlAccessDenied(errmsg = ex.args[0])
else:
msg = ex.args[0]
print msg
raise FpdbError(msg)
elif backend == Database.SQLITE:
logging.info("Connecting to SQLite: %(database)s" % {'database':database})
import sqlite3
if use_pool:
sqlite3 = pool.manage(sqlite3, pool_size=1)
else:
logging.warning("SQLite won't work well without 'sqlalchemy' installed.")
if database != ":memory:":
if not os.path.isdir(self.config.dir_databases):
print "Creating directory: '%s'" % (self.config.dir_databases)
logging.info("Creating directory: '%s'" % (self.config.dir_databases))
os.mkdir(self.config.dir_databases)
database = os.path.join(self.config.dir_databases, database)
logging.info(" sqlite db: " + database)
self.connection = sqlite3.connect(database, detect_types=sqlite3.PARSE_DECLTYPES )
sqlite3.register_converter("bool", lambda x: bool(int(x)))
sqlite3.register_adapter(bool, lambda x: "1" if x else "0")
self.connection.create_function("floor", 1, math.floor)
tmp = sqlitemath()
self.connection.create_function("mod", 2, tmp.mod)
if use_numpy:
self.connection.create_aggregate("variance", 1, VARIANCE)
else:
logging.warning("Some database functions will not work without NumPy support")
else:
raise FpdbError("unrecognised database backend:"+backend)
self.cursor = self.connection.cursor()
self.cursor.execute(self.sql.query['set tx level'])
self.check_version(database=database, create=True)
def check_version(self, database, create):
self.wrongDbVersion = False
try:
self.cursor.execute("SELECT * FROM Settings")
settings = self.cursor.fetchone()
if settings[0] != DB_VERSION:
logging.error("outdated or too new database version (%s) - please recreate tables"
% (settings[0]))
self.wrongDbVersion = True
except:# _mysql_exceptions.ProgrammingError:
if database != ":memory:":
if create:
print "Failed to read settings table - recreating tables"
log.info("failed to read settings table - recreating tables")
self.recreate_tables()
self.check_version(database=database, create=False)
if not self.wrongDbVersion:
msg = "Edit your screen_name and hand history path in the supported_sites "\
+"section of the \nPreferences window (Main menu) before trying to import hands"
print "\n%s" % msg
log.warning(msg)
else:
print "Failed to read settings table - please recreate tables"
log.info("failed to read settings table - please recreate tables")
self.wrongDbVersion = True
else:
self.wrongDbVersion = True
#end def connect
def commit(self):
self.fdb.db.commit()
self.connection.commit()
def rollback(self):
self.fdb.db.rollback()
self.connection.rollback()
def connected(self):
return self.__connected
@ -274,11 +453,18 @@ class Database:
def disconnect(self, due_to_error=False):
"""Disconnects the DB (rolls back if param is true, otherwise commits"""
self.fdb.disconnect(due_to_error)
if due_to_error:
self.connection.rollback()
else:
self.connection.commit()
self.cursor.close()
self.connection.close()
def reconnect(self, due_to_error=False):
"""Reconnects the DB"""
self.fdb.reconnect(due_to_error=False)
#print "started reconnect"
self.disconnect(due_to_error)
self.connect(self.backend, self.host, self.database, self.user, self.password)
def get_backend_name(self):
"""Returns the name of the currently used backend"""
@ -291,6 +477,9 @@ class Database:
else:
raise FpdbError("invalid backend")
def get_db_info(self):
return (self.host, self.database, self.user, self.password)
def get_table_name(self, hand_id):
c = self.connection.cursor()
c.execute(self.sql.query['get_table_name'], (hand_id, ))
@ -846,6 +1035,7 @@ class Database:
self.create_tables()
self.createAllIndexes()
self.commit()
print "Finished recreating tables"
log.info("Finished recreating tables")
#end def recreate_tables
@ -1111,7 +1301,7 @@ class Database:
def fillDefaultData(self):
c = self.get_cursor()
c.execute("INSERT INTO Settings (version) VALUES (118);")
c.execute("INSERT INTO Settings (version) VALUES (%s);" % (DB_VERSION))
c.execute("INSERT INTO Sites (name,currency) VALUES ('Full Tilt Poker', 'USD')")
c.execute("INSERT INTO Sites (name,currency) VALUES ('PokerStars', 'USD')")
c.execute("INSERT INTO Sites (name,currency) VALUES ('Everleaf', 'USD')")
@ -1267,7 +1457,7 @@ class Database:
try:
self.get_cursor().execute(self.sql.query['lockForInsert'])
except:
print "Error during fdb.lock_for_insert:", str(sys.exc_value)
print "Error during lock_for_insert:", str(sys.exc_value)
#end def lock_for_insert
###########################
@ -1286,6 +1476,7 @@ class Database:
p['tableName'],
p['gameTypeId'],
p['siteHandNo'],
0, # tourneyId: 0 means not a tourney hand
p['handStart'],
datetime.today(), #importtime
p['seats'],

View File

@ -27,8 +27,8 @@ import gobject
#import pokereval
import Configuration
import fpdb_db
import FpdbSQLQueries
import Database
import SQL
import Charset
class Filters(threading.Thread):
@ -790,10 +790,10 @@ def main(argv=None):
config = Configuration.Config()
db = None
db = fpdb_db.fpdb_db()
db = Database.Database()
db.do_connect(config)
qdict = FpdbSQLQueries.FpdbSQLQueries(db.get_backend_name())
qdict = SQL.SQL(db.get_backend_name())
i = Filters(db, config, qdict)
main_window = gtk.Window()

View File

@ -300,7 +300,6 @@ if __name__== "__main__":
(options, argv) = parser.parse_args()
config = Configuration.Config()
# db = fpdb_db.fpdb_db()
settings = {}
settings['minPrint'] = options.minPrint

View File

@ -27,7 +27,6 @@ from time import time, strftime
import Card
import fpdb_import
import Database
import fpdb_db
import Filters
import Charset

View File

@ -588,8 +588,8 @@ Left-Drag to Move"
</hhcs>
<supported_databases>
<database db_name="fpdb" db_server="mysql" db_ip="localhost" db_user="fpdb" db_pass="YOUR MYSQL PASSWORD"></database>
<!-- <database db_ip="localhost" db_name="fpdb" db_pass="fpdb" db_server="sqlite" db_user="fpdb"/> -->
<!-- <database db_name="fpdb" db_server="mysql" db_ip="localhost" db_user="fpdb" db_pass="YOUR MYSQL PASSWORD"></database> -->
<database db_ip="localhost" db_server="sqlite" db_name="fpdb.db3" db_user="fpdb" db_pass="fpdb"/>
</supported_databases>
</FreePokerToolsConfig>

View File

@ -205,7 +205,7 @@ dealt whether they were seen in a 'dealt to' line
def insert(self, db):
""" Function to insert Hand into database
Should not commit, and do minimal selects. Callers may want to cache commits
db: a connected fpdb_db object"""
db: a connected Database object"""
self.stats.getStats(self)

View File

@ -58,6 +58,27 @@ class Sql:
self.query['drop_table'] = """DROP TABLE IF EXISTS """
##################################################################
# Set transaction isolation level
##################################################################
if db_server == 'mysql' or db_server == 'postgresql':
self.query['set tx level'] = """SET SESSION TRANSACTION
ISOLATION LEVEL READ COMMITTED"""
elif db_server == 'sqlite':
self.query['set tx level'] = """ """
################################
# Select basic info
################################
self.query['getSiteId'] = """SELECT id from Sites where name = %s"""
self.query['getGames'] = """SELECT DISTINCT category from Gametypes"""
self.query['getLimits'] = """SELECT DISTINCT bigBlind from Gametypes ORDER by bigBlind DESC"""
################################
# Create Settings
################################
@ -214,6 +235,7 @@ class Sql:
id BIGINT UNSIGNED AUTO_INCREMENT NOT NULL, PRIMARY KEY (id),
tableName VARCHAR(22) NOT NULL,
siteHandNo BIGINT NOT NULL,
tourneyId INT UNSIGNED NOT NULL,
gametypeId SMALLINT UNSIGNED NOT NULL, FOREIGN KEY (gametypeId) REFERENCES Gametypes(id),
handStart DATETIME NOT NULL,
importTime DATETIME NOT NULL,
@ -249,6 +271,7 @@ class Sql:
id BIGSERIAL, PRIMARY KEY (id),
tableName VARCHAR(22) NOT NULL,
siteHandNo BIGINT NOT NULL,
tourneyId INT NOT NULL,
gametypeId INT NOT NULL, FOREIGN KEY (gametypeId) REFERENCES Gametypes(id),
handStart timestamp without time zone NOT NULL,
importTime timestamp without time zone NOT NULL,
@ -283,6 +306,7 @@ class Sql:
id INTEGER PRIMARY KEY,
tableName TEXT(22) NOT NULL,
siteHandNo INT NOT NULL,
tourneyId INT NOT NULL,
gametypeId INT NOT NULL,
handStart REAL NOT NULL,
importTime REAL NOT NULL,
@ -3437,6 +3461,7 @@ class Sql:
tablename,
gametypeid,
sitehandno,
tourneyId,
handstart,
importtime,
seats,
@ -3467,7 +3492,7 @@ class Sql:
VALUES
(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
%s, %s, %s, %s, %s, %s, %s, %s, %s)"""
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"""
self.query['store_hands_players'] = """INSERT INTO HandsPlayers (

View File

@ -185,7 +185,7 @@ class Tourney(object):
def old_insert_from_Hand(self, db):
""" Function to insert Hand into database
Should not commit, and do minimal selects. Callers may want to cache commits
db: a connected fpdb_db object"""
db: a connected Database object"""
# TODO:
# Players - base playerid and siteid tuple
sqlids = db.getSqlPlayerIDs([p[1] for p in self.players], self.siteId)

View File

@ -112,7 +112,6 @@ import GuiGraphViewer
import GuiSessionViewer
import SQL
import Database
import FpdbSQLQueries
import Configuration
import Exceptions

View File

@ -16,207 +16,6 @@
#In the "official" distribution you can find the license in
#agpl-3.0.txt in the docs folder of the package.
import os
import re
import sys
import logging
import math
from time import time, strftime
from Exceptions import *
try:
import sqlalchemy.pool as pool
use_pool = True
except ImportError:
logging.info("Not using sqlalchemy connection pool.")
use_pool = False
try:
from numpy import var
use_numpy = True
except ImportError:
logging.info("Not using numpy to define variance in sqlite.")
use_numpy = False
import FpdbSQLQueries
import Configuration
# Variance created as sqlite has a bunch of undefined aggregate functions.
class VARIANCE:
def __init__(self):
self.store = []
def step(self, value):
self.store.append(value)
def finalize(self):
return float(var(self.store))
class sqlitemath:
def mod(self, a, b):
return a%b
class fpdb_db:
MYSQL_INNODB = 2
PGSQL = 3
SQLITE = 4
def __init__(self):
"""Simple constructor, doesnt really do anything"""
self.db = None
self.cursor = None
self.sql = {}
#end def __init__
def do_connect(self, config=None):
"""Connects a database using information in config"""
if config is None:
raise FpdbError('Configuration not defined')
self.settings = {}
self.settings['os'] = "linuxmac" if os.name != "nt" else "windows"
db = config.get_db_parameters()
self.connect(backend=db['db-backend'],
host=db['db-host'],
database=db['db-databaseName'],
user=db['db-user'],
password=db['db-password'])
#end def do_connect
def connect(self, backend=None, host=None, database=None,
user=None, password=None):
"""Connects a database with the given parameters"""
if backend is None:
raise FpdbError('Database backend not defined')
self.backend = backend
self.host = host
self.user = user
self.password = password
self.database = database
if backend == fpdb_db.MYSQL_INNODB:
import MySQLdb
if use_pool:
MySQLdb = pool.manage(MySQLdb, pool_size=5)
try:
self.db = MySQLdb.connect(host=host, user=user, passwd=password, db=database, use_unicode=True)
#TODO: Add port option
except MySQLdb.Error, ex:
if ex.args[0] == 1045:
raise FpdbMySQLAccessDenied(ex.args[0], ex.args[1])
elif ex.args[0] == 2002 or ex.args[0] == 2003: # 2002 is no unix socket, 2003 is no tcp socket
raise FpdbMySQLNoDatabase(ex.args[0], ex.args[1])
else:
print "*** WARNING UNKNOWN MYSQL ERROR", ex
elif backend == fpdb_db.PGSQL:
import psycopg2
import psycopg2.extensions
if use_pool:
psycopg2 = pool.manage(psycopg2, pool_size=5)
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
# If DB connection is made over TCP, then the variables
# host, user and password are required
# For local domain-socket connections, only DB name is
# needed, and everything else is in fact undefined and/or
# flat out wrong
# sqlcoder: This database only connect failed in my windows setup??
# Modifed it to try the 4 parameter style if the first connect fails - does this work everywhere?
connected = False
if self.host == "localhost" or self.host == "127.0.0.1":
try:
self.db = psycopg2.connect(database = database)
connected = True
except:
# direct connection failed so try user/pass/... version
pass
if not connected:
try:
self.db = psycopg2.connect(host = host,
user = user,
password = password,
database = database)
except Exception, ex:
if 'Connection refused' in ex.args[0]:
# meaning eg. db not running
raise FpdbPostgresqlNoDatabase(errmsg = ex.args[0])
elif 'password authentication' in ex.args[0]:
raise FpdbPostgresqlAccessDenied(errmsg = ex.args[0])
else:
msg = ex.args[0]
print msg
raise FpdbError(msg)
elif backend == fpdb_db.SQLITE:
logging.info("Connecting to SQLite:%(database)s" % {'database':database})
import sqlite3
if use_pool:
sqlite3 = pool.manage(sqlite3, pool_size=1)
else:
logging.warning("SQLite won't work well without 'sqlalchemy' installed.")
if not os.path.isdir(Configuration.DIR_DATABASES) and not database == ":memory:":
print "Creating directory: '%s'" % (Configuration.DIR_DATABASES)
os.mkdir(Configuration.DIR_DATABASES)
database = os.path.join(Configuration.DIR_DATABASES, database)
self.db = sqlite3.connect(database, detect_types=sqlite3.PARSE_DECLTYPES )
sqlite3.register_converter("bool", lambda x: bool(int(x)))
sqlite3.register_adapter(bool, lambda x: "1" if x else "0")
self.db.create_function("floor", 1, math.floor)
tmp = sqlitemath()
self.db.create_function("mod", 2, tmp.mod)
if use_numpy:
self.db.create_aggregate("variance", 1, VARIANCE)
else:
logging.warning("Some database functions will not work without NumPy support")
else:
raise FpdbError("unrecognised database backend:"+backend)
self.cursor = self.db.cursor()
# Set up query dictionary as early in the connection process as we can.
self.sql = FpdbSQLQueries.FpdbSQLQueries(self.get_backend_name())
self.cursor.execute(self.sql.query['set tx level'])
self.wrongDbVersion = False
try:
self.cursor.execute("SELECT * FROM Settings")
settings = self.cursor.fetchone()
if settings[0] != 118:
print "outdated or too new database version - please recreate tables"
self.wrongDbVersion = True
except:# _mysql_exceptions.ProgrammingError:
if database != ":memory:": print "failed to read settings table - please recreate tables"
self.wrongDbVersion = True
#end def connect
def disconnect(self, due_to_error=False):
"""Disconnects the DB"""
if due_to_error:
self.db.rollback()
else:
self.db.commit()
self.cursor.close()
self.db.close()
#end def disconnect
def reconnect(self, due_to_error=False):
"""Reconnects the DB"""
#print "started fpdb_db.reconnect"
self.disconnect(due_to_error)
self.connect(self.backend, self.host, self.database, self.user, self.password)
def get_backend_name(self):
"""Returns the name of the currently used backend"""
if self.backend==2:
return "MySQL InnoDB"
elif self.backend==3:
return "PostgreSQL"
elif self.backend==4:
return "SQLite"
else:
raise FpdbError("invalid backend")
#end def get_backend_name
def get_db_info(self):
return (self.host, self.database, self.user, self.password)
#end def get_db_info
#end class fpdb_db

View File

@ -35,7 +35,6 @@ import gtk
# fpdb/FreePokerTools modules
import fpdb_db
import Database
import Configuration
import Exceptions

View File

@ -19,8 +19,11 @@
import os
import sys
# sys.path[0] holds the dir run_fpdb.py was in
# sys.path[0] holds the directory run_fpdb.py is in
sys.path[0] = sys.path[0]+os.sep+"pyfpdb"
os.chdir(sys.path[0])
#print "sys.path[0] =", sys.path[0], "cwd =", os.getcwd()
import fpdb