diff --git a/pyfpdb/Configuration.py b/pyfpdb/Configuration.py index 13e7cd28..e01a6b43 100755 --- a/pyfpdb/Configuration.py +++ b/pyfpdb/Configuration.py @@ -521,6 +521,8 @@ class Config: db['db-backend'] = 2 elif string.lower(self.supported_databases[name].db_server) == 'postgresql': db['db-backend'] = 3 + elif string.lower(self.supported_databases[name].db_server) == 'sqlite': + db['db-backend'] = 4 else: db['db-backend'] = None # this is big trouble return db diff --git a/pyfpdb/FpdbSQLQueries.py b/pyfpdb/FpdbSQLQueries.py index db2d7231..127644b3 100644 --- a/pyfpdb/FpdbSQLQueries.py +++ b/pyfpdb/FpdbSQLQueries.py @@ -38,7 +38,9 @@ class FpdbSQLQueries: elif(self.dbname == 'PostgreSQL'): self.query['list_tables'] = """SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'""" elif(self.dbname == 'SQLite'): - self.query['list_tables'] = """ """ + self.query['list_tables'] = """SELECT name FROM sqlite_master + WHERE type='table' + ORDER BY name;""" ################################################################## # Drop Tables - MySQL, PostgreSQL and SQLite all share same syntax @@ -63,8 +65,8 @@ class FpdbSQLQueries: self.query['createSettingsTable'] = """CREATE TABLE Settings (version SMALLINT)""" elif(self.dbname == 'SQLite'): - #Probably doesn't work. - self.query['createSettingsTable'] = """ """ + self.query['createSettingsTable'] = """CREATE TABLE Settings + (version INTEGER) """ ################################ @@ -83,7 +85,10 @@ class FpdbSQLQueries: name varchar(32), currency char(3))""" elif(self.dbname == 'SQLite'): - self.query['createSitesTable'] = """ """ + self.query['createSitesTable'] = """CREATE TABLE Sites ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + currency TEXT NOT NULL)""" ################################ @@ -118,7 +123,19 @@ class FpdbSQLQueries: smallBet int, bigBet int)""" elif(self.dbname == 'SQLite'): - self.query['createGametypesTable'] = """ """ + self.query['createGametypesTable'] = """CREATE TABLE GameTypes ( + id INTEGER PRIMARY KEY, + siteId INTEGER, + type TEXT, + base TEXT, + category TEXT, + limitType TEXT, + hiLo TEXT, + smallBlind INTEGER, + bigBlind INTEGER, + smallBet INTEGER, + bigBet INTEGER, + FOREIGN KEY(siteId) REFERENCES Sites(id) ON DELETE CASCADE)""" ################################ @@ -141,7 +158,13 @@ class FpdbSQLQueries: comment text, commentTs timestamp without time zone)""" elif(self.dbname == 'SQLite'): - self.query['createPlayersTable'] = """ """ + self.query['createPlayersTable'] = """CREATE TABLE Players ( + id INTEGER PRIMARY KEY, + name TEXT, + siteId INTEGER, + comment TEXT, + commentTs BLOB, + FOREIGN KEY(siteId) REFERENCES Sites(id) ON DELETE CASCADE)""" ################################ @@ -245,7 +268,18 @@ class FpdbSQLQueries: comment TEXT, commentTs timestamp without time zone)""" elif(self.dbname == 'SQLite'): - self.query['createHandsTable'] = """ """ + self.query['createHandsTable'] = """CREATE TABLE Hands ( + id INTEGER PRIMARY KEY, + tableName TEXT(20), + siteHandNo INTEGER, + gametypeId INTEGER, + handStart BLOB, + importTime BLOB, + seats INTEGER, + maxSeats INTEGER, + comment TEXT, + commentTs BLOB, + FOREIGN KEY(gametypeId) REFERENCES Gametypes(id) ON DELETE CASCADE)""" ################################ @@ -299,7 +333,14 @@ class FpdbSQLQueries: comment TEXT, commentTs timestamp without time zone)""" elif(self.dbname == 'SQLite'): - self.query['createTourneysTable'] = """ """ + self.query['createTourneysTable'] = """CREATE TABLE TourneyTypes ( + id INTEGER PRIMARY KEY, + siteId INTEGER, + buyin INTEGER, + fee INTEGER, + knockout INTEGER, + rebuyOrAddon BOOL, + FOREIGN KEY(siteId) REFERENCES Sites(id) ON DELETE CASCADE)""" ################################ # Create HandsPlayers @@ -832,6 +873,13 @@ class FpdbSQLQueries: elif(self.dbname == 'SQLite'): self.query['addPlayersIndex'] = """ """ + + if(self.dbname == 'MySQL InnoDB' or self.dbname == 'PostgreSQL'): + self.query['set tx level'] = """SET SESSION TRANSACTION + ISOLATION LEVEL READ COMMITTED""" + elif(self.dbname == 'SQLite'): + self.query['set tx level'] = """ """ + ################################ # Queries used in GuiGraphViewer ################################ diff --git a/pyfpdb/HandHistoryConverter.py b/pyfpdb/HandHistoryConverter.py index 7e08554b..16a89267 100644 --- a/pyfpdb/HandHistoryConverter.py +++ b/pyfpdb/HandHistoryConverter.py @@ -28,91 +28,51 @@ import codecs from decimal import Decimal import operator from xml.dom.minidom import Node -# from pokereval import PokerEval import time import datetime -import gettext - -#from pokerengine.pokercards import * -# provides letter2name{}, letter2names{}, visible_card(), not_visible_card(), is_visible(), card_value(), class PokerCards -# but it's probably not installed so here are the ones we may want: -letter2name = { - 'A': 'Ace', - 'K': 'King', - 'Q': 'Queen', - 'J': 'Jack', - 'T': 'Ten', - '9': 'Nine', - '8': 'Eight', - '7': 'Seven', - '6': 'Six', - '5': 'Five', - '4': 'Four', - '3': 'Trey', - '2': 'Deuce' - } - -letter2names = { - 'A': 'Aces', - 'K': 'Kings', - 'Q': 'Queens', - 'J': 'Jacks', - 'T': 'Tens', - '9': 'Nines', - '8': 'Eights', - '7': 'Sevens', - '6': 'Sixes', - '5': 'Fives', - '4': 'Fours', - '3': 'Treys', - '2': 'Deuces' - } import gettext -gettext.install('myapplication') +gettext.install('fpdb') class HandHistoryConverter(): READ_CHUNK_SIZE = 10000 # bytes to read at a time from file (in tail mode) def __init__(self, in_path = '-', out_path = '-', sitename = None, follow=False): - logging.info("HandHistory init called") + logging.info("HandHistory init") # default filetype and codepage. Subclasses should set these properly. self.filetype = "text" self.codepage = "utf8" - + self.in_path = in_path self.out_path = out_path - if self.out_path == '-': - # write to stdout + + if in_path == '-': + self.in_fh = sys.stdin + + if out_path == '-': self.out_fh = sys.stdout else: - # TODO: out_path should be sanity checked before opening. Perhaps in fpdb_import? - # I'm not sure what we're looking for, although we don't want out_path==in_path!='-' - self.out_fh = open(self.out_path, 'w') # doomswitch is now on :| + # TODO: out_path should be sanity checked. + self.out_fh = open(self.out_path, 'w') + self.sitename = sitename self.follow = follow self.compiledPlayers = set() self.maxseats = 10 def __str__(self): - #TODO : I got rid of most of the hhdir stuff. - tmp = "HandHistoryConverter: '%s'\n" % (self.sitename) - #tmp = tmp + "\thhbase: '%s'\n" % (self.hhbase) - #tmp = tmp + "\thhdir: '%s'\n" % (self.hhdir) - tmp = tmp + "\tfiletype: '%s'\n" % (self.filetype) - tmp = tmp + "\tinfile: '%s'\n" % (self.in_path) - tmp = tmp + "\toutfile: '%s'\n" % (self.out_path) - #tmp = tmp + "\tgametype: '%s'\n" % (self.gametype[0]) - #tmp = tmp + "\tgamebase: '%s'\n" % (self.gametype[1]) - #tmp = tmp + "\tlimit: '%s'\n" % (self.gametype[2]) - #tmp = tmp + "\tsb/bb: '%s/%s'\n" % (self.gametype[3], self.gametype[4]) - return tmp + return """ +HandHistoryConverter: '%(sitename)s' + filetype: '%(filetype)s' + in_path: '%(in_path)s' + out_path: '%(out_path)s' + """ % { 'sitename':self.sitename, 'filetype':self.filetype, 'in_path':self.in_path, 'out_path':self.out_path } def start(self): """process a hand at a time from the input specified by in_path. If in follow mode, wait for more data to turn up. -Otherwise, finish at eof... +Otherwise, finish at eof. """ starttime = time.time() @@ -224,7 +184,6 @@ which it expects to find at self.re_TailSplitHands -- see for e.g. Everleaf.py. base = gametype['base'] limit = gametype['limitType'] l = [type] + [base] + [limit] - hand = None if l in self.readSupportedGames(): hand = None if gametype['base'] == 'hold': diff --git a/pyfpdb/PokerStarsToFpdb.py b/pyfpdb/PokerStarsToFpdb.py index 5661e16e..b855586b 100755 --- a/pyfpdb/PokerStarsToFpdb.py +++ b/pyfpdb/PokerStarsToFpdb.py @@ -1,4 +1,4 @@ - #!/usr/bin/env python +#!/usr/bin/env python # -*- coding: utf-8 -*- # # Copyright 2008, Carl Gherardi @@ -298,7 +298,7 @@ follow : whether to tail -f the input""" hand.addHoleCards(street, hand.hero, closed=newcards, shown=False, mucked=False, dealt=True) for street, text in hand.streets.iteritems(): - if street in ('PREFLOP', 'DEAL'): continue # already done these + if not text or street in ('PREFLOP', 'DEAL'): continue # already done these m = self.re_HeroCards.finditer(hand.streets[street]) for found in m: player = found.group('PNAME') @@ -421,7 +421,7 @@ follow : whether to tail -f the input""" if __name__ == "__main__": parser = OptionParser() - parser.add_option("-i", "--input", dest="ipath", help="parse input hand history", default="regression-test-files/pokerstars/HH20090226 Natalie V - $0.10-$0.20 - HORSE.txt") + parser.add_option("-i", "--input", dest="ipath", help="parse input hand history", default="regression-test-files/stars/horse/HH20090226 Natalie V - $0.10-$0.20 - HORSE.txt") parser.add_option("-o", "--output", dest="opath", help="output translation to", default="-") parser.add_option("-f", "--follow", dest="follow", help="follow (tail -f) the input", action="store_true", default=False) parser.add_option("-q", "--quiet", diff --git a/pyfpdb/fpdb.py b/pyfpdb/fpdb.py index cdb89894..5da4b746 100755 --- a/pyfpdb/fpdb.py +++ b/pyfpdb/fpdb.py @@ -19,7 +19,6 @@ import os import sys import Options import string - cl_options = string.join(sys.argv[1:]) (options, sys.argv) = Options.fpdb_options() @@ -28,6 +27,8 @@ if not options.errorsToConsole: errorFile = open('fpdb-error-log.txt', 'w', 0) sys.stderr = errorFile +import logging + import pygtk pygtk.require('2.0') import gtk diff --git a/pyfpdb/fpdb_db.py b/pyfpdb/fpdb_db.py index 0af29aa1..30101d0c 100644 --- a/pyfpdb/fpdb_db.py +++ b/pyfpdb/fpdb_db.py @@ -18,20 +18,21 @@ import os import re import sys +import logging from time import time, strftime import fpdb_simple import FpdbSQLQueries class fpdb_db: + MYSQL_INNODB = 2 + PGSQL = 3 + SQLITE = 4 def __init__(self): """Simple constructor, doesnt really do anything""" self.db = None self.cursor = None self.sql = {} - self.MYSQL_INNODB = 2 - self.PGSQL = 3 - self.SQLITE = 4 # Data Structures for index and foreign key creation # drop_code is an int with possible values: 0 - don't drop for bulk import @@ -72,6 +73,8 @@ class fpdb_db: , {'tab':'TourneysPlayers', 'col':'tourneyId', 'drop':0} , {'tab':'TourneyTypes', 'col':'siteId', 'drop':0} ] + , [ # indexes for sqlite (list index 4) + ] ] self.foreignKeys = [ @@ -146,12 +149,12 @@ class fpdb_db: self.settings = {} self.settings['os'] = "linuxmac" if os.name != "nt" else "windows" - self.settings.update(config.get_db_parameters()) - self.connect(self.settings['db-backend'], - self.settings['db-host'], - self.settings['db-databaseName'], - self.settings['db-user'], - self.settings['db-password']) + db = config.get_db_parameters() + self.connect(backend=db['db-backend'], + host=db['db-host'], + database=db['db-databaseName'], + user=db['db-user'], + password=db['db-password']) #end def do_connect def connect(self, backend=None, host=None, database=None, @@ -164,13 +167,13 @@ class fpdb_db: self.user=user self.password=password self.database=database - if backend==self.MYSQL_INNODB: + if backend==fpdb_db.MYSQL_INNODB: import MySQLdb try: self.db = MySQLdb.connect(host = host, user = user, passwd = password, db = database, use_unicode=True) except: raise fpdb_simple.FpdbError("MySQL connection failed") - elif backend==self.PGSQL: + elif backend==fpdb_db.PGSQL: import psycopg2 import psycopg2.extensions psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) @@ -201,12 +204,19 @@ class fpdb_db: msg = "PostgreSQL connection to database (%s) user (%s) failed." % (database, user) print msg raise fpdb_simple.FpdbError(msg) + elif backend==fpdb_db.SQLITE: + logging.info("Connecting to SQLite:%(database)s" % {'database':database}) + import sqlite3 + self.db = sqlite3.connect(database,detect_types=sqlite3.PARSE_DECLTYPES) + sqlite3.register_converter("bool", lambda x: bool(int(x))) + sqlite3.register_adapter(bool, lambda x: "1" if x else "0") + else: raise fpdb_simple.FpdbError("unrecognised database backend:"+backend) self.cursor=self.db.cursor() - self.cursor.execute('SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED') # Set up query dictionary as early in the connection process as we can. self.sql = FpdbSQLQueries.FpdbSQLQueries(self.get_backend_name()) + self.cursor.execute(self.sql.query['set tx level']) self.wrongDbVersion=False try: self.cursor.execute("SELECT * FROM Settings") @@ -237,7 +247,9 @@ class fpdb_db: def create_tables(self): #todo: should detect and fail gracefully if tables already exist. + logging.debug(self.sql.query['createSettingsTable']) self.cursor.execute(self.sql.query['createSettingsTable']) + logging.debug(self.sql.query['createSitesTable']) self.cursor.execute(self.sql.query['createSitesTable']) self.cursor.execute(self.sql.query['createGametypesTable']) self.cursor.execute(self.sql.query['createPlayersTable']) @@ -274,10 +286,12 @@ class fpdb_db: for table in tables: self.cursor.execute(self.sql.query['drop_table'] + table[0] + ' cascade') elif(self.get_backend_name() == 'SQLite'): - #todo: sqlite version here - print "Empty function here" + self.cursor.execute(self.sql.query['list_tables']) + for table in self.cursor.fetchall(): + logging.debug(self.sql.query['drop_table'] + table[0]) + self.cursor.execute(self.sql.query['drop_table'] + table[0]) - self.db.commit() + self.db.commit() #end def drop_tables def drop_referential_integrity(self): @@ -306,6 +320,8 @@ class fpdb_db: return "MySQL InnoDB" elif self.backend==3: return "PostgreSQL" + elif self.backend==4: + return "SQLite" else: raise fpdb_simple.FpdbError("invalid backend") #end def get_backend_name @@ -315,12 +331,14 @@ class fpdb_db: #end def get_db_info def fillDefaultData(self): - self.cursor.execute("INSERT INTO Settings VALUES (118);") - self.cursor.execute("INSERT INTO Sites VALUES (DEFAULT, 'Full Tilt Poker', 'USD');") - self.cursor.execute("INSERT INTO Sites VALUES (DEFAULT, 'PokerStars', 'USD');") - self.cursor.execute("INSERT INTO Sites VALUES (DEFAULT, 'Everleaf', 'USD');") - self.cursor.execute("INSERT INTO Sites VALUES (DEFAULT, 'Win2day', 'USD');") - self.cursor.execute("INSERT INTO TourneyTypes VALUES (DEFAULT, 1, 0, 0, 0, False);") + self.cursor.execute("INSERT INTO Settings (version) VALUES (118);") + self.cursor.execute("INSERT INTO Sites (name,currency) VALUES ('Full Tilt Poker', 'USD')") + self.cursor.execute("INSERT INTO Sites (name,currency) VALUES ('PokerStars', 'USD')") + self.cursor.execute("INSERT INTO Sites (name,currency) VALUES ('Everleaf', 'USD')") + self.cursor.execute("INSERT INTO Sites (name,currency) VALUES ('Win2day', 'USD')") + self.cursor.execute("""INSERT INTO TourneyTypes + (siteId,buyin,fee,knockout,rebuyOrAddon) VALUES + (1,0,0,0,?)""",(False,) ) #end def fillDefaultData def recreate_tables(self): @@ -618,7 +636,7 @@ class fpdb_db: ret = -1 else: ret = row[0] - elif self.backend == self.SQLITE: + elif self.backend == fpdb_db.SQLITE: # don't know how to do this in sqlite print "getLastInsertId(): not coded for sqlite yet" ret = -1 @@ -629,27 +647,70 @@ class fpdb_db: def storeHand(self, p): #stores into table hands: - self.cursor.execute ("""INSERT INTO Hands - (siteHandNo, gametypeId, handStart, seats, tableName, importTime, maxSeats - ,boardcard1, boardcard2, boardcard3, boardcard4, boardcard5 - ,playersVpi, playersAtStreet1, playersAtStreet2 - ,playersAtStreet3, playersAtStreet4, playersAtShowdown - ,street0Raises, street1Raises, street2Raises - ,street3Raises, street4Raises, street1Pot - ,street2Pot, street3Pot, street4Pot - ,showdownPot + self.cursor.execute ("""INSERT INTO Hands ( + tablename, + sitehandno, + gametypeid, + handstart, + importtime, + seats, + maxseats, + boardcard1, + boardcard2, + boardcard3, + boardcard4, + boardcard5, +-- texture, + playersVpi, + playersAtStreet1, + playersAtStreet2, + playersAtStreet3, + playersAtStreet4, + playersAtShowdown, + street0Raises, + street1Raises, + street2Raises, + street3Raises, + street4Raises, +-- street1Pot, +-- street2Pot, +-- street3Pot, +-- street4Pot, +-- showdownPot ) VALUES - (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""" - ,(p['siteHandNo'], gametype_id, p['handStart'], len(names), p['tableName'], datetime.datetime.today(), p['maxSeats'] - ,p['boardcard1'], ['boardcard2'], p['boardcard3'], ['boardcard4'], ['boardcard5'] - ,hudCache['playersVpi'], hudCache['playersAtStreet1'], hudCache['playersAtStreet2'] - ,hudCache['playersAtStreet3'], hudCache['playersAtStreet4'], hudCache['playersAtShowdown'] - ,hudCache['street0Raises'], hudCache['street1Raises'], hudCache['street2Raises'] - ,hudCache['street3Raises'], hudCache['street4Raises'], hudCache['street1Pot'] - ,hudCache['street2Pot'], hudCache['street3Pot'], hudCache['street4Pot'] - ,hudCache['showdownPot'] - ) - ) + (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, %s, %s)""", + ( + p['tablename'], + p['sitehandno'], + p['gametypeid'], + p['handStart'], + datetime.datetime.today(), + len(p['names']), + p['maxSeats'], + p['boardcard1'], + p['boardcard2'], + p['boardcard3'], + p['boardcard4'], + p['boardcard5'], + hudCache['playersVpi'], + hudCache['playersAtStreet1'], + hudCache['playersAtStreet2'], + hudCache['playersAtStreet3'], + hudCache['playersAtStreet4'], + hudCache['playersAtShowdown'], + hudCache['street0Raises'], + hudCache['street1Raises'], + hudCache['street2Raises'], + hudCache['street3Raises'], + hudCache['street4Raises'], + hudCache['street1Pot'], + hudCache['street2Pot'], + hudCache['street3Pot'], + hudCache['street4Pot'], + hudCache['showdownPot'] + ) + ) #return getLastInsertId(backend, conn, cursor) #end class fpdb_db diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 2468fecf..3129c22c 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -22,6 +22,7 @@ import os # todo: remove this once import_dir is in fpdb_import import sys from time import time, strftime +import logging import traceback import math import datetime