evilny0/fpdb
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'master' of git://git.assembla.com/fpdb-sql.git

Browse Source
This commit is contained in:
eblade 2009-08-04 20:22:25 -04:00
commit 9656db0b80
2 changed files with 60 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
pyfpdb/GuiBulkImport.py
View File

@ -36,7 +36,7 @@ import Configuration
class GuiBulkImport(): class GuiBulkImport():
# CONFIGURATION - update these as preferred: # CONFIGURATION - update these as preferred:
allowThreads = False # set to True to try out the threads field allowThreads = True # set to True to try out the threads field
# not used # not used
def import_dir(self): def import_dir(self):
@ -71,12 +71,19 @@ class GuiBulkImport():
self.importer.setHandsInDB(self.n_hands_in_db) self.importer.setHandsInDB(self.n_hands_in_db)
cb_model = self.cb_dropindexes.get_model() cb_model = self.cb_dropindexes.get_model()
cb_index = self.cb_dropindexes.get_active() cb_index = self.cb_dropindexes.get_active()
cb_hmodel = self.cb_drophudcache.get_model()
cb_hindex = self.cb_drophudcache.get_active()
self.lab_info.set_text("Importing")
if cb_index: if cb_index:
self.importer.setDropIndexes(cb_model[cb_index][0]) self.importer.setDropIndexes(cb_model[cb_index][0])
else: else:
self.importer.setDropIndexes("auto") self.importer.setDropIndexes("auto")
if cb_hindex:
self.importer.setDropHudCache(cb_hmodel[cb_hindex][0])
else:
self.importer.setDropHudCache("auto")
sitename = self.cbfilter.get_model()[self.cbfilter.get_active()][0] sitename = self.cbfilter.get_model()[self.cbfilter.get_active()][0]
self.lab_info.set_text("Importing")
self.importer.addBulkImportImportFileOrDir(self.inputFile, site = sitename) self.importer.addBulkImportImportFileOrDir(self.inputFile, site = sitename)
self.importer.setCallHud(False) self.importer.setCallHud(False)
@ -115,7 +122,7 @@ class GuiBulkImport():
self.chooser.show() self.chooser.show()
# Table widget to hold the settings # Table widget to hold the settings
self.table = gtk.Table(rows = 3, columns = 5, homogeneous = False) self.table = gtk.Table(rows = 5, columns = 5, homogeneous = False)
self.vbox.add(self.table) self.vbox.add(self.table)
self.table.show() self.table.show()
@ -130,6 +137,7 @@ class GuiBulkImport():
self.table.attach(self.lab_status, 1, 2, 0, 1, xpadding = 0, ypadding = 0, yoptions=gtk.SHRINK) self.table.attach(self.lab_status, 1, 2, 0, 1, xpadding = 0, ypadding = 0, yoptions=gtk.SHRINK)
self.lab_status.show() self.lab_status.show()
self.lab_status.set_justify(gtk.JUSTIFY_RIGHT) self.lab_status.set_justify(gtk.JUSTIFY_RIGHT)
self.lab_status.set_alignment(1.0, 0.5)
# spin button - status # spin button - status
status_adj = gtk.Adjustment(value=100, lower=0, upper=300, step_incr=10, page_incr=1, page_size=0) #not sure what upper value should be! status_adj = gtk.Adjustment(value=100, lower=0, upper=300, step_incr=10, page_incr=1, page_size=0) #not sure what upper value should be!
@ -141,13 +149,15 @@ class GuiBulkImport():
self.lab_threads = gtk.Label("Number of threads:") self.lab_threads = gtk.Label("Number of threads:")
self.table.attach(self.lab_threads, 3, 4, 0, 1, xpadding = 0, ypadding = 0, yoptions=gtk.SHRINK) self.table.attach(self.lab_threads, 3, 4, 0, 1, xpadding = 0, ypadding = 0, yoptions=gtk.SHRINK)
self.lab_threads.show() self.lab_threads.show()
self.lab_threads.set_sensitive(False) if not self.allowThreads:
self.lab_threads.set_sensitive(False)
self.lab_threads.set_justify(gtk.JUSTIFY_RIGHT) self.lab_threads.set_justify(gtk.JUSTIFY_RIGHT)
self.lab_threads.set_alignment(1.0, 0.5)
# spin button - threads # spin button - threads
threads_adj = gtk.Adjustment(value=0, lower=0, upper=10, step_incr=1, page_incr=1, page_size=0) #not sure what upper value should be! threads_adj = gtk.Adjustment(value=0, lower=0, upper=10, step_incr=1, page_incr=1, page_size=0) #not sure what upper value should be!
self.spin_threads = gtk.SpinButton(adjustment=threads_adj, climb_rate=0.0, digits=0) self.spin_threads = gtk.SpinButton(adjustment=threads_adj, climb_rate=0.0, digits=0)
self.table.attach(self.spin_threads, 4, 5, 0, 1, xpadding = 0, ypadding = 0, yoptions=gtk.SHRINK) self.table.attach(self.spin_threads, 4, 5, 0, 1, xpadding = 10, ypadding = 0, yoptions=gtk.SHRINK)
self.spin_threads.show() self.spin_threads.show()
if not self.allowThreads: if not self.allowThreads:
self.spin_threads.set_sensitive(False) self.spin_threads.set_sensitive(False)
@ -162,6 +172,7 @@ class GuiBulkImport():
self.table.attach(self.lab_hands, 1, 2, 1, 2, xpadding = 0, ypadding = 0, yoptions=gtk.SHRINK) self.table.attach(self.lab_hands, 1, 2, 1, 2, xpadding = 0, ypadding = 0, yoptions=gtk.SHRINK)
self.lab_hands.show() self.lab_hands.show()
self.lab_hands.set_justify(gtk.JUSTIFY_RIGHT) self.lab_hands.set_justify(gtk.JUSTIFY_RIGHT)
self.lab_hands.set_alignment(1.0, 0.5)
# spin button - hands to import # spin button - hands to import
hands_adj = gtk.Adjustment(value=0, lower=0, upper=10, step_incr=1, page_incr=1, page_size=0) #not sure what upper value should be! hands_adj = gtk.Adjustment(value=0, lower=0, upper=10, step_incr=1, page_incr=1, page_size=0) #not sure what upper value should be!
@ -174,6 +185,7 @@ class GuiBulkImport():
self.table.attach(self.lab_drop, 3, 4, 1, 2, xpadding = 0, ypadding = 0, yoptions=gtk.SHRINK) self.table.attach(self.lab_drop, 3, 4, 1, 2, xpadding = 0, ypadding = 0, yoptions=gtk.SHRINK)
self.lab_drop.show() self.lab_drop.show()
self.lab_drop.set_justify(gtk.JUSTIFY_RIGHT) self.lab_drop.set_justify(gtk.JUSTIFY_RIGHT)
self.lab_drop.set_alignment(1.0, 0.5)
# ComboBox - drop indexes # ComboBox - drop indexes
self.cb_dropindexes = gtk.combo_box_new_text() self.cb_dropindexes = gtk.combo_box_new_text()
@ -186,9 +198,10 @@ class GuiBulkImport():
# label - filter # label - filter
self.lab_filter = gtk.Label("Site filter:") self.lab_filter = gtk.Label("Site filter:")
self.table.attach(self.lab_filter, 2, 3, 2, 3, xpadding = 0, ypadding = 0, yoptions=gtk.SHRINK) self.table.attach(self.lab_filter, 1, 2, 2, 3, xpadding = 0, ypadding = 0, yoptions=gtk.SHRINK)
self.lab_filter.show() self.lab_filter.show()
self.lab_filter.set_justify(gtk.JUSTIFY_RIGHT) self.lab_filter.set_justify(gtk.JUSTIFY_RIGHT)
self.lab_filter.set_alignment(1.0, 0.5)
# ComboBox - filter # ComboBox - filter
self.cbfilter = gtk.combo_box_new_text() self.cbfilter = gtk.combo_box_new_text()
@ -196,21 +209,42 @@ class GuiBulkImport():
print w print w
self.cbfilter.append_text(w) self.cbfilter.append_text(w)
self.cbfilter.set_active(0) self.cbfilter.set_active(0)
self.table.attach(self.cbfilter, 3, 4, 2, 3, xpadding = 10, ypadding = 0, yoptions=gtk.SHRINK) self.table.attach(self.cbfilter, 2, 3, 2, 3, xpadding = 10, ypadding = 1, yoptions=gtk.SHRINK)
self.cbfilter.show() self.cbfilter.show()
# label - info # label - drop hudcache
self.lab_info = gtk.Label() self.lab_drop = gtk.Label("Drop HudCache:")
self.table.attach(self.lab_info, 0, 4, 2, 3, xpadding = 0, ypadding = 0, yoptions=gtk.SHRINK) self.table.attach(self.lab_drop, 3, 4, 2, 3, xpadding = 0, ypadding = 0, yoptions=gtk.SHRINK)
self.lab_info.show() self.lab_drop.show()
self.lab_drop.set_justify(gtk.JUSTIFY_RIGHT)
self.lab_drop.set_alignment(1.0, 0.5)
# ComboBox - drop hudcache
self.cb_drophudcache = gtk.combo_box_new_text()
self.cb_drophudcache.append_text('auto')
self.cb_drophudcache.append_text("don't drop")
self.cb_drophudcache.append_text('drop')
self.cb_drophudcache.set_active(0)
self.table.attach(self.cb_drophudcache, 4, 5, 2, 3, xpadding = 10, ypadding = 0, yoptions=gtk.SHRINK)
self.cb_drophudcache.show()
# button - Import # button - Import
self.load_button = gtk.Button('Import') # todo: rename variables to import too self.load_button = gtk.Button('Import') # todo: rename variables to import too
self.load_button.connect('clicked', self.load_clicked, self.load_button.connect('clicked', self.load_clicked,
'Import clicked') 'Import clicked')
self.table.attach(self.load_button, 4, 5, 2, 3, xpadding = 0, ypadding = 0, yoptions=gtk.SHRINK) self.table.attach(self.load_button, 2, 3, 4, 5, xpadding = 0, ypadding = 0, yoptions=gtk.SHRINK)
self.load_button.show() self.load_button.show()
# label - spacer (keeps rows 3 & 5 apart)
self.lab_spacer = gtk.Label()
self.table.attach(self.lab_spacer, 3, 5, 3, 4, xpadding = 0, ypadding = 0, yoptions=gtk.SHRINK)
self.lab_spacer.show()
# label - info
self.lab_info = gtk.Label()
self.table.attach(self.lab_info, 3, 5, 4, 5, xpadding = 0, ypadding = 0, yoptions=gtk.SHRINK)
self.lab_info.show()
# see how many hands are in the db and adjust accordingly # see how many hands are in the db and adjust accordingly
tcursor = self.importer.database.cursor tcursor = self.importer.database.cursor
tcursor.execute("Select count(1) from Hands") tcursor.execute("Select count(1) from Hands")
@ -222,6 +256,9 @@ class GuiBulkImport():
self.cb_dropindexes.set_active(2) self.cb_dropindexes.set_active(2)
self.cb_dropindexes.set_sensitive(False) self.cb_dropindexes.set_sensitive(False)
self.lab_drop.set_sensitive(False) self.lab_drop.set_sensitive(False)
self.cb_drophudcache.set_active(2)
self.cb_drophudcache.set_sensitive(False)
self.lab_drop.set_sensitive(False)
def main(argv=None): def main(argv=None):
"""main can also be called in the python interpreter, by supplying the command line as the argument.""" """main can also be called in the python interpreter, by supplying the command line as the argument."""

20
pyfpdb/fpdb_import.py
View File

@ -78,19 +78,18 @@ class Importer:
#Set defaults #Set defaults
self.callHud = self.config.get_import_parameters().get("callFpdbHud") self.callHud = self.config.get_import_parameters().get("callFpdbHud")
# CONFIGURATION OPTIONS - update allowHudcacheRebuild and forceThreads for faster imports # CONFIGURATION OPTIONS
self.settings.setdefault("minPrint", 30) self.settings.setdefault("minPrint", 30)
self.settings.setdefault("handCount", 0) self.settings.setdefault("handCount", 0)
self.settings.setdefault("allowHudcacheRebuild", False) # if True speeds up big imports a lot, also #self.settings.setdefault("allowHudcacheRebuild", True) # NOT USED NOW
# stops deadlock problems with threaded imports #self.settings.setdefault("forceThreads", 2) # NOT USED NOW
self.settings.setdefault("forceThreads", 0) # set to 1/2/more for faster imports
self.settings.setdefault("writeQSize", 1000) # no need to change self.settings.setdefault("writeQSize", 1000) # no need to change
self.settings.setdefault("writeQMaxWait", 10) # not used self.settings.setdefault("writeQMaxWait", 10) # not used
self.writeq = None self.writeq = None
self.database = Database.Database(self.config, sql = self.sql) self.database = Database.Database(self.config, sql = self.sql)
self.writerdbs = [] self.writerdbs = []
self.settings.setdefault("threads", 1) # value overridden by GuiBulkImport - use forceThreads above self.settings.setdefault("threads", 1) # value set by GuiBulkImport
for i in xrange(self.settings['threads']): for i in xrange(self.settings['threads']):
self.writerdbs.append( Database.Database(self.config, sql = self.sql) ) self.writerdbs.append( Database.Database(self.config, sql = self.sql) )
@ -124,6 +123,9 @@ class Importer:
def setDropIndexes(self, value): def setDropIndexes(self, value):
self.settings['dropIndexes'] = value self.settings['dropIndexes'] = value
def setDropHudCache(self, value):
self.settings['dropHudCache'] = value
# def setWatchTime(self): # def setWatchTime(self):
# self.updated = time() # self.updated = time()
@ -186,8 +188,8 @@ class Importer:
def runImport(self): def runImport(self):
""""Run full import on self.filelist. This is called from GuiBulkImport.py""" """"Run full import on self.filelist. This is called from GuiBulkImport.py"""
if self.settings['forceThreads'] > 0: # use forceThreads until threading enabled in GuiBulkImport #if self.settings['forceThreads'] > 0: # use forceThreads until threading enabled in GuiBulkImport
self.setThreads(self.settings['forceThreads']) # self.setThreads(self.settings['forceThreads'])
# Initial setup # Initial setup
start = datetime.datetime.now() start = datetime.datetime.now()
@ -195,7 +197,7 @@ class Importer:
print "Started at", start, "--", len(self.filelist), "files to import.", self.settings['dropIndexes'] print "Started at", start, "--", len(self.filelist), "files to import.", self.settings['dropIndexes']
if self.settings['dropIndexes'] == 'auto': if self.settings['dropIndexes'] == 'auto':
self.settings['dropIndexes'] = self.calculate_auto2(self.database, 12.0, 500.0) self.settings['dropIndexes'] = self.calculate_auto2(self.database, 12.0, 500.0)
if self.settings['allowHudcacheRebuild']: if self.settings['dropHudCache'] == 'auto':
self.settings['dropHudCache'] = self.calculate_auto2(self.database, 25.0, 500.0) # returns "drop"/"don't drop" self.settings['dropHudCache'] = self.calculate_auto2(self.database, 25.0, 500.0) # returns "drop"/"don't drop"
if self.settings['dropIndexes'] == 'drop': if self.settings['dropIndexes'] == 'drop':
@ -237,7 +239,7 @@ class Importer:
self.database.afterBulkImport() self.database.afterBulkImport()
else: else:
print "No need to rebuild indexes." print "No need to rebuild indexes."
if self.settings['allowHudcacheRebuild'] and self.settings['dropHudCache'] == 'drop': if self.settings['dropHudCache'] == 'drop':
self.database.rebuild_hudcache() self.database.rebuild_hudcache()
else: else:
print "No need to rebuild hudcache." print "No need to rebuild hudcache."