From 784e60713533f8ed2f407c4cb0cb88dfa2b4ad86 Mon Sep 17 00:00:00 2001 From: Aiwota Programmer Date: Fri, 22 Sep 2006 05:16:44 +0900 Subject: [PATCH] Use progressbar of board window. --- src/FukuiNoNamari/board_data.py | 170 +++++++++++++++++++++++++++----------- src/FukuiNoNamari/board_window.py | 4 + src/FukuiNoNamari/cachefile.py | 2 +- src/FukuiNoNamari/session.py | 4 +- 4 files changed, 130 insertions(+), 50 deletions(-) diff --git a/src/FukuiNoNamari/board_data.py b/src/FukuiNoNamari/board_data.py index 298df3b..2f4f500 100644 --- a/src/FukuiNoNamari/board_data.py +++ b/src/FukuiNoNamari/board_data.py @@ -16,6 +16,7 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA import gobject +import gtk import os.path import glob import codecs @@ -31,6 +32,17 @@ from http_sub import HTTPRedirectHandler302, HTTPDebugHandler BOARD_DATA_INVALID_VALUE = 0 +def accumulate(iterable, initial_value=0): + sum_value = initial_value + for value in iterable: + sum_value += value + yield sum_value + +def follow(iterable, under_value=0): + before = under_value + for item in iterable: + yield before, item + before = item class BoardData: @@ -40,6 +52,9 @@ class BoardData: def set_status(self, text): pass + def set_fraction(self, fraction): + pass + def _merge_new_thread(self, datalist, id, title, res, num, lastmod): average = 0 if lastmod != 0: @@ -74,14 +89,19 @@ class BoardData: "lastModified": "", "average": average} def merge_local_subjecttxt(self, datalist): - f = lambda id, title, res, num, lastmod: \ + iterable = self._load_subjecttxt() + + for id, title, res, num, lastmod in iterable: self._merge_new_thread(datalist, id, title, res, num, lastmod) - self._load_subjecttxt(f) + + status = "Complete subject file." + gobject.idle_add(self.set_status, status) def merge_remote_subjecttxt(self, datalist): - f = lambda id, title, res, num, lastmod: \ + iterable = self._get_subjecttxt() + + for id, title, res, num, lastmod in iterable: self._merge_new_thread(datalist, id, title, res, num, lastmod) - self._get_subjecttxt(f) def _init_extra_data(self, dic): dic["num"] = 0 @@ -89,19 +109,54 @@ class BoardData: dic["average"] = 0 return dic + def _progressing(self, iterable): + for before, fraction in follow(iterable): + if int(before*10) != int(fraction*10): + gtk.threads_enter() + try: + self.set_fraction(fraction) + finally: + gtk.threads_leave() + yield fraction + def load_idxfiles(self): print "load_cache" - iterable = cachefile.load_cache(self.bbs_type) - iterable = itertools.imap(self._init_extra_data, iterable) - datalist = dict([(dic["id"], dic) for dic in iterable]) - + datalist = self._load_cache() print "load_idx" self._load_modified_idxfiles(datalist) print "save_cache" cachefile.save_cache(self.bbs_type, datalist) + status = "Complete index files." + gobject.idle_add(self.set_status, status) return datalist + def _load_cache(self): + try: + total = os.path.getsize(misc.get_board_cache_path(self.bbs_type)) + except OSError: + total = -1 + + iterable = cachefile.load_cache(self.bbs_type) + + # split + iterable_dic, iterable_line = itertools.tee(iterable) + iterable_dic = itertools.starmap(lambda x, y: x, iterable_dic) + iterable_line = itertools.starmap(lambda x, y: y, iterable_line) + + iterable_line = itertools.imap(lambda x :len(x), iterable_line) + iterable_line = accumulate(iterable_line) + iterable_line = itertools.imap( + lambda value: float(value) / total / 2, iterable_line) + iterable_line = self._progressing(iterable_line) + + # union + iterable = itertools.imap(lambda x, y: x, iterable_dic, iterable_line) + + iterable = itertools.imap(self._init_extra_data, iterable) + + return dict([(dic["id"], dic) for dic in iterable]) + def _load_modified_idxfiles(self, datalist): basedir = misc.get_thread_idx_dir_path(self.bbs_type) ext = ".idx" @@ -152,7 +207,7 @@ class BoardData: return id, title, res return None - def _load_subjecttxt(self, func): + def _load_subjecttxt(self): lastmod = self.load_board_idx() try: lastmod = misc.httpdate_to_secs(lastmod) @@ -161,20 +216,31 @@ class BoardData: subjecttxt_path = misc.get_board_subjecttxt_path(self.bbs_type) try: - for num, line_encoded \ - in itertools.izip(itertools.count(1), - file(subjecttxt_path)): - result = self._split_record(line_encoded) - if result: - id, title, res = result - try: - func(id, title, res, num, lastmod) - except: - traceback.print_exc() - except IOError: - traceback.print_exc() + total = os.path.getsize(subjecttxt_path) + except OSError: + total = -1 + + iterable = file(subjecttxt_path) + + # split + iterable, iterable_len = itertools.tee(iterable) - def _get_subjecttxt(self, func): + iterable_len = itertools.imap(lambda l: len(l), iterable_len) + iterable_len = accumulate(iterable_len) + iterable_len = itertools.imap( + lambda value: float(value) / total / 2 + 0.5, iterable_len) + iterable_len = self._progressing(iterable_len) + + # union + iterable = itertools.imap(lambda x, y: x, iterable, iterable_len) + + for num, line_encoded in itertools.izip(itertools.count(1), iterable): + result = self._split_record(line_encoded) + if result: + id, title, res = result + yield id, title, res, num, lastmod + + def _get_subjecttxt(self): # get subject.txt @@ -207,36 +273,44 @@ class BoardData: lastmod = 0 subjecttxt_path = misc.get_board_subjecttxt_path(self.bbs_type) - basedir = os.path.dirname(subjecttxt_path) - if not os.path.isdir(basedir): - os.makedirs(basedir) - f = None - try: - f = file(subjecttxt_path, "w") - except IOError: - traceback.print_exc() + f = misc.FileWrap(subjecttxt_path, "w") try: - for num, line_encoded in itertools.izip(itertools.count(1), - response): - if f: - try: - f.write(line_encoded) - except IOError: - traceback.print_exc() - result = self._split_record(line_encoded) - if result: - id, title, res = result - try: - func(id, title, res, num, lastmod) - except: - traceback.print_exc() + total = int(info["Content-Length"]) except: - traceback.print_exc() + total = -1 + + def saving(line_encoded): + try: + f.write(line_encoded) + except IOError: + traceback.print_exc() + return line_encoded + + iterable = response + + # split + iterable, iterable_len = itertools.tee(iterable) + + iterable_len = itertools.imap(lambda l: len(l), iterable_len) + iterable_len = accumulate(iterable_len) + iterable_len = itertools.imap( + lambda value: float(value) / total / 2 + 0.5, iterable_len) + iterable_len = self._progressing(iterable_len) + + # union + iterable = itertools.imap(lambda x, y: x, iterable, iterable_len) + + iterable = itertools.imap(saving, iterable) + iterable = itertools.izip(itertools.count(1), iterable) + + for num, line_encoded in iterable: + result = self._split_record(line_encoded) + if result: + id, title, res = result + yield id, title, res, num, lastmod - if f: - f.close() - f = None + f.close() def load_board_idx(self): lastmod = "" diff --git a/src/FukuiNoNamari/board_window.py b/src/FukuiNoNamari/board_window.py index 60cab34..6383d18 100644 --- a/src/FukuiNoNamari/board_window.py +++ b/src/FukuiNoNamari/board_window.py @@ -99,6 +99,7 @@ class WinWrap(winwrapbase.WinWrapBase, board_data.BoardData): self.popupmenu = self.widget_tree.get_widget("popup_treeview_menu") self.toolbar = self.widget_tree.get_widget("toolbar") self.statusbar = self.widget_tree.get_widget("appbar") + self.progress = self.statusbar.get_progress() self.filterbar = self.widget_tree.get_widget( "bonobodockitem_filterbar") self.entry_filterbar = self.widget_tree.get_widget("entry_filterbar") @@ -107,6 +108,9 @@ class WinWrap(winwrapbase.WinWrapBase, board_data.BoardData): def set_status(self, text): self.statusbar.set_status(text) + def set_fraction(self, fraction): + self.progress.set_fraction(fraction) + def destroy(self): self.save() self.window.destroy() diff --git a/src/FukuiNoNamari/cachefile.py b/src/FukuiNoNamari/cachefile.py index 8a7af70..0b042f4 100644 --- a/src/FukuiNoNamari/cachefile.py +++ b/src/FukuiNoNamari/cachefile.py @@ -85,7 +85,7 @@ def load_cache(bbs_type): # only if id exists if "id" in dic and dic["id"]: - yield dic + yield dic, line except IOError: traceback.print_exc() diff --git a/src/FukuiNoNamari/session.py b/src/FukuiNoNamari/session.py index 920e173..99b2319 100644 --- a/src/FukuiNoNamari/session.py +++ b/src/FukuiNoNamari/session.py @@ -121,4 +121,6 @@ def start(): restore() if not _windows: uri_opener.open_uri("http://ex11.2ch.net/morningcoffee/") - gtk.main() + gtk.threads_enter() + gtk.main() + gtk.threads_leave() -- 2.11.0