# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
import gobject
+import gtk
import os.path
import glob
import codecs
BOARD_DATA_INVALID_VALUE = 0
+def accumulate(iterable, initial_value=0):
+ sum_value = initial_value
+ for value in iterable:
+ sum_value += value
+ yield sum_value
+
+def follow(iterable, under_value=0):
+ before = under_value
+ for item in iterable:
+ yield before, item
+ before = item
class BoardData:
def set_status(self, text):
pass
+ def set_fraction(self, fraction):
+ pass
+
def _merge_new_thread(self, datalist, id, title, res, num, lastmod):
average = 0
if lastmod != 0:
"lastModified": "", "average": average}
def merge_local_subjecttxt(self, datalist):
- f = lambda id, title, res, num, lastmod: \
+ iterable = self._load_subjecttxt()
+
+ for id, title, res, num, lastmod in iterable:
self._merge_new_thread(datalist, id, title, res, num, lastmod)
- self._load_subjecttxt(f)
+
+ status = "Complete subject file."
+ gobject.idle_add(self.set_status, status)
def merge_remote_subjecttxt(self, datalist):
- f = lambda id, title, res, num, lastmod: \
+ iterable = self._get_subjecttxt()
+
+ for id, title, res, num, lastmod in iterable:
self._merge_new_thread(datalist, id, title, res, num, lastmod)
- self._get_subjecttxt(f)
def _init_extra_data(self, dic):
dic["num"] = 0
dic["average"] = 0
return dic
+ def _progressing(self, iterable):
+ for before, fraction in follow(iterable):
+ if int(before*10) != int(fraction*10):
+ gtk.threads_enter()
+ try:
+ self.set_fraction(fraction)
+ finally:
+ gtk.threads_leave()
+ yield fraction
+
def load_idxfiles(self):
print "load_cache"
- iterable = cachefile.load_cache(self.bbs_type)
- iterable = itertools.imap(self._init_extra_data, iterable)
- datalist = dict([(dic["id"], dic) for dic in iterable])
-
+ datalist = self._load_cache()
print "load_idx"
self._load_modified_idxfiles(datalist)
print "save_cache"
cachefile.save_cache(self.bbs_type, datalist)
+ status = "Complete index files."
+ gobject.idle_add(self.set_status, status)
return datalist
+ def _load_cache(self):
+ try:
+ total = os.path.getsize(misc.get_board_cache_path(self.bbs_type))
+ except OSError:
+ total = -1
+
+ iterable = cachefile.load_cache(self.bbs_type)
+
+ # split
+ iterable_dic, iterable_line = itertools.tee(iterable)
+ iterable_dic = itertools.starmap(lambda x, y: x, iterable_dic)
+ iterable_line = itertools.starmap(lambda x, y: y, iterable_line)
+
+ iterable_line = itertools.imap(lambda x :len(x), iterable_line)
+ iterable_line = accumulate(iterable_line)
+ iterable_line = itertools.imap(
+ lambda value: float(value) / total / 2, iterable_line)
+ iterable_line = self._progressing(iterable_line)
+
+ # union
+ iterable = itertools.imap(lambda x, y: x, iterable_dic, iterable_line)
+
+ iterable = itertools.imap(self._init_extra_data, iterable)
+
+ return dict([(dic["id"], dic) for dic in iterable])
+
def _load_modified_idxfiles(self, datalist):
basedir = misc.get_thread_idx_dir_path(self.bbs_type)
ext = ".idx"
return id, title, res
return None
- def _load_subjecttxt(self, func):
+ def _load_subjecttxt(self):
lastmod = self.load_board_idx()
try:
lastmod = misc.httpdate_to_secs(lastmod)
subjecttxt_path = misc.get_board_subjecttxt_path(self.bbs_type)
try:
- for num, line_encoded \
- in itertools.izip(itertools.count(1),
- file(subjecttxt_path)):
- result = self._split_record(line_encoded)
- if result:
- id, title, res = result
- try:
- func(id, title, res, num, lastmod)
- except:
- traceback.print_exc()
- except IOError:
- traceback.print_exc()
+ total = os.path.getsize(subjecttxt_path)
+ except OSError:
+ total = -1
+
+ iterable = file(subjecttxt_path)
+
+ # split
+ iterable, iterable_len = itertools.tee(iterable)
- def _get_subjecttxt(self, func):
+ iterable_len = itertools.imap(lambda l: len(l), iterable_len)
+ iterable_len = accumulate(iterable_len)
+ iterable_len = itertools.imap(
+ lambda value: float(value) / total / 2 + 0.5, iterable_len)
+ iterable_len = self._progressing(iterable_len)
+
+ # union
+ iterable = itertools.imap(lambda x, y: x, iterable, iterable_len)
+
+ for num, line_encoded in itertools.izip(itertools.count(1), iterable):
+ result = self._split_record(line_encoded)
+ if result:
+ id, title, res = result
+ yield id, title, res, num, lastmod
+
+ def _get_subjecttxt(self):
# get subject.txt
lastmod = 0
subjecttxt_path = misc.get_board_subjecttxt_path(self.bbs_type)
- basedir = os.path.dirname(subjecttxt_path)
- if not os.path.isdir(basedir):
- os.makedirs(basedir)
- f = None
- try:
- f = file(subjecttxt_path, "w")
- except IOError:
- traceback.print_exc()
+ f = misc.FileWrap(subjecttxt_path, "w")
try:
- for num, line_encoded in itertools.izip(itertools.count(1),
- response):
- if f:
- try:
- f.write(line_encoded)
- except IOError:
- traceback.print_exc()
- result = self._split_record(line_encoded)
- if result:
- id, title, res = result
- try:
- func(id, title, res, num, lastmod)
- except:
- traceback.print_exc()
+ total = int(info["Content-Length"])
except:
- traceback.print_exc()
+ total = -1
+
+ def saving(line_encoded):
+ try:
+ f.write(line_encoded)
+ except IOError:
+ traceback.print_exc()
+ return line_encoded
+
+ iterable = response
+
+ # split
+ iterable, iterable_len = itertools.tee(iterable)
+
+ iterable_len = itertools.imap(lambda l: len(l), iterable_len)
+ iterable_len = accumulate(iterable_len)
+ iterable_len = itertools.imap(
+ lambda value: float(value) / total / 2 + 0.5, iterable_len)
+ iterable_len = self._progressing(iterable_len)
+
+ # union
+ iterable = itertools.imap(lambda x, y: x, iterable, iterable_len)
+
+ iterable = itertools.imap(saving, iterable)
+ iterable = itertools.izip(itertools.count(1), iterable)
+
+ for num, line_encoded in iterable:
+ result = self._split_record(line_encoded)
+ if result:
+ id, title, res = result
+ yield id, title, res, num, lastmod
- if f:
- f.close()
- f = None
+ f.close()
def load_board_idx(self):
lastmod = ""
OSDN Git Service
