src/FukuiNoNamari/board_data.py

   1 # Copyright (C) 2006 by Aiwota Programmer
   2 # aiwotaprog@tetteke.tk
   3 #
   4 # This program is free software; you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation; either version 2 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program; if not, write to the Free Software
  16 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17
  18 import gobject
  19 import gtk
  20 import os.path
  21 import glob
  22 import codecs
  23 import urllib2
  24 import traceback
  25 import itertools
  26
  27 import cachefile
  28 import idxfile
  29 import misc
  30 import config
  31 from http_sub import HTTPRedirectHandler302, HTTPDebugHandler
  32
  33 BOARD_DATA_INVALID_VALUE = 0
  34
  35 def accumulate(iterable, initial_value=0):
  36     sum_value = initial_value
  37     for value in iterable:
  38         sum_value += value
  39         yield sum_value
  40
  41 def follow(iterable, under_value=0):
  42     before = under_value
  43     for item in iterable:
  44         yield before, item
  45         before = item
  46
  47 class BoardData:
  48
  49     def __init__(self, bbs_type):
  50         self.bbs_type = bbs_type
  51
  52     def set_status(self, text):
  53         pass
  54
  55     def set_fraction(self, fraction):
  56         pass
  57
  58     def _merge_new_thread(self, datalist, id, title, res, num, lastmod):
  59         average = 0
  60         if lastmod != 0:
  61             try:
  62                 start = int(id)
  63             except ValueError:
  64                 pass
  65             else:
  66                 # avoid the Last-Modified time of subject.txt and
  67                 # the build time of thread is equal (zero division)
  68                 dur = lastmod - start
  69                 if dur == 0:
  70                     average = 999999
  71                 else:
  72                     average = round(res * 60 * 60 * 24.0 / dur, 2)
  73
  74         if id in datalist:
  75             item = datalist[id]
  76             if item["num"]:
  77                 # already exists in datalist and num is not 0, then this thread
  78                 # is duplicate in subject.txt.
  79                 # ignore second.
  80                 pass
  81             else:
  82                 item["num"] = num
  83                 item["title"] = title
  84                 item["res"] = res
  85                 item["average"] = average
  86         else:
  87             datalist[id] = {"id": id, "num": num, "title": title,
  88                             "res": res, "lineCount": BOARD_DATA_INVALID_VALUE,
  89                             "lastModified": "", "average": average}
  90
  91     def merge_local_subjecttxt(self, datalist):
  92         iterable = self._load_subjecttxt()
  93
  94         for id, title, res, num, lastmod in iterable:
  95             self._merge_new_thread(datalist, id, title, res, num, lastmod)
  96
  97         status = "Complete subject file."
  98         gobject.idle_add(self.set_status, status)
  99
 100     def merge_remote_subjecttxt(self, datalist):
 101         iterable = self._get_subjecttxt()
 102
 103         for id, title, res, num, lastmod in iterable:
 104             self._merge_new_thread(datalist, id, title, res, num, lastmod)
 105
 106     def _init_extra_data(self, dic):
 107         dic["num"] = 0
 108         dic["res"] = 0
 109         dic["average"] = 0
 110         return dic
 111
 112     def _progressing(self, iterable):
 113         for before, fraction in follow(iterable):
 114             if int(before*10) != int(fraction*10):
 115                 gtk.threads_enter()
 116                 try:
 117                     self.set_fraction(fraction)
 118                 finally:
 119                     gtk.threads_leave()
 120             yield fraction
 121
 122     def load_idxfiles(self):
 123         print "load_cache"
 124         datalist = self._load_cache()
 125         print "load_idx"
 126         self._load_modified_idxfiles(datalist)
 127         print "save_cache"
 128         cachefile.save_cache(self.bbs_type, datalist)
 129
 130         status = "Complete index files."
 131         gobject.idle_add(self.set_status, status)
 132         return datalist
 133
 134     def _load_cache(self):
 135         try:
 136             total = os.path.getsize(misc.get_board_cache_path(self.bbs_type))
 137         except OSError:
 138             total = -1
 139
 140         iterable = cachefile.load_cache(self.bbs_type)
 141
 142         # split
 143         iterable_dic, iterable_line = itertools.tee(iterable)
 144         iterable_dic = itertools.starmap(lambda x, y: x, iterable_dic)
 145         iterable_line = itertools.starmap(lambda x, y: y, iterable_line)
 146
 147         iterable_line = itertools.imap(lambda x :len(x), iterable_line)
 148         iterable_line = accumulate(iterable_line)
 149         iterable_line = itertools.imap(
 150             lambda value: float(value) / total / 5 * 2, iterable_line)
 151         iterable_line = self._progressing(iterable_line)
 152
 153         # union
 154         iterable = itertools.imap(lambda x, y: x, iterable_dic, iterable_line)
 155
 156         iterable = itertools.imap(self._init_extra_data, iterable)
 157
 158         return dict([(dic["id"], dic) for dic in iterable])
 159
 160     def _load_modified_idxfiles(self, datalist):
 161         ext = ".idx"
 162
 163         def id_and_lastmod(file_path):
 164             thread_id = os.path.basename(file_path)[:len(ext)*-1]
 165             try:
 166                 idxlastModified = os.path.getmtime(file_path)
 167                 return thread_id, idxlastModified
 168             except OSError:
 169                 pass
 170
 171         def _do_new_thread(thread_id, idxlastModified):
 172             print "new", thread_id
 173
 174             dic = idxfile.load_idx(self.bbs_type.clone_with_thread(thread_id))
 175             dic["id"] = thread_id
 176             dic["idxlastModified"] = idxlastModified
 177             dic = self._init_extra_data(dic)
 178             datalist[thread_id] = dic
 179             return thread_id, idxlastModified
 180
 181         def _do_modified_thread(thread_id, idxlastModified):
 182             print "modified", thread_id
 183
 184             datalist[thread_id]["idxlastModified"] = idxlastModified
 185             dic = idxfile.load_idx(self.bbs_type.clone_with_thread(thread_id))
 186             for key, value in dic.iteritems():
 187                 datalist[thread_id][key] = value
 188             return thread_id, idxlastModified
 189
 190         def new_or_modified_thread(thread_id, idxlastModified):
 191             if thread_id not in datalist:
 192                 return _do_new_thread(thread_id, idxlastModified)
 193             elif idxlastModified > datalist[thread_id]["idxlastModified"]:
 194                 return _do_modified_thread(thread_id, idxlastModified)
 195             return thread_id, idxlastModified
 196
 197         basedir = misc.get_thread_idx_dir_path(self.bbs_type)
 198
 199         filelist = glob.glob(os.path.join(basedir, "*"+ext))
 200         total = len(filelist)
 201
 202         iterable = filelist
 203
 204         # split
 205         iterable, iterable_count = itertools.tee(iterable)
 206
 207         iterable_count = itertools.izip(itertools.count(1), iterable_count)
 208         iterable_count = itertools.starmap(lambda x, y: x, iterable_count)
 209         iterable_count = itertools.imap(
 210             lambda x: float(x)/total/10 + 0.4, iterable_count)
 211         iterable_count = self._progressing(iterable_count)
 212
 213         # union
 214         iterable = itertools.imap(lambda x, y: x, iterable, iterable_count)
 215
 216         iterable = itertools.imap(id_and_lastmod, iterable)
 217         iterable = itertools.ifilter(None, iterable)
 218         iterable = itertools.starmap(new_or_modified_thread, iterable)
 219         exist_key_set = frozenset([x for x, y in iterable])
 220
 221         # delete from datalist if idx file does not exist.
 222         datalist_key_set = frozenset(datalist.iterkeys())
 223         delete_key_set = datalist_key_set - exist_key_set
 224         for key in delete_key_set:
 225             del datalist[key]
 226             print "del", key
 227
 228     def _split_record(self, line_encoded):
 229         line = line_encoded.decode(self.bbs_type.encoding, "replace")
 230         m = self.bbs_type.subject_reg.match(line)
 231         if m:
 232             id = m.group("id")
 233             title = m.group("title")
 234             try:
 235                 res = int(m.group("res"))
 236             except ValueError:
 237                 res = 0
 238             return id, title, res
 239         return None
 240
 241     def _load_subjecttxt(self):
 242         lastmod = self.load_board_idx()
 243         try:
 244             lastmod = misc.httpdate_to_secs(lastmod)
 245         except ValueError:
 246             lastmod = 0
 247
 248         subjecttxt_path = misc.get_board_subjecttxt_path(self.bbs_type)
 249         try:
 250             total = os.path.getsize(subjecttxt_path)
 251         except OSError:
 252             total = -1
 253
 254         iterable = file(subjecttxt_path)
 255
 256         # split
 257         iterable, iterable_len = itertools.tee(iterable)
 258
 259         iterable_len = itertools.imap(lambda l: len(l), iterable_len)
 260         iterable_len = accumulate(iterable_len)
 261         iterable_len = itertools.imap(
 262             lambda value: float(value) / total / 2 + 0.5, iterable_len)
 263         iterable_len = self._progressing(iterable_len)
 264
 265         # union
 266         iterable = itertools.imap(lambda x, y: x, iterable, iterable_len)
 267
 268         for num, line_encoded in itertools.izip(itertools.count(1), iterable):
 269             result = self._split_record(line_encoded)
 270             if result:
 271                 id, title, res = result
 272                 yield id, title, res, num, lastmod
 273
 274     def _get_subjecttxt(self):
 275
 276         # get subject.txt
 277
 278         opener = urllib2.build_opener(HTTPRedirectHandler302, HTTPDebugHandler)
 279         request = urllib2.Request(self.bbs_type.get_subject_txt_uri())
 280         request.add_header("User-agent", config.User_Agent)
 281         try:
 282             response = opener.open(request)
 283         except urllib2.HTTPError, e:
 284             gobject.idle_add(self.set_status, "%d %s" % (e.code, e.msg))
 285             print "switch to local"
 286             self._load_subjecttxt(func)
 287         except urllib2.URLError, e:
 288             print e
 289             gobject.idle_add(self.set_status, str(e))
 290             print "switch to local"
 291             self._load_subjecttxt(func)
 292         else:
 293             status = "%d %s" % (response.code, response.msg)
 294             gobject.idle_add(self.set_status, status)
 295             info = response.info()
 296
 297             lastmod = 0
 298             if "Last-Modified" in info:
 299                 _lastmod = info["Last-Modified"]
 300                 self.save_board_idx(_lastmod)
 301                 try:
 302                     lastmod = misc.httpdate_to_secs(_lastmod)
 303                 except ValueError:
 304                     lastmod = 0
 305
 306             subjecttxt_path = misc.get_board_subjecttxt_path(self.bbs_type)
 307             f = misc.FileWrap(subjecttxt_path, "w")
 308
 309             try:
 310                 total = int(info["Content-Length"])
 311             except:
 312                 total = -1
 313
 314             def saving(line_encoded):
 315                 try:
 316                     f.write(line_encoded)
 317                 except IOError:
 318                     traceback.print_exc()
 319                 return line_encoded
 320
 321             iterable = response
 322
 323             # split
 324             iterable, iterable_len = itertools.tee(iterable)
 325
 326             iterable_len = itertools.imap(lambda l: len(l), iterable_len)
 327             iterable_len = accumulate(iterable_len)
 328             iterable_len = itertools.imap(
 329                 lambda value: float(value) / total / 2 + 0.5, iterable_len)
 330             iterable_len = self._progressing(iterable_len)
 331
 332             # union
 333             iterable = itertools.imap(lambda x, y: x, iterable, iterable_len)
 334
 335             iterable = itertools.imap(saving, iterable)
 336             iterable = itertools.izip(itertools.count(1), iterable)
 337
 338             for num, line_encoded in iterable:
 339                 result = self._split_record(line_encoded)
 340                 if result:
 341                     id, title, res = result
 342                     yield id, title, res, num, lastmod
 343
 344             f.close()
 345
 346     def load_board_idx(self):
 347         lastmod = ""
 348         boardidxfile = misc.get_board_idx_path(self.bbs_type)
 349         try:
 350             for line in file(boardidxfile):
 351                 if line.startswith("lastModified="):
 352                     lastmod = line[len("lastModified="):].rstrip("\n")
 353                     break
 354         except IOError:
 355             traceback.print_exc()
 356         return lastmod
 357
 358     def save_board_idx(self, lastmod):
 359         if not lastmod:
 360             return
 361
 362         boardidx_path = misc.get_board_idx_path(self.bbs_type)
 363         basedir = os.path.dirname(boardidx_path)
 364         if not os.path.isdir(basedir):
 365             os.makedirs(basedir)
 366
 367         f = file(boardidx_path, "w")
 368         f.write("lastModified=" + lastmod + "\n")
 369         f.close()