src/FukuiNoNamari/board_data.py

   1 # Copyright (C) 2006 by Aiwota Programmer
   2 # aiwotaprog@tetteke.tk
   3 #
   4 # This program is free software; you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation; either version 2 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program; if not, write to the Free Software
  16 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17
  18 import gobject
  19 import os.path
  20 import glob
  21 import codecs
  22 import urllib2
  23 import traceback
  24 import itertools
  25
  26 import cachefile
  27 import idxfile
  28 import misc
  29 import config
  30 from http_sub import HTTPRedirectHandler302, HTTPDebugHandler
  31
  32 BOARD_DATA_INVALID_VALUE = 0
  33
  34
  35 class BoardData:
  36
  37     def __init__(self, bbs_type):
  38         self.bbs_type = bbs_type
  39
  40     def set_status(self, text):
  41         pass
  42
  43     def _merge_new_thread(self, datalist, id, title, res, num, lastmod):
  44         average = 0
  45         if lastmod != 0:
  46             try:
  47                 start = int(id)
  48             except ValueError:
  49                 pass
  50             else:
  51                 # avoid the Last-Modified time of subject.txt and
  52                 # the build time of thread is equal (zero division)
  53                 dur = lastmod - start
  54                 if dur == 0:
  55                     average = 999999
  56                 else:
  57                     average = round(res * 60 * 60 * 24.0 / dur, 2)
  58
  59         if id in datalist:
  60             item = datalist[id]
  61             if item["num"]:
  62                 # already exists in datalist and num is not 0, then this thread
  63                 # is duplicate in subject.txt.
  64                 # ignore second.
  65                 pass
  66             else:
  67                 item["num"] = num
  68                 item["title"] = title
  69                 item["res"] = res
  70                 item["average"] = average
  71         else:
  72             datalist[id] = {"id": id, "num": num, "title": title,
  73                             "res": res, "lineCount": BOARD_DATA_INVALID_VALUE,
  74                             "lastModified": "", "average": average}
  75
  76     def merge_local_subjecttxt(self, datalist):
  77         f = lambda id, title, res, num, lastmod: \
  78             self._merge_new_thread(datalist, id, title, res, num, lastmod)
  79         self._load_subjecttxt(f)
  80
  81     def merge_remote_subjecttxt(self, datalist):
  82         f = lambda id, title, res, num, lastmod: \
  83             self._merge_new_thread(datalist, id, title, res, num, lastmod)
  84         self._get_subjecttxt(f)
  85
  86     def _init_extra_data(self, dic):
  87         dic["num"] = 0
  88         dic["res"] = 0
  89         dic["average"] = 0
  90         return dic
  91
  92     def load_idxfiles(self):
  93         datalist = {}
  94
  95         def on_load_record(id, metadata_dic):
  96             datalist[id] = self._init_extra_data(metadata_dic)
  97
  98         print "load_cache"
  99         cachefile.load_cache(self.bbs_type, on_load_record)
 100         print "load_idx"
 101         self._load_modified_idxfiles(datalist)
 102         print "save_cache"
 103         cachefile.save_cache(self.bbs_type, datalist)
 104
 105         return datalist
 106
 107     def _load_modified_idxfiles(self, datalist):
 108         basedir = misc.get_thread_idx_dir_path(self.bbs_type)
 109         ext = ".idx"
 110         exist_key_set = set()
 111         if os.path.isdir(basedir):
 112             for idxfile_path in glob.glob(os.path.join(basedir, "*"+ext)):
 113                 basename = os.path.basename(idxfile_path)
 114                 thread_id = basename[:len(ext)*-1]
 115                 try:
 116                     idxlastModified = os.path.getmtime(idxfile_path)
 117                 except OSError:
 118                     continue
 119                 exist_key_set.add(thread_id)
 120                 if thread_id not in datalist:
 121                     print "new", thread_id
 122                     bbs_type_for_thread = self.bbs_type.clone_with_thread(
 123                         thread_id)
 124                     dic = idxfile.load_idx(bbs_type_for_thread)
 125                     dic["id"] = thread_id
 126                     dic["idxlastModified"] = idxlastModified
 127                     dic = self._init_extra_data(dic)
 128                     datalist[thread_id] = dic
 129                 elif idxlastModified > datalist[thread_id]["idxlastModified"]:
 130                     print "modified", thread_id
 131                     bbs_type_for_thread = self.bbs_type.clone_with_thread(
 132                         thread_id)
 133                     datalist[thread_id]["idxlastModified"] = idxlastModified
 134                     dic = idxfile.load_idx(bbs_type_for_thread)
 135                     for key, value in dic.iteritems():
 136                         datalist[thread_id][key] = value
 137
 138         # delete from datalist if idx file does not exist.
 139         for key in datalist.keys():
 140             if key not in exist_key_set:
 141                 del datalist[key]
 142                 print "del", key
 143
 144     def _split_record(self, line_encoded):
 145         line = line_encoded.decode(self.bbs_type.encoding, "replace")
 146         m = self.bbs_type.subject_reg.match(line)
 147         if m:
 148             id = m.group("id")
 149             title = m.group("title")
 150             try:
 151                 res = int(m.group("res"))
 152             except ValueError:
 153                 res = 0
 154             return id, title, res
 155         return None
 156
 157     def _load_subjecttxt(self, func):
 158         lastmod = self.load_board_idx()
 159         try:
 160             lastmod = misc.httpdate_to_secs(lastmod)
 161         except ValueError:
 162             lastmod = 0
 163
 164         subjecttxt_path = misc.get_board_subjecttxt_path(self.bbs_type)
 165         try:
 166             for num, line_encoded \
 167                     in itertools.izip(itertools.count(1),
 168                                       file(subjecttxt_path)):
 169                 result = self._split_record(line_encoded)
 170                 if result:
 171                     id, title, res = result
 172                     try:
 173                         func(id, title, res, num, lastmod)
 174                     except:
 175                         traceback.print_exc()
 176         except IOError:
 177             traceback.print_exc()
 178
 179     def _get_subjecttxt(self, func):
 180
 181         # get subject.txt
 182
 183         opener = urllib2.build_opener(HTTPRedirectHandler302, HTTPDebugHandler)
 184         request = urllib2.Request(self.bbs_type.get_subject_txt_uri())
 185         request.add_header("User-agent", config.User_Agent)
 186         try:
 187             response = opener.open(request)
 188         except urllib2.HTTPError, e:
 189             gobject.idle_add(self.set_status, "%d %s" % (e.code, e.msg))
 190             print "switch to local"
 191             self._load_subjecttxt(func)
 192         except urllib2.URLError, e:
 193             print e
 194             gobject.idle_add(self.set_status, str(e))
 195             print "switch to local"
 196             self._load_subjecttxt(func)
 197         else:
 198             status = "%d %s" % (response.code, response.msg)
 199             gobject.idle_add(self.set_status, status)
 200             info = response.info()
 201
 202             lastmod = 0
 203             if "Last-Modified" in info:
 204                 _lastmod = info["Last-Modified"]
 205                 self.save_board_idx(_lastmod)
 206                 try:
 207                     lastmod = misc.httpdate_to_secs(_lastmod)
 208                 except ValueError:
 209                     lastmod = 0
 210
 211             subjecttxt_path = misc.get_board_subjecttxt_path(self.bbs_type)
 212             basedir = os.path.dirname(subjecttxt_path)
 213             if not os.path.isdir(basedir):
 214                 os.makedirs(basedir)
 215             f = None
 216             try:
 217                 f = file(subjecttxt_path, "w")
 218             except IOError:
 219                 traceback.print_exc()
 220
 221             try:
 222                 for num, line_encoded in itertools.izip(itertools.count(1),
 223                                                         response):
 224                     if f:
 225                         try:
 226                             f.write(line_encoded)
 227                         except IOError:
 228                             traceback.print_exc()
 229                     result = self._split_record(line_encoded)
 230                     if result:
 231                         id, title, res = result
 232                         try:
 233                             func(id, title, res, num, lastmod)
 234                         except:
 235                             traceback.print_exc()
 236             except:
 237                 traceback.print_exc()
 238
 239             if f:
 240                 f.close()
 241                 f = None
 242
 243     def load_board_idx(self):
 244         lastmod = ""
 245         boardidxfile = misc.get_board_idx_path(self.bbs_type)
 246         try:
 247             for line in file(boardidxfile):
 248                 if line.startswith("lastModified="):
 249                     lastmod = line[len("lastModified="):].rstrip("\n")
 250                     break
 251         except IOError:
 252             traceback.print_exc()
 253         return lastmod
 254
 255     def save_board_idx(self, lastmod):
 256         if not lastmod:
 257             return
 258
 259         boardidx_path = misc.get_board_idx_path(self.bbs_type)
 260         basedir = os.path.dirname(boardidx_path)
 261         if not os.path.isdir(basedir):
 262             os.makedirs(basedir)
 263
 264         f = file(boardidx_path, "w")
 265         f.write("lastModified=" + lastmod + "\n")
 266         f.close()