1 # Copyright (C) 2006 by Aiwota Programmer
2 # aiwotaprog@tetteke.tk
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 2 of the License, or
7 # (at your option) any later version.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software
16 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
30 from http_sub import HTTPRedirectHandler302
32 BOARD_DATA_INVALID_VALUE = 0
33 REG_EXPR = re.compile("(?P<id>.*).dat<>(?P<title>.*)\((?P<res>\d*)\)")
38 def __init__(self, bbs_type):
39 self.bbs_type = bbs_type
41 def set_status(self, text):
44 def _merge_new_thread(self, datalist, id, title, res, num, lastmod):
52 # avoid the Last-Modified time of subject.txt and
53 # the build time of thread is equal (zero division)
58 average = round(res * 60 * 60 * 24.0 / dur, 2)
65 item["average"] = average
67 datalist[id] = {"num": num, "title": title,
68 "res": res, "lineCount": BOARD_DATA_INVALID_VALUE,
69 "lastModified": "", "average": average}
71 def merge_local_subjecttxt(self, datalist):
72 f = lambda id, title, res, num, lastmod: \
73 self._merge_new_thread(datalist, id, title, res, num, lastmod)
74 self._load_subjecttxt(f)
76 def merge_remote_subjecttxt(self, datalist):
77 f = lambda id, title, res, num, lastmod: \
78 self._merge_new_thread(datalist, id, title, res, num, lastmod)
79 self._get_subjecttxt(f)
81 def _add_idx(self, datalist, id, dic):
87 def load_idxfiles(self):
90 def on_load_record(id, metadata_dic):
91 idxfile_path = misc.get_thread_idx_path(
92 self.bbs_type.bbs_type, self.bbs_type.board, id)
93 if os.path.exists(idxfile_path):
94 self._add_idx(datalist, id, metadata_dic)
98 self.bbs_type.bbs_type, self.bbs_type.board, on_load_record)
100 self._load_modified_idxfiles(datalist)
102 cachefile.save_cache(
103 self.bbs_type.bbs_type, self.bbs_type.board, datalist)
107 def _load_modified_idxfiles(self, datalist):
108 basedir = misc.get_thread_idx_dir_path(
109 self.bbs_type.bbs_type, self.bbs_type.board)
110 if os.path.isdir(basedir):
111 for idxfile_path in glob.glob(os.path.join(basedir, "*.idx")):
112 thread_id, ext = os.path.splitext(
113 os.path.basename(idxfile_path))
114 idxlastModified = os.path.getmtime(idxfile_path)
115 if thread_id not in datalist:
117 dic = idxfile.load_idx(
118 self.bbs_type.bbs_type, self.bbs_type.board, thread_id)
120 dic["idxlastModified"] = idxlastModified
121 self._add_idx(datalist, thread_id, dic)
122 elif idxlastModified > datalist[thread_id]["idxlastModified"]:
124 datalist[thread_id]["idxlastModified"] = idxlastModified
125 dic = idxfile.load_idx(
126 self.bbs_type.bbs_type, self.bbs_type.board, thread_id)
127 for name in idxfile.metadata_namelist:
128 datalist[thread_id][name] = dic[name]
130 def _split_record(self, line):
131 m = REG_EXPR.match(line)
134 title = m.group("title")
136 res = int(m.group("res"))
139 return id, title, res
142 def _load_subjecttxt(self, func):
143 lastmod = self.load_board_idx()
145 lastmod = misc.httpdate_to_secs(lastmod)
149 subjecttxt_path = misc.get_board_subjecttxt_path(
150 self.bbs_type.bbs_type, self.bbs_type.board)
152 for num, line_encoded \
153 in itertools.izip(itertools.count(1),
154 file(subjecttxt_path)):
155 result = self._split_record(
156 line_encoded.decode("cp932", "replace"))
158 id, title, res = result
160 func(id, title, res, num, lastmod)
162 traceback.print_exc()
164 traceback.print_exc()
166 def _get_subjecttxt(self, func):
170 opener = urllib2.build_opener(HTTPRedirectHandler302)
172 response = opener.open(self.bbs_type.get_subject_txt_uri())
173 except urllib2.HTTPError, e:
174 print "%d %s" % (e.code, e.msg)
175 gobject.idle_add(self.set_status, "%d %s" % (e.code, e.msg))
177 print "switch to local"
178 self._load_subjecttxt(func)
179 except urllib2.URLError, e:
181 gobject.idle_add(self.set_status, str(e))
182 print "switch to local"
183 self._load_subjecttxt(func)
185 status = "%d %s" % (response.code, response.msg)
187 gobject.idle_add(self.set_status, status)
188 info = response.info()
192 if "Last-Modified" in info:
193 _lastmod = info["Last-Modified"]
194 self.save_board_idx(_lastmod)
196 lastmod = misc.httpdate_to_secs(_lastmod)
200 subjecttxt_path = misc.get_board_subjecttxt_path(
201 self.bbs_type.bbs_type, self.bbs_type.board)
202 basedir = os.path.dirname(subjecttxt_path)
203 if not os.path.isdir(basedir):
207 f = file(subjecttxt_path, "w")
209 traceback.print_exc()
212 for num, line_encoded in itertools.izip(itertools.count(1),
216 f.write(line_encoded)
218 traceback.print_exc()
219 result = self._split_record(
220 line_encoded.decode("cp932", "replace"))
222 id, title, res = result
224 func(id, title, res, num, lastmod)
226 traceback.print_exc()
228 traceback.print_exc()
234 def load_board_idx(self):
236 boardidxfile = misc.get_board_idx_path(
237 self.bbs_type.bbs_type, self.bbs_type.board)
239 for line in file(boardidxfile):
240 if line.startswith("lastModified="):
241 lastmod = line[len("lastModified="):].rstrip("\n")
244 traceback.print_exc()
247 def save_board_idx(self, lastmod):
251 boardidx_path = misc.get_board_idx_path(
252 self.bbs_type.bbs_type, self.bbs_type.board)
253 basedir = os.path.dirname(boardidx_path)
254 if not os.path.isdir(basedir):
257 f = file(boardidx_path, "w")
258 f.write("lastModified=" + lastmod + "\n")