OSDN Git Service

298df3b9db0b836b699079701373fb2daecd1db9
[fukui-no-namari/fukui-no-namari.git] / src / FukuiNoNamari / board_data.py
1 # Copyright (C) 2006 by Aiwota Programmer
2 # aiwotaprog@tetteke.tk
3 #
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 2 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software
16 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
17
18 import gobject
19 import os.path
20 import glob
21 import codecs
22 import urllib2
23 import traceback
24 import itertools
25
26 import cachefile
27 import idxfile
28 import misc
29 import config
30 from http_sub import HTTPRedirectHandler302, HTTPDebugHandler
31
32 BOARD_DATA_INVALID_VALUE = 0
33
34
35 class BoardData:
36
37     def __init__(self, bbs_type):
38         self.bbs_type = bbs_type
39
40     def set_status(self, text):
41         pass
42
43     def _merge_new_thread(self, datalist, id, title, res, num, lastmod):
44         average = 0
45         if lastmod != 0:
46             try:
47                 start = int(id)
48             except ValueError:
49                 pass
50             else:
51                 # avoid the Last-Modified time of subject.txt and
52                 # the build time of thread is equal (zero division)
53                 dur = lastmod - start
54                 if dur == 0:
55                     average = 999999
56                 else:
57                     average = round(res * 60 * 60 * 24.0 / dur, 2)
58
59         if id in datalist:
60             item = datalist[id]
61             if item["num"]:
62                 # already exists in datalist and num is not 0, then this thread
63                 # is duplicate in subject.txt.
64                 # ignore second.
65                 pass
66             else:
67                 item["num"] = num
68                 item["title"] = title
69                 item["res"] = res
70                 item["average"] = average
71         else:
72             datalist[id] = {"id": id, "num": num, "title": title,
73                             "res": res, "lineCount": BOARD_DATA_INVALID_VALUE,
74                             "lastModified": "", "average": average}
75
76     def merge_local_subjecttxt(self, datalist):
77         f = lambda id, title, res, num, lastmod: \
78             self._merge_new_thread(datalist, id, title, res, num, lastmod)
79         self._load_subjecttxt(f)
80
81     def merge_remote_subjecttxt(self, datalist):
82         f = lambda id, title, res, num, lastmod: \
83             self._merge_new_thread(datalist, id, title, res, num, lastmod)
84         self._get_subjecttxt(f)
85
86     def _init_extra_data(self, dic):
87         dic["num"] = 0
88         dic["res"] = 0
89         dic["average"] = 0
90         return dic
91
92     def load_idxfiles(self):
93         print "load_cache"
94         iterable = cachefile.load_cache(self.bbs_type)
95         iterable = itertools.imap(self._init_extra_data, iterable)
96         datalist = dict([(dic["id"], dic) for dic in iterable])
97
98         print "load_idx"
99         self._load_modified_idxfiles(datalist)
100         print "save_cache"
101         cachefile.save_cache(self.bbs_type, datalist)
102
103         return datalist
104
105     def _load_modified_idxfiles(self, datalist):
106         basedir = misc.get_thread_idx_dir_path(self.bbs_type)
107         ext = ".idx"
108         exist_key_set = set()
109         if os.path.isdir(basedir):
110             for idxfile_path in glob.glob(os.path.join(basedir, "*"+ext)):
111                 basename = os.path.basename(idxfile_path)
112                 thread_id = basename[:len(ext)*-1]
113                 try:
114                     idxlastModified = os.path.getmtime(idxfile_path)
115                 except OSError:
116                     continue
117                 exist_key_set.add(thread_id)
118                 if thread_id not in datalist:
119                     print "new", thread_id
120                     bbs_type_for_thread = self.bbs_type.clone_with_thread(
121                         thread_id)
122                     dic = idxfile.load_idx(bbs_type_for_thread)
123                     dic["id"] = thread_id
124                     dic["idxlastModified"] = idxlastModified
125                     dic = self._init_extra_data(dic)
126                     datalist[thread_id] = dic
127                 elif idxlastModified > datalist[thread_id]["idxlastModified"]:
128                     print "modified", thread_id
129                     bbs_type_for_thread = self.bbs_type.clone_with_thread(
130                         thread_id)
131                     datalist[thread_id]["idxlastModified"] = idxlastModified
132                     dic = idxfile.load_idx(bbs_type_for_thread)
133                     for key, value in dic.iteritems():
134                         datalist[thread_id][key] = value
135
136         # delete from datalist if idx file does not exist.
137         for key in datalist.keys():
138             if key not in exist_key_set:
139                 del datalist[key]
140                 print "del", key
141
142     def _split_record(self, line_encoded):
143         line = line_encoded.decode(self.bbs_type.encoding, "replace")
144         m = self.bbs_type.subject_reg.match(line)
145         if m:
146             id = m.group("id")
147             title = m.group("title")
148             try:
149                 res = int(m.group("res"))
150             except ValueError:
151                 res = 0
152             return id, title, res
153         return None
154
155     def _load_subjecttxt(self, func):
156         lastmod = self.load_board_idx()
157         try:
158             lastmod = misc.httpdate_to_secs(lastmod)
159         except ValueError:
160             lastmod = 0
161
162         subjecttxt_path = misc.get_board_subjecttxt_path(self.bbs_type)
163         try:
164             for num, line_encoded \
165                     in itertools.izip(itertools.count(1),
166                                       file(subjecttxt_path)):
167                 result = self._split_record(line_encoded)
168                 if result:
169                     id, title, res = result
170                     try:
171                         func(id, title, res, num, lastmod)
172                     except:
173                         traceback.print_exc()
174         except IOError:
175             traceback.print_exc()
176
177     def _get_subjecttxt(self, func):
178
179         # get subject.txt
180
181         opener = urllib2.build_opener(HTTPRedirectHandler302, HTTPDebugHandler)
182         request = urllib2.Request(self.bbs_type.get_subject_txt_uri())
183         request.add_header("User-agent", config.User_Agent)
184         try:
185             response = opener.open(request)
186         except urllib2.HTTPError, e:
187             gobject.idle_add(self.set_status, "%d %s" % (e.code, e.msg))
188             print "switch to local"
189             self._load_subjecttxt(func)
190         except urllib2.URLError, e:
191             print e
192             gobject.idle_add(self.set_status, str(e))
193             print "switch to local"
194             self._load_subjecttxt(func)
195         else:
196             status = "%d %s" % (response.code, response.msg)
197             gobject.idle_add(self.set_status, status)
198             info = response.info()
199
200             lastmod = 0
201             if "Last-Modified" in info:
202                 _lastmod = info["Last-Modified"]
203                 self.save_board_idx(_lastmod)
204                 try:
205                     lastmod = misc.httpdate_to_secs(_lastmod)
206                 except ValueError:
207                     lastmod = 0
208
209             subjecttxt_path = misc.get_board_subjecttxt_path(self.bbs_type)
210             basedir = os.path.dirname(subjecttxt_path)
211             if not os.path.isdir(basedir):
212                 os.makedirs(basedir)
213             f = None
214             try:
215                 f = file(subjecttxt_path, "w")
216             except IOError:
217                 traceback.print_exc()
218
219             try:
220                 for num, line_encoded in itertools.izip(itertools.count(1),
221                                                         response):
222                     if f:
223                         try:
224                             f.write(line_encoded)
225                         except IOError:
226                             traceback.print_exc()
227                     result = self._split_record(line_encoded)
228                     if result:
229                         id, title, res = result
230                         try:
231                             func(id, title, res, num, lastmod)
232                         except:
233                             traceback.print_exc()
234             except:
235                 traceback.print_exc()
236
237             if f:
238                 f.close()
239                 f = None
240
241     def load_board_idx(self):
242         lastmod = ""
243         boardidxfile = misc.get_board_idx_path(self.bbs_type)
244         try:
245             for line in file(boardidxfile):
246                 if line.startswith("lastModified="):
247                     lastmod = line[len("lastModified="):].rstrip("\n")
248                     break
249         except IOError:
250             traceback.print_exc()
251         return lastmod
252
253     def save_board_idx(self, lastmod):
254         if not lastmod:
255             return
256
257         boardidx_path = misc.get_board_idx_path(self.bbs_type)
258         basedir = os.path.dirname(boardidx_path)
259         if not os.path.isdir(basedir):
260             os.makedirs(basedir)
261
262         f = file(boardidx_path, "w")
263         f.write("lastModified=" + lastmod + "\n")
264         f.close()