OSDN Git Service

Average column is changed to float.
[fukui-no-namari/fukui-no-namari.git] / src / FukuiNoNamari / board_data.py
1 # Copyright (C) 2006 by Aiwota Programmer
2 # aiwotaprog@tetteke.tk
3 #
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 2 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software
16 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
17
18 import gobject
19 import os.path
20 import glob
21 import re
22 import codecs
23 import urllib2
24 import traceback
25 import itertools
26
27 import cachefile
28 import idxfile
29 import misc
30 from http_sub import HTTPRedirectHandler302
31
32 BOARD_DATA_INVALID_VALUE = 0
33 REG_EXPR = re.compile("(?P<id>.*).dat<>(?P<title>.*)\((?P<res>\d*)\)")
34
35
36 class BoardData:
37
38     def __init__(self, bbs_type):
39         self.bbs_type = bbs_type
40
41     def set_status(self, text):
42         pass
43
44     def _merge_new_thread(self, datalist, id, title, res, num, lastmod):
45         average = 0
46         if lastmod != 0:
47             try:
48                 start = int(id)
49             except ValueError:
50                 pass
51             else:
52                 # avoid the Last-Modified time of subject.txt and
53                 # the build time of thread is equal (zero division)
54                 dur = lastmod - start
55                 if dur == 0:
56                     average = 999999
57                 else:
58                     average = round(res * 60 * 60 * 24.0 / dur, 2)
59
60         if id in datalist:
61             item = datalist[id]
62             item["num"] = num
63             item["title"] = title
64             item["res"] = res
65             item["average"] = average
66         else:
67             datalist[id] = {"num": num, "title": title,
68                             "res": res, "lineCount": BOARD_DATA_INVALID_VALUE,
69                             "lastModified": "", "average": average}
70
71     def merge_local_subjecttxt(self, datalist):
72         f = lambda id, title, res, num, lastmod: \
73             self._merge_new_thread(datalist, id, title, res, num, lastmod)
74         self._load_subjecttxt(f)
75
76     def merge_remote_subjecttxt(self, datalist):
77         f = lambda id, title, res, num, lastmod: \
78             self._merge_new_thread(datalist, id, title, res, num, lastmod)
79         self._get_subjecttxt(f)
80
81     def _add_idx(self, datalist, id, dic):
82         datalist[id] = dic
83         dic["num"] = 0
84         dic["res"] = 0
85         dic["average"] = 0
86         
87     def load_idxfiles(self):
88         datalist = {}
89
90         def on_load_record(id, metadata_dic):
91             idxfile_path = misc.get_thread_idx_path(
92                 self.bbs_type.bbs_type, self.bbs_type.board, id)
93             if os.path.exists(idxfile_path):
94                 self._add_idx(datalist, id, metadata_dic)
95
96         print "load_cache"
97         cachefile.load_cache(
98             self.bbs_type.bbs_type, self.bbs_type.board, on_load_record)
99         print "load_idx"
100         self._load_modified_idxfiles(datalist)
101         print "save_cache"
102         cachefile.save_cache(
103             self.bbs_type.bbs_type, self.bbs_type.board, datalist)
104
105         return datalist
106
107     def _load_modified_idxfiles(self, datalist):
108         basedir = misc.get_thread_idx_dir_path(
109             self.bbs_type.bbs_type, self.bbs_type.board)
110         if os.path.isdir(basedir):
111             for idxfile_path in glob.glob(os.path.join(basedir, "*.idx")):
112                 thread_id, ext = os.path.splitext(
113                     os.path.basename(idxfile_path))
114                 idxlastModified = os.path.getmtime(idxfile_path)
115                 if thread_id not in datalist:
116                     print "new"
117                     dic = idxfile.load_idx(
118                         self.bbs_type.bbs_type, self.bbs_type.board, thread_id)
119                     #dic.pop("etag")
120                     dic["idxlastModified"] = idxlastModified
121                     self._add_idx(datalist, thread_id, dic)
122                 elif idxlastModified > datalist[thread_id]["idxlastModified"]:
123                     print "modified"
124                     datalist[thread_id]["idxlastModified"] = idxlastModified
125                     dic = idxfile.load_idx(
126                         self.bbs_type.bbs_type, self.bbs_type.board, thread_id)
127                     for name in idxfile.metadata_namelist:
128                         datalist[thread_id][name] = dic[name]
129
130     def _split_record(self, line):
131         m = REG_EXPR.match(line)
132         if m:
133             id = m.group("id")
134             title = m.group("title")
135             try:
136                 res = int(m.group("res"))
137             except ValueError:
138                 res = 0
139             return id, title, res
140         return None
141
142     def _load_subjecttxt(self, func):
143         lastmod = self.load_board_idx()
144         try:
145             lastmod = misc.httpdate_to_secs(lastmod)
146         except ValueError:
147             lastmod = 0
148
149         subjecttxt_path = misc.get_board_subjecttxt_path(
150             self.bbs_type.bbs_type, self.bbs_type.board)
151         try:
152             for num, line_encoded \
153                     in itertools.izip(itertools.count(1),
154                                       file(subjecttxt_path)):
155                 result = self._split_record(
156                     line_encoded.decode("cp932", "replace"))
157                 if result:
158                     id, title, res = result
159                     try:
160                         func(id, title, res, num, lastmod)
161                     except:
162                         traceback.print_exc()
163         except IOError:
164             traceback.print_exc()
165
166     def _get_subjecttxt(self, func):
167
168         # get subject.txt
169
170         opener = urllib2.build_opener(HTTPRedirectHandler302)
171         try:
172             response = opener.open(self.bbs_type.get_subject_txt_uri())
173         except urllib2.HTTPError, e:
174             print "%d %s" % (e.code, e.msg)
175             gobject.idle_add(self.set_status, "%d %s" % (e.code, e.msg))
176             print e.info()
177             print "switch to local"
178             self._load_subjecttxt(func)
179         except urllib2.URLError, e:
180             print e
181             gobject.idle_add(self.set_status, str(e))
182             print "switch to local"
183             self._load_subjecttxt(func)
184         else:
185             status = "%d %s" % (response.code, response.msg)
186             print status
187             gobject.idle_add(self.set_status, status)
188             info = response.info()
189             print info
190
191             lastmod = 0
192             if "Last-Modified" in info:
193                 _lastmod = info["Last-Modified"]
194                 self.save_board_idx(_lastmod)
195                 try:
196                     lastmod = misc.httpdate_to_secs(_lastmod)
197                 except ValueError:
198                     lastmod = 0
199
200             subjecttxt_path = misc.get_board_subjecttxt_path(
201                 self.bbs_type.bbs_type, self.bbs_type.board)
202             basedir = os.path.dirname(subjecttxt_path)
203             if not os.path.isdir(basedir):
204                 os.makedirs(basedir)
205             f = None
206             try:
207                 f = file(subjecttxt_path, "w")
208             except IOError:
209                 traceback.print_exc()
210
211             try:
212                 for num, line_encoded in itertools.izip(itertools.count(1),
213                                                         response):
214                     if f:
215                         try:
216                             f.write(line_encoded)
217                         except IOError:
218                             traceback.print_exc()
219                     result = self._split_record(
220                         line_encoded.decode("cp932", "replace"))
221                     if result:
222                         id, title, res = result
223                         try:
224                             func(id, title, res, num, lastmod)
225                         except:
226                             traceback.print_exc()
227             except:
228                 traceback.print_exc()
229
230             if f:
231                 f.close()
232                 f = None
233
234     def load_board_idx(self):
235         lastmod = ""
236         boardidxfile = misc.get_board_idx_path(
237             self.bbs_type.bbs_type, self.bbs_type.board)
238         try:
239             for line in file(boardidxfile):
240                 if line.startswith("lastModified="):
241                     lastmod = line[len("lastModified="):].rstrip("\n")
242                     break
243         except IOError:
244             traceback.print_exc()
245         return lastmod
246
247     def save_board_idx(self, lastmod):
248         if not lastmod:
249             return
250
251         boardidx_path = misc.get_board_idx_path(
252             self.bbs_type.bbs_type, self.bbs_type.board)
253         basedir = os.path.dirname(boardidx_path)
254         if not os.path.isdir(basedir):
255             os.makedirs(basedir)
256
257         f = file(boardidx_path, "w")
258         f.write("lastModified=" + lastmod + "\n")
259         f.close()