OSDN Git Service

Re imprement board_data.
authorAiwota Programmer <aiwotaprog@tetteke.tk>
Mon, 4 Sep 2006 06:31:40 +0000 (15:31 +0900)
committerAiwota Programmer <aiwotaprog@tetteke.tk>
Mon, 4 Sep 2006 06:31:40 +0000 (15:31 +0900)
src/FukuiNoNamari/board_data.py
src/FukuiNoNamari/board_window.py
src/FukuiNoNamari/subjecttxtfile.py [deleted file]

index 309ad72..b38b5c8 100644 (file)
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 
-import threading
 import gobject
 import os.path
 import glob
+import re
+import codecs
+import urllib2
+import traceback
+import itertools
 
-import subjecttxtfile
 import cachefile
 import idxfile
 import misc
+from http_sub import HTTPRedirectHandler302
 
 BOARD_DATA_INVALID_VALUE = 0
+REG_EXPR = re.compile("(?P<id>.*).dat<>(?P<title>.*)\((?P<res>\d*)\)")
 
-def merge_new_thread(oldlist, newlist, id, title, res):
-    num = len(newlist) + 1
-    if id in oldlist:
-        item = oldlist.pop(id)
-        item["num"] = num
-        item["title"] = title
-        item["res"] = res
-        newlist[id] = item
-    else:
-        newlist[id] = {"num": num, "title": title,
-                        "res": res, "lineCount": BOARD_DATA_INVALID_VALUE,
-                        "lastModified": ""}
-
-def merge_cached_thread(oldlist, newlist):
-    for id, value in oldlist.iteritems():
-        if value["lineCount"] != BOARD_DATA_INVALID_VALUE:
-            value["num"] = BOARD_DATA_INVALID_VALUE
-            value["res"] = BOARD_DATA_INVALID_VALUE
-            newlist[id] = value
-
-def merge_local_subjecttxt(bbs, board, datalist):
-    oldlist = datalist.copy()
-    datalist.clear()
-
-    h = lambda id,title,res: merge_new_thread(oldlist,datalist,id,title,res)
-    subjecttxtfile.load_subjecttxt(bbs, board, h)
-    merge_cached_thread(oldlist, datalist)
-
-def merge_subjecttxt(subjecttxt, datalist):
-    oldlist = datalist.copy()
-    datalist.clear()
-
-    h = lambda id,title,res: merge_new_thread(oldlist,datalist,id,title,res)
-    subjecttxtfile.analyze_subjecttxt(subjecttxt, h)
-    merge_cached_thread(oldlist, datalist)
-
-def load_idxfiles(bbs, board):
-    datalist = {}
-
-    def on_load_record(id, metadata_dic):
-        idxfile_path = misc.get_thread_idx_path(bbs, board, id)
-        if os.path.exists(idxfile_path):
-            datalist[id] = metadata_dic
-
-    print "load_cache"
-    cachefile.load_cache(bbs, board, on_load_record)
-    print "load_idx"
-    load_modified_idxfiles(bbs, board, datalist)
-    print "save_cache"
-    cachefile.save_cache(bbs, board, datalist)
-
-    return datalist
-
-def load_modified_idxfiles(bbs, board, datalist):
-    basedir = misc.get_thread_idx_dir_path(bbs, board)
-    if os.path.isdir(basedir):
-        for idxfile_path in glob.glob(os.path.join(basedir, "*.idx")):
-            thread_id, ext = os.path.splitext(os.path.basename(idxfile_path))
-            idxlastModified = os.path.getmtime(idxfile_path)
-            if thread_id not in datalist:
-                print "new"
-                dic = idxfile.load_idx(bbs, board, thread_id)
-                #dic.pop("etag")
-                dic["idxlastModified"] = idxlastModified
-                datalist[thread_id] = dic
-            elif idxlastModified > datalist[thread_id]["idxlastModified"]:
-                print "modified"
-                datalist[thread_id]["idxlastModified"] = idxlastModified
-                dic = idxfile.load_idx(bbs, board, thread_id)
-                for name in idxfile.metadata_namelist:
-                    datalist[thread_id][name] = dic[name]
-
-
-class LoadLocal(threading.Thread):
-    def __init__(self, bbs, board, on_end):
-        super(LoadLocal, self).__init__()
-        self.bbs = bbs
-        self.board = board
-        self.on_end = on_end
-
-    def run(self):
-        datalist = load_idxfiles(self.bbs, self.board)
-        merge_local_subjecttxt(self.bbs, self.board, datalist)
-        lastmod = subjecttxtfile.load_board_idx(self.bbs, self.board)
-        gobject.idle_add(self.on_end, datalist, lastmod)
-
-
-class GetRemote(threading.Thread):
-    def __init__(self, bbs, board, uri, on_end):
-        super(GetRemote, self).__init__()
-        self.bbs = bbs
-        self.board = board
-        self.uri = uri
-        self.on_end = on_end
-
-    def run(self):
-        print "start get subject.txt"
-        subjecttxt, lastmod = subjecttxtfile.get_subjecttxt(
-            self.bbs, self.board, self.uri)
-        datalist = load_idxfiles(self.bbs, self.board)
-        merge_subjecttxt(subjecttxt, datalist)
-        gobject.idle_add(self.on_end, datalist, lastmod)
+
+class BoardData:
+
+    def __init__(self, bbs_type):
+        self.bbs_type = bbs_type
+
+    def set_status(self, text):
+        pass
+
+    def _merge_new_thread(self, datalist, id, title, res, num, lastmod):
+        average = 0
+        if lastmod != 0:
+            try:
+                start = int(id)
+            except ValueError:
+                pass
+            else:
+                # avoid the Last-Modified time of subject.txt and
+                # the build time of thread is equal (zero division)
+                dur = lastmod - start
+                if dur == 0:
+                    average = 999999
+                else:
+                    average = (res * 60 * 60 * 24 / dur)
+
+        if id in datalist:
+            item = datalist[id]
+            item["num"] = num
+            item["title"] = title
+            item["res"] = res
+            item["average"] = average
+        else:
+            datalist[id] = {"num": num, "title": title,
+                            "res": res, "lineCount": BOARD_DATA_INVALID_VALUE,
+                            "lastModified": "", "average": average}
+
+    def merge_local_subjecttxt(self, datalist):
+        f = lambda id, title, res, num, lastmod: \
+            self._merge_new_thread(datalist, id, title, res, num, lastmod)
+        self._load_subjecttxt(f)
+
+    def merge_remote_subjecttxt(self, datalist):
+        f = lambda id, title, res, num, lastmod: \
+            self._merge_new_thread(datalist, id, title, res, num, lastmod)
+        self._get_subjecttxt(f)
+
+    def _add_idx(self, datalist, id, dic):
+        datalist[id] = dic
+        dic["num"] = 0
+        dic["res"] = 0
+        dic["average"] = 0
+        
+    def load_idxfiles(self):
+        datalist = {}
+
+        def on_load_record(id, metadata_dic):
+            idxfile_path = misc.get_thread_idx_path(
+                self.bbs_type.bbs_type, self.bbs_type.board, id)
+            if os.path.exists(idxfile_path):
+                self._add_idx(datalist, id, metadata_dic)
+
+        print "load_cache"
+        cachefile.load_cache(
+            self.bbs_type.bbs_type, self.bbs_type.board, on_load_record)
+        print "load_idx"
+        self._load_modified_idxfiles(datalist)
+        print "save_cache"
+        cachefile.save_cache(
+            self.bbs_type.bbs_type, self.bbs_type.board, datalist)
+
+        return datalist
+
+    def _load_modified_idxfiles(self, datalist):
+        basedir = misc.get_thread_idx_dir_path(
+            self.bbs_type.bbs_type, self.bbs_type.board)
+        if os.path.isdir(basedir):
+            for idxfile_path in glob.glob(os.path.join(basedir, "*.idx")):
+                thread_id, ext = os.path.splitext(
+                    os.path.basename(idxfile_path))
+                idxlastModified = os.path.getmtime(idxfile_path)
+                if thread_id not in datalist:
+                    print "new"
+                    dic = idxfile.load_idx(
+                        self.bbs_type.bbs_type, self.bbs_type.board, thread_id)
+                    #dic.pop("etag")
+                    dic["idxlastModified"] = idxlastModified
+                    self._add_idx(datalist, thread_id, dic)
+                elif idxlastModified > datalist[thread_id]["idxlastModified"]:
+                    print "modified"
+                    datalist[thread_id]["idxlastModified"] = idxlastModified
+                    dic = idxfile.load_idx(
+                        self.bbs_type.bbs_type, self.bbs_type.board, thread_id)
+                    for name in idxfile.metadata_namelist:
+                        datalist[thread_id][name] = dic[name]
+
+    def _split_record(self, line):
+        m = REG_EXPR.match(line)
+        if m:
+            id = m.group("id")
+            title = m.group("title")
+            try:
+                res = int(m.group("res"))
+            except ValueError:
+                res = 0
+            return id, title, res
+        return None
+
+    def _load_subjecttxt(self, func):
+        lastmod = self.load_board_idx()
+        try:
+            lastmod = misc.httpdate_to_secs(lastmod)
+        except ValueError:
+            lastmod = 0
+
+        subjecttxt_path = misc.get_board_subjecttxt_path(
+            self.bbs_type.bbs_type, self.bbs_type.board)
+        try:
+            for num, line_encoded \
+                    in itertools.izip(itertools.count(1),
+                                      file(subjecttxt_path)):
+                result = self._split_record(
+                    line_encoded.decode("cp932", "replace"))
+                if result:
+                    id, title, res = result
+                    try:
+                        func(id, title, res, num, lastmod)
+                    except:
+                        traceback.print_exc()
+        except IOError:
+            traceback.print_exc()
+
+    def _get_subjecttxt(self, func):
+
+        # get subject.txt
+
+        opener = urllib2.build_opener(HTTPRedirectHandler302)
+        try:
+            response = opener.open(self.bbs_type.get_subject_txt_uri())
+        except urllib2.HTTPError, e:
+            print "%d %s" % (e.code, e.msg)
+            gobject.idle_add(self.set_status, "%d %s" % (e.code, e.msg))
+            print e.info()
+            print "switch to local"
+            self._load_subjecttxt(func)
+        except urllib2.URLError, e:
+            print e
+            gobject.idle_add(self.set_status, str(e))
+            print "switch to local"
+            self._load_subjecttxt(func)
+        else:
+            status = "%d %s" % (response.code, response.msg)
+            print status
+            gobject.idle_add(self.set_status, status)
+            info = response.info()
+            print info
+
+            lastmod = 0
+            if "Last-Modified" in info:
+                _lastmod = info["Last-Modified"]
+                self.save_board_idx(_lastmod)
+                try:
+                    lastmod = misc.httpdate_to_secs(_lastmod)
+                except ValueError:
+                    lastmod = 0
+
+            subjecttxt_path = misc.get_board_subjecttxt_path(
+                self.bbs_type.bbs_type, self.bbs_type.board)
+            basedir = os.path.dirname(subjecttxt_path)
+            if not os.path.isdir(basedir):
+                os.makedirs(basedir)
+            f = None
+            try:
+                f = file(subjecttxt_path, "w")
+            except IOError:
+                traceback.print_exc()
+
+            try:
+                for num, line_encoded in itertools.izip(itertools.count(1),
+                                                        response):
+                    if f:
+                        try:
+                            f.write(line_encoded)
+                        except IOError:
+                            traceback.print_exc()
+                    result = self._split_record(
+                        line_encoded.decode("cp932", "replace"))
+                    if result:
+                        id, title, res = result
+                        try:
+                            func(id, title, res, num, lastmod)
+                        except:
+                            traceback.print_exc()
+            except:
+                traceback.print_exc()
+
+            if f:
+                f.close()
+                f = None
+
+    def load_board_idx(self):
+        lastmod = ""
+        boardidxfile = misc.get_board_idx_path(
+            self.bbs_type.bbs_type, self.bbs_type.board)
+        try:
+            for line in file(boardidxfile):
+                if line.startswith("lastModified="):
+                    lastmod = line[len("lastModified="):].rstrip("\n")
+                    break
+        except IOError:
+            traceback.print_exc()
+        return lastmod
+
+    def save_board_idx(self, lastmod):
+        if not lastmod:
+            return
+
+        boardidx_path = misc.get_board_idx_path(
+            self.bbs_type.bbs_type, self.bbs_type.board)
+        basedir = os.path.dirname(boardidx_path)
+        if not os.path.isdir(basedir):
+            os.makedirs(basedir)
+
+        f = file(boardidx_path, "w")
+        f.write("lastModified=" + lastmod + "\n")
+        f.close()
index a597538..1c3875b 100644 (file)
@@ -33,6 +33,7 @@ from BbsType import bbs_type_judge_uri
 import config
 import session
 import winwrapbase
+from misc import ThreadInvoker
 
 GLADE_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                          "..", "data")
@@ -53,11 +54,12 @@ def open_board(uri, update=False):
         winwrap.load(update)
 
 
-class WinWrap(winwrapbase.WinWrapBase):
+class WinWrap(winwrapbase.WinWrapBase, board_data.BoardData):
 
     def __init__(self, uri):
 
         self.bbs_type = bbs_type_judge_uri.get_type(uri)
+        board_data.BoardData.__init__(self, self.bbs_type)
         self.bbs = self.bbs_type.bbs_type
         self.host = self.bbs_type.host
         self.board = self.bbs_type.board
@@ -125,6 +127,9 @@ class WinWrap(winwrapbase.WinWrapBase):
 
         self.created()
 
+    def set_status(self, text):
+        self.statusbar.set_status(text)
+
     def destroy(self):
         self.save()
         self.window.destroy()
@@ -194,10 +199,7 @@ class WinWrap(winwrapbase.WinWrapBase):
         self.destroy()
 
     def on_refresh_activate(self, widget):
-        t = board_data.GetRemote(
-            self.bbs, self.board, self.bbs_type.get_subject_txt_uri(),
-            self.update_datastore)
-        t.start()
+        self.load(True)
 
     def on_column_clicked(self, treeviewcolumn, column_name):
         model = self.treeview.get_model()
@@ -253,33 +255,13 @@ class WinWrap(winwrapbase.WinWrapBase):
                 self.popupmenu.popup(None, None, None, event.button, time)
             return 1
 
-    def update_datastore(self, datalist, lastmod):
+    def update_datastore(self, datalist):
         print "reflesh datastore"
 
-        try:
-            lastmod = misc.httpdate_to_secs(lastmod)
-        except:
-            lastmod = 0
-
-        time_start = time.time()
         list_list = []
         for id, dic in datalist.iteritems():
             dic["id"] = id
 
-            # average
-            if lastmod == 0 or dic["num"] == 0:
-                dic["average"] = 0
-            else:
-                res = dic["res"]
-                start = int(id)
-                # avoid the Last-Modified time of subject.txt and
-                # the build time of thread is equal (zero division)
-                dur = lastmod - start
-                if dur == 0:
-                    dic["average"] = 999999
-                else:
-                    dic["average"] = int(res * 60 * 60 * 24 / dur)
-
             # lastModified
             httpdate = dic["lastModified"]
             try:
@@ -287,7 +269,7 @@ class WinWrap(winwrapbase.WinWrapBase):
                 dic["lastModified"] = secs
             except ValueError:
                 dic["lastModified"] = 0
-            
+
             list_list.append(dic)
 
         model = self.treeview.get_model()
@@ -299,8 +281,6 @@ class WinWrap(winwrapbase.WinWrapBase):
         self.reset_sort_indicator()
 
         print "end"
-        time_end = time.time()
-        print time_end - time_start
 
     def on_thread_idx_updated(self, thread_uri, idx_dic):
         if not thread_uri or not idx_dic:
@@ -325,18 +305,27 @@ class WinWrap(winwrapbase.WinWrapBase):
             model.modify_row(idx_dic)
 
     def load(self, update=False):
+
+        def load_local():
+            datalist = self.load_idxfiles()
+            self.merge_local_subjecttxt(datalist)
+            gobject.idle_add(self.update_datastore, datalist)
+
+        def get_remote():
+            print "start get subject.txt"
+            datalist = self.load_idxfiles()
+            self.merge_remote_subjecttxt(datalist)
+            gobject.idle_add(self.update_datastore, datalist)
+
         sbj_path = misc.get_board_subjecttxt_path(
             self.bbs_type.bbs_type, self.bbs_type.board)
         sbj_exists = os.path.exists(sbj_path)
 
         if update or not sbj_exists:
-            t = board_data.GetRemote(
-                self.bbs, self.board, self.bbs_type.get_subject_txt_uri(),
-                self.update_datastore)
+            t = ThreadInvoker(lambda *args: -1, get_remote)
             t.start()
         else:
-            t = board_data.LoadLocal(
-                self.bbs, self.board, self.update_datastore)
+            t = ThreadInvoker(lambda *args: -1, load_local)
             t.start()
 
     def save(self):
diff --git a/src/FukuiNoNamari/subjecttxtfile.py b/src/FukuiNoNamari/subjecttxtfile.py
deleted file mode 100644 (file)
index 49b92f3..0000000
+++ /dev/null
@@ -1,104 +0,0 @@
-# Copyright (C) 2006 by Aiwota Programmer
-# aiwotaprog@tetteke.tk
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-
-import re
-import os.path
-import codecs
-import urllib2
-from misc import get_board_subjecttxt_path
-from misc import get_board_idx_path
-from http_sub import HTTPRedirectHandler302
-
-REG_EXPR = re.compile("(?P<id>.*).dat<>(?P<title>.*)\((?P<res>\d*)\)")
-
-def expr(line, func):
-    m = REG_EXPR.match(line)
-    if m:
-        id = m.group("id")
-        title = m.group("title")
-        try:
-            res = int(m.group("res"))
-        except:
-            res = 0
-        func(id, title, res)
-
-def analyze_subjecttxt(subjecttxt, func):
-    lines = subjecttxt.splitlines()
-    for line in lines:
-        expr(line, func)
-    
-def load_subjecttxt(bbs, board, func):
-    subjecttxt_path = get_board_subjecttxt_path(bbs, board)
-    if not os.path.exists(subjecttxt_path):
-        return
-
-    f = open(subjecttxt_path)
-    subjecttxt_encoded = f.read()
-    f.close()
-
-    analyze_subjecttxt(subjecttxt_encoded.decode("cp932", "replace"), func)
-
-def get_subjecttxt(bbs, board, uri):
-
-    # get subject.txt
-
-    opener = urllib2.build_opener(HTTPRedirectHandler302)
-    response = opener.open(uri)
-    info = response.info()
-    print info
-    subjecttxt_encoded = response.read()
-
-    # save subject.txt
-    subjecttxt_path = get_board_subjecttxt_path(bbs, board)
-    
-    basedir = os.path.dirname(subjecttxt_path)
-    if not os.path.isdir(basedir):
-        os.makedirs(basedir)
-
-    f = open(subjecttxt_path, "w")
-    f.write(subjecttxt_encoded)
-    f.close()
-
-    lastmod = None
-    if "Last-Modified" in info:
-        lastmod = info["Last-Modified"]
-        save_board_idx(bbs, board, lastmod)
-
-    return subjecttxt_encoded.decode("cp932", "replace"), lastmod
-
-def load_board_idx(bbs, board):
-    lastmod = ""
-    boardidxfile = get_board_idx_path(bbs, board)
-    if os.path.exists(boardidxfile):
-        for line in file(boardidxfile):
-            if line.startswith("lastModified="):
-                lastmod = line[len("lastModified="):].rstrip("\n")
-                break
-    return lastmod
-
-def save_board_idx(bbs, board, lastmod):
-    if not lastmod:
-        return
-
-    boardidx_path = get_board_idx_path(bbs, board)
-    basedir = os.path.dirname(boardidx_path)
-    if not os.path.isdir(basedir):
-        os.makedirs(basedir)
-
-    f = file(boardidx_path, "w")
-    f.write("lastModified=" + lastmod + "\n")
-    f.close()