OSDN Git Service

Multithreads are abandoned. Alternatly, The asyncore substitutes.(#16776) master
authorAiwota Programmer <aiwotaprog@tetteke.tk>
Mon, 25 May 2009 13:01:22 +0000 (22:01 +0900)
committerAiwota Programmer <aiwotaprog@tetteke.tk>
Mon, 25 May 2009 13:01:22 +0000 (22:01 +0900)
src/FukuiNoNamari/board_data.py
src/FukuiNoNamari/board_window.py
src/FukuiNoNamari/misc.py
src/FukuiNoNamari/network_manager.py [new file with mode: 0644]
src/FukuiNoNamari/session.py
src/FukuiNoNamari/thread_window.py
src/fukui-no-namari

index 0f41c98..48cc21c 100644 (file)
@@ -23,15 +23,21 @@ import codecs
 import urllib2
 import traceback
 import itertools
 import urllib2
 import traceback
 import itertools
+from StringIO import StringIO
 
 import cachefile
 import idxfile
 import misc
 import config
 from http_sub import HTTPRedirectHandler302, HTTPDebugHandler
 
 import cachefile
 import idxfile
 import misc
 import config
 from http_sub import HTTPRedirectHandler302, HTTPDebugHandler
+import network_manager
 
 BOARD_DATA_INVALID_VALUE = 0
 
 
 BOARD_DATA_INVALID_VALUE = 0
 
+
+class NothingToDoException: pass
+
+
 def accumulate(iterable, initial_value=0):
     sum_value = initial_value
     for value in iterable:
 def accumulate(iterable, initial_value=0):
     sum_value = initial_value
     for value in iterable:
@@ -48,6 +54,7 @@ class BoardData:
 
     def __init__(self, bbs_type):
         self.bbs_type = bbs_type
 
     def __init__(self, bbs_type):
         self.bbs_type = bbs_type
+        self.lastmod = ""
 
     def set_status(self, text):
         pass
 
     def set_status(self, text):
         pass
@@ -89,25 +96,19 @@ class BoardData:
                             "lastModified": 0, "average": average, "oldRes": 0}
 
     def merge_local_subjecttxt(self, datalist):
                             "lastModified": 0, "average": average, "oldRes": 0}
 
     def merge_local_subjecttxt(self, datalist):
-        try:
-            for id, title, res, num, lastmod in self._load_subjecttxt():
-                self._merge_new_thread(datalist, id, title, res, num, lastmod)
-        except IOError:
-            pass
-        except:
-            tracebakc.print_exc()
-        else:
-            status = "Complete subject file."
-            gobject.idle_add(self.set_status, status)
+        for id, title, res, num, lastmod in self._load_subjecttxt():
+            self._merge_new_thread(datalist, id, title, res, num, lastmod)
+            yield
+        status = "Complete subject file."
+        lastmod = self.load_board_idx()
+        if lastmod:
+            self.lastmod = lastmod
+            status = "%s [%s]" % (status, lastmod)
+        self.set_status(status)
 
 
-    def merge_remote_subjecttxt(self, datalist):
-        try:
-            for id, title, res, num, lastmod in self._get_subjecttxt():
-                self._merge_new_thread(datalist, id, title, res, num, lastmod)
-        except IOError:
-            pass
-        except:
-            traceback.print_exc()
+    def merge_remote_subjecttxt(self, datalist, iterable):
+        for id, title, res, num, lastmod in iterable:
+            yield self._merge_new_thread(datalist, id, title, res, num, lastmod)
 
     def _init_extra_data(self, dic):
         dic["num"] = 0
 
     def _init_extra_data(self, dic):
         dic["num"] = 0
@@ -119,11 +120,7 @@ class BoardData:
     def _progressing(self, iterable):
         for before, fraction in follow(iterable):
             if int(before*10) != int(fraction*10):
     def _progressing(self, iterable):
         for before, fraction in follow(iterable):
             if int(before*10) != int(fraction*10):
-                gtk.gdk.threads_enter()
-                try:
-                    self.set_fraction(fraction)
-                finally:
-                    gtk.gdk.threads_leave()
+                self.set_fraction(fraction)
             yield fraction
 
     def _modify_dict(self, item_dict):
             yield fraction
 
     def _modify_dict(self, item_dict):
@@ -137,30 +134,31 @@ class BoardData:
             item_dict["lastModified"] = secs
         return item_dict
 
             item_dict["lastModified"] = secs
         return item_dict
 
-    def load_idxfiles(self):
-        print "load_cache"
+    def load_idxfiles(self, datalist):
         try:
         try:
-            datalist = self._load_cache()
+            for i in self._load_cache(datalist):
+                yield
         except IOError:
         except IOError:
-            datalist = {}
-        print "load_idx"
-        self._load_modified_idxfiles(datalist)
-        print "save_cache"
-        try:
-            self._save_cache(datalist)
-        except IOError:
-            traceback.print_exc()
+            # the ".cache" file does not exist.
+            pass
+        else:
+            self.set_status("Complete load cache.")
+
+        for i in self._load_modified_idxfiles(datalist):
+            yield
+
+        self.set_status("Complete load idx files.")
+
+        self._save_cache(datalist)
+        # do not wait to save
 
         # adjustment after cache save, before load subject.txt
         iterable = datalist.itervalues()
         iterable = itertools.imap(self._modify_dict, iterable)
 
         # adjustment after cache save, before load subject.txt
         iterable = datalist.itervalues()
         iterable = itertools.imap(self._modify_dict, iterable)
-        for i in iterable: -1
-
-        status = "Complete index files."
-        gobject.idle_add(self.set_status, status)
-        return datalist
+        for i in iterable:
+            yield
 
 
-    def _load_cache(self):
+    def _load_cache(self, datalist):
         try:
             total = os.path.getsize(misc.get_board_cache_path(self.bbs_type))
         except OSError:
         try:
             total = os.path.getsize(misc.get_board_cache_path(self.bbs_type))
         except OSError:
@@ -185,7 +183,9 @@ class BoardData:
 
         iterable = itertools.imap(self._init_extra_data, iterable)
 
 
         iterable = itertools.imap(self._init_extra_data, iterable)
 
-        return dict([(dic["id"], dic) for dic in iterable])
+        for dic in iterable:
+            datalist[dic["id"]] = dic
+            yield
 
     def _load_modified_idxfiles(self, datalist):
         ext = ".idx"
 
     def _load_modified_idxfiles(self, datalist):
         ext = ".idx"
@@ -246,7 +246,13 @@ class BoardData:
         iterable = itertools.imap(id_and_lastmod, iterable)
         iterable = itertools.ifilter(None, iterable)
         iterable = itertools.starmap(new_or_modified_thread, iterable)
         iterable = itertools.imap(id_and_lastmod, iterable)
         iterable = itertools.ifilter(None, iterable)
         iterable = itertools.starmap(new_or_modified_thread, iterable)
-        exist_key_set = frozenset([x for x, y in iterable])
+
+        exist_key_set = set()
+        iterable = itertools.starmap(lambda x, y: exist_key_set.add(x),
+            iterable)
+
+        for i in iterable:
+            yield
 
         # delete from datalist if idx file does not exist.
         datalist_key_set = frozenset(datalist.iterkeys())
 
         # delete from datalist if idx file does not exist.
         datalist_key_set = frozenset(datalist.iterkeys())
@@ -254,12 +260,13 @@ class BoardData:
         for key in delete_key_set:
             del datalist[key]
             print "del", key
         for key in delete_key_set:
             del datalist[key]
             print "del", key
+            yield
 
     def _save_cache(self, datalist):
 
     def _save_cache(self, datalist):
-        iterable = datalist.iteritems()
+        iterable = datalist.items()
         iterable = cachefile.dict_to_formatted(iterable)
         c_file = misc.FileWrap(misc.get_board_cache_path(self.bbs_type), "w")
         iterable = cachefile.dict_to_formatted(iterable)
         c_file = misc.FileWrap(misc.get_board_cache_path(self.bbs_type), "w")
-        c_file.writelines(iterable)
+        misc.chain(c_file.write, c_file.close, iterable)
 
     def _split_record(self, line_encoded):
         line = line_encoded.decode(self.bbs_type.encoding, "replace")
 
     def _split_record(self, line_encoded):
         line = line_encoded.decode(self.bbs_type.encoding, "replace")
@@ -312,78 +319,85 @@ class BoardData:
 
         return main_process()
 
 
         return main_process()
 
-    def _get_subjecttxt(self):
-
-        # get subject.txt
-
-        opener = urllib2.build_opener(HTTPRedirectHandler302, HTTPDebugHandler)
+    def get_subjecttxt(self, on_received):
+        uri = self.bbs_type.get_subject_txt_uri()
         request = urllib2.Request(self.bbs_type.get_subject_txt_uri())
         request.add_header("User-agent", config.User_Agent)
         request = urllib2.Request(self.bbs_type.get_subject_txt_uri())
         request.add_header("User-agent", config.User_Agent)
+        if self.lastmod:
+            request.add_header("If-modified-since", self.lastmod)
+
         try:
         try:
-            response = opener.open(request)
-        except urllib2.HTTPError, e:
-            gobject.idle_add(self.set_status, "%d %s" % (e.code, e.msg))
-            print "switch to local"
-            return self._load_subjecttxt()
-        except urllib2.URLError, e:
-            print e
-            gobject.idle_add(self.set_status, str(e))
-            print "switch to local"
-            return self._load_subjecttxt()
+            network_manager.request_get(uri, request.headers, on_received)
+        except network_manager.BusyException:
+            self.set_status("The network is busy. Try later.")
+            raise NothingToDoException()
         else:
         else:
-            status = "%d %s" % (response.code, response.msg)
-            gobject.idle_add(self.set_status, status)
-            info = response.info()
+            self.set_status("GET...")
 
 
-            lastmod = 0
-            if "Last-Modified" in info:
-                _lastmod = info["Last-Modified"]
-                self.save_board_idx(_lastmod)
-                try:
-                    lastmod = misc.httpdate_to_secs(_lastmod)
-                except ValueError:
-                    lastmod = 0
+    def progress_response(self, response):
+        status = response.status
+        headers = response.headers
+        message = StringIO(response.message)
+
+        if "last-modified".capitalize() in headers:
+            self.set_status("%s [%s]" % (status,
+                headers["last-modified".capitalize()]))
+        else:
+            self.set_status("%s" % status)
+
+        version, code, msg = status.split(None, 2)
+        code = int(code)
+        if code != 200:
+            raise misc.StopChainException()
+
+        lastmod = 0
+        if "last-modified".capitalize() in headers:
+            _lastmod = headers["last-modified".capitalize()]
+            self.lastmod = _lastmod
+            self.save_board_idx(_lastmod)
+            try:
+                lastmod = misc.httpdate_to_secs(_lastmod)
+            except ValueError:
+                lastmod = 0
+
+        subjecttxt_path = misc.get_board_subjecttxt_path(self.bbs_type)
+        f = misc.FileWrap(subjecttxt_path, "w")
 
 
-            subjecttxt_path = misc.get_board_subjecttxt_path(self.bbs_type)
-            f = misc.FileWrap(subjecttxt_path, "w")
+        try:
+            total = int(headers["content-length".capitalize()])
+        except:
+            total = -1
 
 
+        def saving(line_encoded):
             try:
             try:
-                total = int(info["Content-Length"])
-            except:
-                total = -1
-
-            def saving(line_encoded):
-                try:
-                    f.write(line_encoded)
-                except IOError:
-                    traceback.print_exc()
-                return line_encoded
-
-            iterable = response
-
-            # split
-            iterable, iterable_len = itertools.tee(iterable)
-
-            iterable_len = itertools.imap(lambda l: len(l), iterable_len)
-            iterable_len = accumulate(iterable_len)
-            iterable_len = itertools.imap(
-                lambda value: float(value) / total, iterable_len)
-            iterable_len = self._progressing(iterable_len)
-
-            # union
-            iterable = itertools.imap(lambda x, y: x, iterable, iterable_len)
-
-            iterable = itertools.imap(saving, iterable)
-            iterable = itertools.izip(itertools.count(1), iterable)
-
-            def main_process():
-                for num, line_encoded in iterable:
-                    result = self._split_record(line_encoded)
-                    if result:
-                        id, title, res = result
-                        yield id, title, res, num, lastmod
-
-            return main_process()
+                f.write(line_encoded)
+            except IOError:
+                traceback.print_exc()
+            return line_encoded
+
+        iterable = message
+
+        # split
+        iterable, iterable_len = itertools.tee(iterable)
+
+        iterable_len = itertools.imap(lambda l: len(l), iterable_len)
+        iterable_len = accumulate(iterable_len)
+        iterable_len = itertools.imap(
+            lambda value: float(value) / total, iterable_len)
+        iterable_len = self._progressing(iterable_len)
+
+        # union
+        iterable = itertools.imap(lambda x, y: x, iterable, iterable_len)
+
+        iterable = itertools.imap(saving, iterable)
+        iterable = itertools.izip(itertools.count(1), iterable)
+
+        for num, line_encoded in iterable:
+            result = self._split_record(line_encoded)
+            if result:
+                id, title, res = result
+                yield id, title, res, num, lastmod
+        f.close()
 
     def load_board_idx(self):
         lastmod = ""
 
     def load_board_idx(self):
         lastmod = ""
index d03e684..16ad091 100644 (file)
@@ -35,7 +35,6 @@ from BbsType import bbs_type_judge_uri
 import config
 import session
 import winwrapbase
 import config
 import session
 import winwrapbase
-from misc import ThreadInvoker
 import bookmark_list
 import bookmark_window
 import board_plugins
 import bookmark_list
 import bookmark_window
 import board_plugins
@@ -66,6 +65,8 @@ class WinWrap(winwrapbase.WinWrapBase, board_data.BoardData):
 
     def __init__(self, uri):
 
 
     def __init__(self, uri):
 
+        self.lock_obj = False
+
         self.bbs_type = bbs_type_judge_uri.get_type(uri)
         board_data.BoardData.__init__(self, self.bbs_type)
 
         self.bbs_type = bbs_type_judge_uri.get_type(uri)
         board_data.BoardData.__init__(self, self.bbs_type)
 
@@ -105,6 +106,19 @@ class WinWrap(winwrapbase.WinWrapBase, board_data.BoardData):
             "bonobodockitem_filterbar")
         self.entry_filterbar = self.widget_tree.get_widget("entry_filterbar")
 
             "bonobodockitem_filterbar")
         self.entry_filterbar = self.widget_tree.get_widget("entry_filterbar")
 
+    def lock(self):
+        if self.lock_obj:
+            print "locked, try later."
+            return False
+        else:
+            print "get lock"
+            self.lock_obj = True
+            return True
+
+    def un_lock(self):
+        self.lock_obj = False
+        print "unlock"
+
     def set_status(self, text):
         self.statusbar.set_status(text)
 
     def set_status(self, text):
         self.statusbar.set_status(text)
 
@@ -177,11 +191,16 @@ class WinWrap(winwrapbase.WinWrapBase, board_data.BoardData):
 
     def load(self, update=False):
 
 
     def load(self, update=False):
 
+        def on_end():
+            self.progress.set_fraction(0.0)
+            self.progress.hide()
+            self.un_lock()
+
         def set_id(thread_id, item_dict):
             item_dict["id"] = thread_id
             return item_dict
 
         def set_id(thread_id, item_dict):
             item_dict["id"] = thread_id
             return item_dict
 
-        def conv_dictdict_to_listdict(dictdict):
+        def conv_dictdict_to_listdict(dictdict, listdict):
             key_iter = dictdict.iterkeys()
             value_iter = dictdict.itervalues()
             iterable = itertools.imap(set_id, key_iter, value_iter)
             key_iter = dictdict.iterkeys()
             value_iter = dictdict.itervalues()
             iterable = itertools.imap(set_id, key_iter, value_iter)
@@ -190,25 +209,41 @@ class WinWrap(winwrapbase.WinWrapBase, board_data.BoardData):
             iterable = itertools.ifilter(
                 lambda dic: dic["num"] > 0 or dic["lineCount"] > 0, iterable)
 
             iterable = itertools.ifilter(
                 lambda dic: dic["num"] > 0 or dic["lineCount"] > 0, iterable)
 
-            return [item_dict for item_dict in iterable]
+            iterable = itertools.imap(listdict.append, iterable)
+            for i in iterable:
+                yield
+
+            self.update_datastore(listdict)
 
         def load_local():
 
         def load_local():
-            datalist = self.load_idxfiles()
-            self.merge_local_subjecttxt(datalist)
-            new_list = conv_dictdict_to_listdict(datalist)
-            gobject.idle_add(self.update_datastore, new_list)
+            datalist = {}
+            new_lst = []
+            iterable = self.load_idxfiles(datalist)
+            iterable = itertools.chain(iterable,
+                self.merge_local_subjecttxt(datalist))
+            iterable = itertools.chain(iterable,
+                conv_dictdict_to_listdict(datalist, new_lst))
+
+            misc.chain(lambda *args: None, on_end, iterable)
 
         def get_remote(datalist):
 
         def get_remote(datalist):
-            print "start get subject.txt"
-            self.merge_remote_subjecttxt(datalist)
-            new_list = conv_dictdict_to_listdict(datalist)
-            gobject.idle_add(self.update_datastore, new_list)
 
 
-        def on_end():
-            def reset_progress():
-                self.progress.set_fraction(0.0)
-                self.progress.hide()
-            gobject.idle_add(reset_progress)
+            def on_received(response):
+                new_lst = []
+                iterable = self.progress_response(response)
+                iterable = self.merge_remote_subjecttxt(datalist, iterable)
+                iterable = itertools.chain(iterable,
+                    conv_dictdict_to_listdict(datalist, new_lst))
+                misc.chain(lambda *args: None, on_end, iterable)
+
+            print "start get subject.txt"
+            try:
+                self.get_subjecttxt(on_received)
+            except board_data.NothingToDoException:
+                on_end()
+            except:
+                on_end()
+                self.set_status(str(sys.exc_info()))
 
         def deep_copy():
             def init_some(dic):
 
         def deep_copy():
             def init_some(dic):
@@ -230,6 +265,9 @@ class WinWrap(winwrapbase.WinWrapBase, board_data.BoardData):
                 new_dict = {}
             return new_dict
 
                 new_dict = {}
             return new_dict
 
+        if not self.lock():
+            return
+
         sbj_path = misc.get_board_subjecttxt_path(self.bbs_type)
         sbj_exists = os.path.exists(sbj_path)
 
         sbj_path = misc.get_board_subjecttxt_path(self.bbs_type)
         sbj_exists = os.path.exists(sbj_path)
 
@@ -237,11 +275,9 @@ class WinWrap(winwrapbase.WinWrapBase, board_data.BoardData):
 
         if update or not sbj_exists:
             new_dict = deep_copy()
 
         if update or not sbj_exists:
             new_dict = deep_copy()
-            t = ThreadInvoker(on_end, lambda *x: get_remote(new_dict))
-            t.start()
+            get_remote(new_dict)
         else:
         else:
-            t = ThreadInvoker(on_end, load_local)
-            t.start()
+            load_local()
 
     def save(self):
         try:
 
     def save(self):
         try:
index ca852b0..9bf7d98 100644 (file)
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 
+import pygtk
+pygtk.require('2.0')
+import gtk
+import gobject
 import os.path
 import re
 import os.path
 import re
-import threading
 from datetime import tzinfo, timedelta, datetime
 import itertools
 from datetime import tzinfo, timedelta, datetime
 import itertools
+import traceback
 
 import config
 from BbsType import bbs_type_exception
 
 import config
 from BbsType import bbs_type_exception
@@ -142,19 +146,6 @@ def httpdate_to_secs(httpdate):
         raise ValueError
 
 
         raise ValueError
 
 
-class ThreadInvoker(threading.Thread):
-    def __init__(self, on_end, *methods):
-        super(ThreadInvoker, self).__init__()
-        self.on_end = on_end
-        self.methods = methods
-    def run(self):
-        try:
-            for m in self.methods:
-                m()
-        finally:
-            self.on_end()
-
-
 class FileWrap:
     def __init__(self, path, mode="a+"):
         self.__file = None
 class FileWrap:
     def __init__(self, path, mode="a+"):
         self.__file = None
@@ -205,3 +196,26 @@ def tabbed_to_dict_generator(tabbed, sep="\t"):
 def tabbed_to_dict(tabbed, sep="\t"):
     """Creates a dict from key equal value pairs seperated with tab"""
     return dict([pair for pair in tabbed_to_dict_generator(tabbed, sep)])
 def tabbed_to_dict(tabbed, sep="\t"):
     """Creates a dict from key equal value pairs seperated with tab"""
     return dict([pair for pair in tabbed_to_dict_generator(tabbed, sep)])
+
+
+class StopChainException: pass
+
+def _do_chain(function, on_end, iterable):
+    try:
+        value = iterable.next()
+        function(value)
+    except StopIteration:
+        # normal end.
+        on_end()
+    except StopChainException:
+        # normal end.
+        on_end()
+    except:
+        # an error is occurred.
+        on_end()
+        traceback.print_exc()
+    else:
+        gobject.idle_add(_do_chain, function, on_end, iterable)
+
+def chain(function, on_end, iterable):
+    gobject.idle_add(_do_chain, function, on_end, iterable)
diff --git a/src/FukuiNoNamari/network_manager.py b/src/FukuiNoNamari/network_manager.py
new file mode 100644 (file)
index 0000000..83d75f9
--- /dev/null
@@ -0,0 +1,166 @@
+# Copyright (C) 2009 by Aiwota Programmer
+# aiwotaprog@tetteke.tk
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+import gobject
+import asyncore
+from asyncore import dispatcher
+import socket
+#import time
+import rfc822
+from StringIO import StringIO
+from urlparse import urlparse
+import os
+
+
+class BusyException: pass
+
+
+class Response:
+
+    def __init__(self):
+        self.status = ""
+        self.headers = {}
+        self.message = ""
+
+
+class HttpClientGet(dispatcher):
+
+    def __init__(self, uri, headers, on_received):
+        dispatcher.__init__(self)
+
+        self.fd = StringIO()
+        self.closed = False
+        self.on_received = on_received
+        self.response = Response()
+
+        port = 80
+        ret = urlparse(uri)
+        if ret.scheme != "http":
+            print "%s is not http" % uri
+            return
+        if ret.port:
+            port = ret.port
+
+        headers["Host".capitalize()] = ret.hostname
+        request = 'GET %s HTTP/1.0' % ret.geturl()
+
+        # build request
+        fd = StringIO()
+        fd.write(request)
+        fd.write('\r\n')
+        for key, value in headers.iteritems():
+            fd.write("%s: %s" % (key, value))
+            fd.write("\r\n")
+        fd.write('\r\n')
+        fd.seek(0)
+        self.buffer = fd.read()
+
+        self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
+        try:
+            self.connect((ret.hostname, port))
+        except socket.gaierror, e:
+            self.del_channel()
+            raise e
+
+    def close(self):
+        dispatcher.close(self)
+        self.closed = True
+
+    def handle_connect(self):
+        print "####request####"
+        print self.buffer.strip()
+        #print time.time(), "connected"
+
+    def handle_close(self):
+        self.close()
+        #print time.time(), "closed"
+
+        # consuming the retrived data.
+        self.fd.seek(0)
+        self.response.message = self.fd.read()
+        self.on_received(self.response)
+
+    def _process_status_headers(self, entity):
+        fd = StringIO(entity)
+        fd.seek(0)
+        self.response.status = fd.readline().strip()
+        print "====response===="
+        print self.response.status
+        msg = rfc822.Message(fd)
+        for key, value in msg.dict.iteritems():
+            key = key.capitalize()
+            self.response.headers[key] = value
+            print "%s: %s" % (key, value)
+        self.fd.write(fd.read())
+
+    def _process_recv_data(self, entity):
+        if not self.response.status:
+            self._process_status_headers(entity)
+        else:
+            self.fd.write(entity)
+
+    def handle_read(self):
+        if not self.closed:
+            entity = self.recv(8192)
+            if entity:
+                self._process_recv_data(entity)
+
+    def handle_error(self):
+        import traceback
+        traceback.print_exc()
+
+    def writable(self):
+        return (len(self.buffer) > 0)
+
+    def handle_write(self):
+        #print time.time(), "send"
+        sent = self.send(self.buffer)
+        self.buffer = self.buffer[sent:]
+
+
+def _get_get_dispatcher():
+    for value in asyncore.socket_map.itervalues():
+        if isinstance(value, HttpClientGet):
+            return value
+
+def _loop_chain():
+    if len(asyncore.socket_map) != 0:
+        asyncore.loop(timeout=5, count=1)
+        gobject.idle_add(_loop_chain)
+
+# facade of 'network manager'
+#   request(uri, on_recived)
+#   on_received function,
+#   "def on_received(response):"
+def request_get(uri, headers, on_received):
+    get_dispatcher = _get_get_dispatcher()
+
+    if get_dispatcher:
+        raise BusyException()
+
+    need_exec_loop = len(asyncore.socket_map) == 0
+
+    get_dispatcher = HttpClientGet(uri, headers, on_received)
+    if need_exec_loop:
+        gobject.idle_add(_loop_chain)
+
+# facade of 'network manager'
+#   cancel_get
+def cancel_get():
+    get_dispatcher = _get_get_dispatcher()
+    if get_dispatcher:
+        get_dispatcher.close()
index ae133c7..81e0e77 100644 (file)
@@ -163,6 +163,4 @@ def start():
         uri_opener.open_uri("http://dubai.2ch.net/morningcoffee/")
     icon = gtk.status_icon_new_from_icon_name("")
     icon.connect("popup-menu", status_icon.on_icon_popup_menu)
         uri_opener.open_uri("http://dubai.2ch.net/morningcoffee/")
     icon = gtk.status_icon_new_from_icon_name("")
     icon.connect("popup-menu", status_icon.on_icon_popup_menu)
-    gtk.gdk.threads_enter()
     gtk.main()
     gtk.main()
-    gtk.gdk.threads_leave()
index aaa09f2..974a57e 100644 (file)
@@ -33,16 +33,16 @@ import traceback
 import itertools
 import os
 import sys
 import itertools
 import os
 import sys
+from StringIO import StringIO
 
 import misc
 
 import misc
-from misc import FileWrap, ThreadInvoker
+from misc import FileWrap
 import datfile
 import barehtmlparser
 import idxfile
 import session
 import board_window
 import uri_opener
 import datfile
 import barehtmlparser
 import idxfile
 import session
 import board_window
 import uri_opener
-from http_sub import HTTPRedirectHandler302, HTTPDebugHandler
 from BbsType import bbs_type_judge_uri
 from BbsType import bbs_type_exception
 import config
 from BbsType import bbs_type_judge_uri
 from BbsType import bbs_type_exception
 import config
@@ -52,6 +52,7 @@ import bookmark_window
 import thread_view
 import thread_popup
 import submit_window
 import thread_view
 import thread_popup
 import submit_window
+import network_manager
 
 GLADE_FILENAME = "thread_window.glade"
 
 
 GLADE_FILENAME = "thread_window.glade"
 
@@ -109,37 +110,294 @@ class HTMLParserToThreadView:
             self.layout = self.threadview.create_res_layout(
                 self.left_margin, self.resnum)
 
             self.layout = self.threadview.create_res_layout(
                 self.left_margin, self.resnum)
 
-        gtk.gdk.threads_enter()
         self.layout.add_text(data, bold, href)
         self.layout.add_text(data, bold, href)
-        gtk.gdk.threads_leave()
 
     def to_thread_view(self):
         if self.layout is not None:
 
     def to_thread_view(self):
         if self.layout is not None:
-            # gobject.idle_add(self.threadview.add_layout, self.layout)
-            gtk.gdk.threads_enter()
             self.threadview.add_layout(self.layout)
             self.threadview.add_layout(self.layout)
-            gtk.gdk.threads_leave()
             self.initialize()
 
 
             self.initialize()
 
 
+class LoadDat:
+
+    def __init__(self, bbs_type, threadwindow, threadview, statusbar):
+        self.bbs_type = bbs_type
+        self.threadwindow = threadwindow
+        self.threadview = threadview
+        self.statusbar = statusbar
+        self.lock_obj = False
+        self.jump_request_num = 0
+        self.size = 0
+        self.num = 0
+        self.request_headers = None
+        self.response_headers = None
+
+        self.statusbar_context_id = self.statusbar.get_context_id(
+            "Thread Window Status")
+        self.statusbar.push(self.statusbar_context_id, "OK.")
+
+    def initialize(self):
+        self.num = 0
+        self.size = 0
+        self.threadview.initialize_buffer()
+
+    def jump_request(self, res_num):
+        if not self.threadview.jump_to_res(res_num):
+            self.jump_request_num = res_num
+
+    def _do_jump_if_need(self):
+        if self.jump_request_num:
+            num = self.jump_request_num
+            self.jump_request_num = 0
+            return self.threadview.jump_to_res(num)
+
+    def _do_jump(self):
+        if not self._do_jump_if_need():
+            return self.threadview.jump_to_the_end()
+
+    def _load(self):
+        dat_path = misc.get_thread_dat_path(self.bbs_type)
+        try:
+            fd = file(dat_path)
+        except IOError:
+            raise misc.StopChainException()
+        else:
+            i = 0
+            for line in fd:
+                self.append_rawres_to_buffer(line)
+                yield
+            fd.close()
+            self._do_jump()
+
+    def _on_end(self):
+        self.request_headers = None
+        self.response_headers = None
+        self._un_lock()
+
+    def _lock(self):
+        if self.lock_obj:
+            print "Busy."
+            return False
+        self.lock_obj = True
+        return True
+
+    def _un_lock(self):
+        self.lock_obj = False
+
+    def load(self):
+        self.jump_request_num = 0
+        if self._lock():
+            misc.chain(lambda *args: None, self._on_end, self._load())
+
+    def _load_and_save(self, fd):
+        dat_path = misc.get_thread_dat_path(self.bbs_type)
+        try:
+            save_fd = FileWrap(dat_path)
+            save_fd.seek(self.size)
+        except IOError:
+            raise misc.StopChainException()
+
+        for line in fd:
+            if not line.endswith("\n"):
+                # the last line is not terminated with '\n'
+                print "does not end with \\n. maybe incomplete"
+                self.response_headers["lastmodified"] = None
+                self.response_headers["etag"] = None
+                raise misc.StopChainException()
+            save_fd.write(line)
+            self.append_rawres_to_buffer(line)
+            yield
+        self.threadview.queue_draw()
+        self._do_jump_if_need()
+
+    def on_received(self, res):
+        headers = res.headers
+        status = res.status
+
+        if "Last-modified".capitalize() in headers:
+            self.statusbar.pop(self.statusbar_context_id)
+            self.statusbar.push(self.statusbar_context_id,
+                "%s [%s]" % (status, headers["last-modified".capitalize()]))
+        else:
+            self.statusbar.pop(self.statusbar_context_id)
+            self.statusbar.push(self.statusbar_context_id, "%s" % status)
+
+        version, code, msg = status.split(None, 2)
+        code = int(code)
+        if code != 200 and code != 206:
+            self._on_end()
+            return
+        res.code = code
+
+        if "Range".capitalize() in self.request_headers and code == 200:
+            self.initialize()
+
+        self.response_headers = headers
+        fd = StringIO(res.message)
+
+        def save_and_end():
+            try:
+                self._saveidx()
+            finally:
+                self._on_end()
+
+        misc.chain(lambda *args: None, save_and_end, self._load_and_save(fd))
+
+    def _saveidx(self):
+        lastmod = ""
+        etag = ""
+        if "last-modified".capitalize() in self.response_headers:
+            lastmod = self.response_headers["last-modified".capitalize()]
+        if "etag".capitalize() in self.response_headers:
+            etag = self.response_headers["etag".capitalize()]
+
+        if self.num > 0:
+            # save idx
+            idx_dic = {"title": self.title, "lineCount": self.num,
+                       "lastModified": lastmod, "etag": etag}
+            idxfile.save_idx(self.bbs_type, idx_dic)
+
+            session.thread_idx_updated(self.bbs_type.get_thread_uri(), idx_dic)
+
+    def _http_get_dat(self):
+
+        datfile_url = self.bbs_type.get_dat_uri()
+
+        headers = {}
+
+        idx_dic = idxfile.load_idx(self.bbs_type)
+        lastmod = idx_dic["lastModified"]
+        etag = idx_dic["etag"]
+
+        req = urllib2.Request(datfile_url)
+        req.add_header("User-agent", config.User_Agent)
+        if self.size > 0:
+            req.add_header("Range", "bytes=" + str(self.size) + "-")
+        if lastmod:
+            req.add_header("If-Modified-Since", lastmod)
+        if etag:
+            req.add_header("If-None-Match", etag)
+
+        req = self.bbs_type.set_extra_dat_request(req, self)
+        try:
+            network_manager.request_get(req.get_full_url(),
+                req.headers, self.on_received)
+        except network_manager.BusyException:
+            self._on_end()
+            self.statusbar.pop(self.statusbar_context_id)
+            self.statusbar.push(self.statusbar_context_id,
+                "The network is busy. Try later.")
+        except:
+            self._on_end()
+            self.statusbar.pop(self.statusbar_context_id)
+            self.statusbar.push(self.statusbar_context_id,
+                str(sys.exc_info()))
+        else:
+            self.statusbar.pop(self.statusbar_context_id)
+            self.statusbar.push(self.statusbar_context_id, "GET...")
+            self.request_headers = req.headers
+
+    def update(self):
+
+        self.jump_request_num = 0
+
+        if not self._lock():
+            return
+
+        line_count = datfile.get_dat_line_count(self.bbs_type)
+        if line_count != self.num:
+            # load dat file once more
+            self.num = 0
+            self.size = 0
+            self.threadview.initialize_buffer()
+            def end_and_get():
+                self._on_end()
+                self._http_get_dat()
+            misc.chain(lambda *args: None, end_and_get, self._load())
+        else:
+            self._http_get_dat()
+
+    def append_rawres_to_buffer(self, line):
+        self.size += len(line)
+        self.num += 1
+
+        if self.num == 1:
+            title = self.bbs_type.get_title_from_dat(line)
+            if title:
+                self.title = title
+                self.threadwindow.set_title(title)
+
+        line = line.decode(self.bbs_type.encoding, "replace")
+        m = self.bbs_type.dat_reg.match(line)
+        if m:
+            name = m.group("name")
+            mail = m.group("mail")
+            date = m.group("date")
+            msg = m.group("msg")
+            try:
+                num = int(m.group("num"))
+            except IndexError:
+                # use simple counter num
+                num = self.num
+            else:
+                # use num in dat
+                self.num = num
+            try:
+                id = m.group("id")
+            except IndexError:
+                pass
+            else:
+                if id:
+                    date += " ID:" + id
+            self.reselems_to_buffer(num, name, mail, date, msg)
+        else:
+            self.reselems_to_buffer(
+                str(self.num), "Invalid Name", "Invalid Mail",
+                "Invalid Date", line)
+            print "maybe syntax error.", self.num, line
+
+    def reselems_to_buffer(self, num, name, mail, date, msg):
+        pipe = HTMLParserToThreadView(self.threadview, num, 0)
+        p = barehtmlparser.BareHTMLParser(
+            pipe.from_html_parser, pipe.on_new_line)
+
+        # First, create a pango layout for num,name,mail,date
+        # 'margin left' is 0
+        # number
+        p.feed(str(num) + " ")
+
+        # name
+        p.feed("<b>" + name + "</b>")
+
+        # mail
+        p.feed("[" + mail + "]")
+
+        # date
+        p.feed(date)
+        p.flush()
+
+        pipe.to_thread_view()
+
+
+        # Second, create a pango layout for message
+        # 'margin left' is 20
+        # msg
+        pipe.set_left_margin(20)
+        p.feed(msg.lstrip(" "))
+
+        p.feed("<br>")
+        p.close()
+
+        pipe.to_thread_view()
+
+
 class WinWrap(winwrapbase.WinWrapBase):
 class WinWrap(winwrapbase.WinWrapBase):
-    hovering_over_link = False
-    hand_cursor = gtk.gdk.Cursor(gtk.gdk.HAND2)
-    regular_cursor = gtk.gdk.Cursor(gtk.gdk.XTERM)
 
     def __init__(self, uri):
 
     def __init__(self, uri):
-        from BbsType import bbs_type_judge_uri
-        from BbsType import bbs_type_exception
         self.bbs_type = bbs_type_judge_uri.get_type(uri)
         if not self.bbs_type.is_thread():
             raise bbs_type_exception.BbsTypeError, \
                   "the uri does not represent thread: " + uri
         self.bbs_type = bbs_type_judge_uri.get_type(uri)
         if not self.bbs_type.is_thread():
             raise bbs_type_exception.BbsTypeError, \
                   "the uri does not represent thread: " + uri
-        self.size = 0
-        self.num = 0
-        self.title = ""
-        self.lock_obj = False
-        self.jump_request_num = 0
-        self.progress = False
 
         glade_path = os.path.join(config.glade_dir, GLADE_FILENAME)
         self.widget_tree = gtk.glade.XML(glade_path)
 
         glade_path = os.path.join(config.glade_dir, GLADE_FILENAME)
         self.widget_tree = gtk.glade.XML(glade_path)
@@ -163,12 +421,11 @@ class WinWrap(winwrapbase.WinWrapBase):
         self.threadpopup.connect(
             "uri-clicked-event", self.on_thread_popup_uri_clicked)
 
         self.threadpopup.connect(
             "uri-clicked-event", self.on_thread_popup_uri_clicked)
 
-        self.statusbar_context_id = self.statusbar.get_context_id(
-            "Thread Window Status")
-        self.statusbar.push(self.statusbar_context_id, "OK.")
-
         self.initialize_buffer()
 
         self.initialize_buffer()
 
+        self.dat_load = LoadDat(self.bbs_type, self.window,
+            self.threadview, self.statusbar)
+
         self.restore()
         self.window.show_all()
 
         self.restore()
         self.window.show_all()
 
@@ -218,8 +475,9 @@ class WinWrap(winwrapbase.WinWrapBase):
         session.main_quit()
 
     def _regist_as_bookmark(self):
         session.main_quit()
 
     def _regist_as_bookmark(self):
+        title = self.window.get_title()
         bookmark_list.bookmark_list.add_bookmark_with_edit(
         bookmark_list.bookmark_list.add_bookmark_with_edit(
-            name=self.title, uri=self.bbs_type.uri)
+            name=title, uri=self.bbs_type.uri)
 
     def _manage_bookmarks(self):
         bookmark_window.open()
 
     def _manage_bookmarks(self):
         bookmark_window.open()
@@ -266,246 +524,11 @@ class WinWrap(winwrapbase.WinWrapBase):
             uri = "http://" + uri
         return uri
 
             uri = "http://" + uri
         return uri
 
-    def http_get_dat(self, on_get_res):
-        datfile_url = self.bbs_type.get_dat_uri()
-
-        idx_dic = idxfile.load_idx(self.bbs_type)
-        lastmod = idx_dic["lastModified"]
-        etag = idx_dic["etag"]
-
-        req = urllib2.Request(datfile_url)
-        req.add_header("User-agent", config.User_Agent)
-        if self.size > 0:
-            req.add_header("Range", "bytes=" + str(self.size) + "-")
-        if lastmod:
-            req.add_header("If-Modified-Since", lastmod)
-        if etag:
-            req.add_header("If-None-Match", etag)
-
-        def push():
-            self.statusbar.pop(self.statusbar_context_id)
-            self.statusbar.push(self.statusbar_context_id, "GET...")
-        gobject.idle_add(push)
-
-        req = self.bbs_type.set_extra_dat_request(req, self)
-
-        opener = urllib2.build_opener(HTTPRedirectHandler302, HTTPDebugHandler)
-        try:
-            res = opener.open(req)
-        except urllib2.HTTPError, e:
-            def push(code, msg):
-                message = "%d %s" % (code, msg)
-                self.statusbar.pop(self.statusbar_context_id)
-                self.statusbar.push(self.statusbar_context_id, message)
-            gobject.idle_add(push, e.code, e.msg)
-        else:
-            headers = res.info()
-
-            if "Last-Modified" in headers:
-                la = headers["Last-Modified"]
-                def push(code, msg, lastm):
-                    message = "%d %s [%s]" % (code, msg, lastm)
-                    self.statusbar.pop(self.statusbar_context_id)
-                    self.statusbar.push(self.statusbar_context_id, message)
-                gobject.idle_add(push, res.code, res.msg, la)
-            else:
-                def push(code, msg):
-                    message = "%d %s" % (code, msg)
-                    self.statusbar.pop(self.statusbar_context_id)
-                    self.statusbar.push(self.statusbar_context_id, message)
-                gobject.idle_add(push, res.code, res.msg)
-
-            maybe_incomplete = False
-            for line in res:
-                if not line.endswith("\n"):
-                    maybe_incomplete = True
-                    print "does not end with \\n. maybe incomplete"
-                    break
-                on_get_res(line)
-
-            res.close()
-
-            if maybe_incomplete:
-                lastmod = None
-                etag = None
-            else:
-                if "Last-Modified" in headers:
-                    lastmod = headers["Last-Modified"]
-                if "ETag" in headers:
-                    etag = headers["Etag"]
-
-            if self.num > 0:
-                # save idx
-                idx_dic = {"title": self.title, "lineCount": self.num,
-                       "lastModified": lastmod, "etag": etag}
-                idxfile.save_idx(self.bbs_type, idx_dic)
-
-                gobject.idle_add(session.thread_idx_updated,
-                                 self.bbs_type.get_thread_uri(), idx_dic)
-
     def update(self):
     def update(self):
-
-        self.jump_request_num = 0
-
-        def load():
-            line_count = datfile.get_dat_line_count(self.bbs_type)
-            if line_count < self.num:
-                self.num = 0
-                self.size = 0
-
-                gobject.idle_add(self.initialize_buffer)
-
-            if line_count > self.num:
-                datfile.load_dat_partly(
-                    self.bbs_type, self.append_rawres_to_buffer, self.num+1)
-
-                def do_jump():
-                    if self.jump_request_num:
-                        if self.jump_request_num <= num:
-                            num = self.jump_request_num
-                            self.jump_request_num = 0
-                            self.jump_to_res(num)
-                    else:
-                        self.jump_to_the_end()
-
-                gobject.idle_add(do_jump)
-
-        def get():
-            dat_path = misc.get_thread_dat_path(self.bbs_type)
-            dat_file = FileWrap(dat_path)
-
-            def save_line_and_append_to_buffer(line):
-                dat_file.seek(self.size)
-                dat_file.write(line)
-                self.append_rawres_to_buffer(line)
-
-            self.http_get_dat(save_line_and_append_to_buffer)
-            gtk.gdk.threads_enter()
-            self.threadview.queue_draw()
-            gtk.gdk.threads_leave()
-            dat_file.close()
-
-            def do_jump():
-                if self.jump_request_num:
-                    num = self.jump_request_num
-                    self.jump_request_num = 0
-                    self.jump_to_res(num)
-
-            gobject.idle_add(do_jump)
-
-        if self.lock():
-
-            def on_end():
-                self.un_lock()
-                self.progress = False
-
-            self.progress = True
-            t = ThreadInvoker(lambda : gobject.idle_add(on_end), load, get)
-            t.start()
+        self.dat_load.update()
 
     def load_dat(self):
 
     def load_dat(self):
-
-        self.size = 0
-        self.num = 0
-        self.jump_request_num = 0
-
-        def load():
-            datfile.load_dat(self.bbs_type, self.append_rawres_to_buffer)
-
-        def jump():
-
-            def do_jump():
-                if self.jump_request_num:
-                    num = self.jump_request_num
-                    self.jump_request_num = 0
-                    self.jump_to_res(num)
-                else:
-                    self.jump_to_the_end()
-
-            gobject.idle_add(do_jump)
-
-        if self.lock():
-
-            def on_end():
-                self.un_lock()
-                self.progress = False
-
-            self.progress = True
-            t = ThreadInvoker(lambda : gobject.idle_add(on_end), load, jump)
-            t.start()
-
-    def append_rawres_to_buffer(self, line):
-        self.size += len(line)
-        self.num += 1
-
-        if not self.title and self.num == 1:
-            title = self.bbs_type.get_title_from_dat(line)
-            if title:
-                self.title = title
-                gobject.idle_add(self.window.set_title, title)
-
-        line = line.decode(self.bbs_type.encoding, "replace")
-        m = self.bbs_type.dat_reg.match(line)
-        if m:
-            name = m.group("name")
-            mail = m.group("mail")
-            date = m.group("date")
-            msg = m.group("msg")
-            try:
-                num = int(m.group("num"))
-            except IndexError:
-                # use simple counter num
-                num = self.num
-            else:
-                # use num in dat
-                self.num = num
-            try:
-                id = m.group("id")
-            except IndexError:
-                pass
-            else:
-                if id:
-                    date += " ID:" + id
-            self.reselems_to_buffer(num, name, mail, date, msg)
-        else:
-            self.reselems_to_buffer(
-                str(self.num), "Invalid Name", "Invalid Mail",
-                "Invalid Date", line)
-            print "maybe syntax error.", self.num, line
-
-    def reselems_to_buffer(self, num, name, mail, date, msg):
-        pipe = HTMLParserToThreadView(self.threadview, num, 0)
-        p = barehtmlparser.BareHTMLParser(
-            pipe.from_html_parser, pipe.on_new_line)
-
-        # First, create a pango layout for num,name,mail,date
-        # 'margin left' is 0
-        # number
-        p.feed(str(num) + " ")
-
-        # name
-        p.feed("<b>" + name + "</b>")
-
-        # mail
-        p.feed("[" + mail + "]")
-
-        # date
-        p.feed(date)
-        p.flush()
-
-        pipe.to_thread_view()
-
-
-        # Second, create a pango layout for message
-        # 'margin left' is 20
-        # msg
-        pipe.set_left_margin(20)
-        p.feed(msg.lstrip(" "))
-
-        p.feed("<br>")
-        p.close()
-
-        pipe.to_thread_view()
+        self.dat_load.load()
 
     def jump(self, value):
         gobject.idle_add(self.threadview.jump, value)
 
     def jump(self, value):
         gobject.idle_add(self.threadview.jump, value)
@@ -516,23 +539,10 @@ class WinWrap(winwrapbase.WinWrapBase):
     def jump_to_the_end(self):
         gobject.idle_add(self.threadview.jump_to_the_end)
 
     def jump_to_the_end(self):
         gobject.idle_add(self.threadview.jump_to_the_end)
 
-    def lock(self):
-        if self.lock_obj:
-            print "locked, try later."
-            return False
-        else:
-            print "get lock"
-            self.lock_obj = True
-            return True
-
-    def un_lock(self):
-        self.lock_obj = False
-        print "unlock"
-
     def jump_to_res(self, resnum):
         if self.threadview.jump_to_res(resnum):
             return
     def jump_to_res(self, resnum):
         if self.threadview.jump_to_res(resnum):
             return
-        self.jump_request_num = resnum
+        self.dat_load.jump_request(resnum)
 
     def load(self, update=False):
         dat_path = misc.get_thread_dat_path(self.bbs_type)
 
     def load(self, update=False):
         dat_path = misc.get_thread_dat_path(self.bbs_type)
index 92d42e4..d935b63 100755 (executable)
@@ -88,7 +88,6 @@ if __name__ == "__main__":
         obj = dbus_object.DBusFukuiNoNamariObject(bus_name, dbus_object_path)
 
         gnome.init(config.APPNAME, config.APPVERSION)
         obj = dbus_object.DBusFukuiNoNamariObject(bus_name, dbus_object_path)
 
         gnome.init(config.APPNAME, config.APPVERSION)
-        gtk.gdk.threads_init()
 
         bookmark_path = os.path.join(config.get_config_dir_path(),
                                      "bookmarks.txt")
 
         bookmark_path = os.path.join(config.get_config_dir_path(),
                                      "bookmarks.txt")