OSDN Git Service

Multithreads are abandoned. Alternatly, The asyncore substitutes.(#16776)
[fukui-no-namari/fukui-no-namari.git] / src / FukuiNoNamari / board_data.py
index 0f41c98..48cc21c 100644 (file)
@@ -23,15 +23,21 @@ import codecs
 import urllib2
 import traceback
 import itertools
+from StringIO import StringIO
 
 import cachefile
 import idxfile
 import misc
 import config
 from http_sub import HTTPRedirectHandler302, HTTPDebugHandler
+import network_manager
 
 BOARD_DATA_INVALID_VALUE = 0
 
+
+class NothingToDoException: pass
+
+
 def accumulate(iterable, initial_value=0):
     sum_value = initial_value
     for value in iterable:
@@ -48,6 +54,7 @@ class BoardData:
 
     def __init__(self, bbs_type):
         self.bbs_type = bbs_type
+        self.lastmod = ""
 
     def set_status(self, text):
         pass
@@ -89,25 +96,19 @@ class BoardData:
                             "lastModified": 0, "average": average, "oldRes": 0}
 
     def merge_local_subjecttxt(self, datalist):
-        try:
-            for id, title, res, num, lastmod in self._load_subjecttxt():
-                self._merge_new_thread(datalist, id, title, res, num, lastmod)
-        except IOError:
-            pass
-        except:
-            tracebakc.print_exc()
-        else:
-            status = "Complete subject file."
-            gobject.idle_add(self.set_status, status)
+        for id, title, res, num, lastmod in self._load_subjecttxt():
+            self._merge_new_thread(datalist, id, title, res, num, lastmod)
+            yield
+        status = "Complete subject file."
+        lastmod = self.load_board_idx()
+        if lastmod:
+            self.lastmod = lastmod
+            status = "%s [%s]" % (status, lastmod)
+        self.set_status(status)
 
-    def merge_remote_subjecttxt(self, datalist):
-        try:
-            for id, title, res, num, lastmod in self._get_subjecttxt():
-                self._merge_new_thread(datalist, id, title, res, num, lastmod)
-        except IOError:
-            pass
-        except:
-            traceback.print_exc()
+    def merge_remote_subjecttxt(self, datalist, iterable):
+        for id, title, res, num, lastmod in iterable:
+            yield self._merge_new_thread(datalist, id, title, res, num, lastmod)
 
     def _init_extra_data(self, dic):
         dic["num"] = 0
@@ -119,11 +120,7 @@ class BoardData:
     def _progressing(self, iterable):
         for before, fraction in follow(iterable):
             if int(before*10) != int(fraction*10):
-                gtk.gdk.threads_enter()
-                try:
-                    self.set_fraction(fraction)
-                finally:
-                    gtk.gdk.threads_leave()
+                self.set_fraction(fraction)
             yield fraction
 
     def _modify_dict(self, item_dict):
@@ -137,30 +134,31 @@ class BoardData:
             item_dict["lastModified"] = secs
         return item_dict
 
-    def load_idxfiles(self):
-        print "load_cache"
+    def load_idxfiles(self, datalist):
         try:
-            datalist = self._load_cache()
+            for i in self._load_cache(datalist):
+                yield
         except IOError:
-            datalist = {}
-        print "load_idx"
-        self._load_modified_idxfiles(datalist)
-        print "save_cache"
-        try:
-            self._save_cache(datalist)
-        except IOError:
-            traceback.print_exc()
+            # the ".cache" file does not exist.
+            pass
+        else:
+            self.set_status("Complete load cache.")
+
+        for i in self._load_modified_idxfiles(datalist):
+            yield
+
+        self.set_status("Complete load idx files.")
+
+        self._save_cache(datalist)
+        # do not wait to save
 
         # adjustment after cache save, before load subject.txt
         iterable = datalist.itervalues()
         iterable = itertools.imap(self._modify_dict, iterable)
-        for i in iterable: -1
-
-        status = "Complete index files."
-        gobject.idle_add(self.set_status, status)
-        return datalist
+        for i in iterable:
+            yield
 
-    def _load_cache(self):
+    def _load_cache(self, datalist):
         try:
             total = os.path.getsize(misc.get_board_cache_path(self.bbs_type))
         except OSError:
@@ -185,7 +183,9 @@ class BoardData:
 
         iterable = itertools.imap(self._init_extra_data, iterable)
 
-        return dict([(dic["id"], dic) for dic in iterable])
+        for dic in iterable:
+            datalist[dic["id"]] = dic
+            yield
 
     def _load_modified_idxfiles(self, datalist):
         ext = ".idx"
@@ -246,7 +246,13 @@ class BoardData:
         iterable = itertools.imap(id_and_lastmod, iterable)
         iterable = itertools.ifilter(None, iterable)
         iterable = itertools.starmap(new_or_modified_thread, iterable)
-        exist_key_set = frozenset([x for x, y in iterable])
+
+        exist_key_set = set()
+        iterable = itertools.starmap(lambda x, y: exist_key_set.add(x),
+            iterable)
+
+        for i in iterable:
+            yield
 
         # delete from datalist if idx file does not exist.
         datalist_key_set = frozenset(datalist.iterkeys())
@@ -254,12 +260,13 @@ class BoardData:
         for key in delete_key_set:
             del datalist[key]
             print "del", key
+            yield
 
     def _save_cache(self, datalist):
-        iterable = datalist.iteritems()
+        iterable = datalist.items()
         iterable = cachefile.dict_to_formatted(iterable)
         c_file = misc.FileWrap(misc.get_board_cache_path(self.bbs_type), "w")
-        c_file.writelines(iterable)
+        misc.chain(c_file.write, c_file.close, iterable)
 
     def _split_record(self, line_encoded):
         line = line_encoded.decode(self.bbs_type.encoding, "replace")
@@ -312,78 +319,85 @@ class BoardData:
 
         return main_process()
 
-    def _get_subjecttxt(self):
-
-        # get subject.txt
-
-        opener = urllib2.build_opener(HTTPRedirectHandler302, HTTPDebugHandler)
+    def get_subjecttxt(self, on_received):
+        uri = self.bbs_type.get_subject_txt_uri()
         request = urllib2.Request(self.bbs_type.get_subject_txt_uri())
         request.add_header("User-agent", config.User_Agent)
+        if self.lastmod:
+            request.add_header("If-modified-since", self.lastmod)
+
         try:
-            response = opener.open(request)
-        except urllib2.HTTPError, e:
-            gobject.idle_add(self.set_status, "%d %s" % (e.code, e.msg))
-            print "switch to local"
-            return self._load_subjecttxt()
-        except urllib2.URLError, e:
-            print e
-            gobject.idle_add(self.set_status, str(e))
-            print "switch to local"
-            return self._load_subjecttxt()
+            network_manager.request_get(uri, request.headers, on_received)
+        except network_manager.BusyException:
+            self.set_status("The network is busy. Try later.")
+            raise NothingToDoException()
         else:
-            status = "%d %s" % (response.code, response.msg)
-            gobject.idle_add(self.set_status, status)
-            info = response.info()
+            self.set_status("GET...")
 
-            lastmod = 0
-            if "Last-Modified" in info:
-                _lastmod = info["Last-Modified"]
-                self.save_board_idx(_lastmod)
-                try:
-                    lastmod = misc.httpdate_to_secs(_lastmod)
-                except ValueError:
-                    lastmod = 0
+    def progress_response(self, response):
+        status = response.status
+        headers = response.headers
+        message = StringIO(response.message)
+
+        if "last-modified".capitalize() in headers:
+            self.set_status("%s [%s]" % (status,
+                headers["last-modified".capitalize()]))
+        else:
+            self.set_status("%s" % status)
+
+        version, code, msg = status.split(None, 2)
+        code = int(code)
+        if code != 200:
+            raise misc.StopChainException()
+
+        lastmod = 0
+        if "last-modified".capitalize() in headers:
+            _lastmod = headers["last-modified".capitalize()]
+            self.lastmod = _lastmod
+            self.save_board_idx(_lastmod)
+            try:
+                lastmod = misc.httpdate_to_secs(_lastmod)
+            except ValueError:
+                lastmod = 0
+
+        subjecttxt_path = misc.get_board_subjecttxt_path(self.bbs_type)
+        f = misc.FileWrap(subjecttxt_path, "w")
 
-            subjecttxt_path = misc.get_board_subjecttxt_path(self.bbs_type)
-            f = misc.FileWrap(subjecttxt_path, "w")
+        try:
+            total = int(headers["content-length".capitalize()])
+        except:
+            total = -1
 
+        def saving(line_encoded):
             try:
-                total = int(info["Content-Length"])
-            except:
-                total = -1
-
-            def saving(line_encoded):
-                try:
-                    f.write(line_encoded)
-                except IOError:
-                    traceback.print_exc()
-                return line_encoded
-
-            iterable = response
-
-            # split
-            iterable, iterable_len = itertools.tee(iterable)
-
-            iterable_len = itertools.imap(lambda l: len(l), iterable_len)
-            iterable_len = accumulate(iterable_len)
-            iterable_len = itertools.imap(
-                lambda value: float(value) / total, iterable_len)
-            iterable_len = self._progressing(iterable_len)
-
-            # union
-            iterable = itertools.imap(lambda x, y: x, iterable, iterable_len)
-
-            iterable = itertools.imap(saving, iterable)
-            iterable = itertools.izip(itertools.count(1), iterable)
-
-            def main_process():
-                for num, line_encoded in iterable:
-                    result = self._split_record(line_encoded)
-                    if result:
-                        id, title, res = result
-                        yield id, title, res, num, lastmod
-
-            return main_process()
+                f.write(line_encoded)
+            except IOError:
+                traceback.print_exc()
+            return line_encoded
+
+        iterable = message
+
+        # split
+        iterable, iterable_len = itertools.tee(iterable)
+
+        iterable_len = itertools.imap(lambda l: len(l), iterable_len)
+        iterable_len = accumulate(iterable_len)
+        iterable_len = itertools.imap(
+            lambda value: float(value) / total, iterable_len)
+        iterable_len = self._progressing(iterable_len)
+
+        # union
+        iterable = itertools.imap(lambda x, y: x, iterable, iterable_len)
+
+        iterable = itertools.imap(saving, iterable)
+        iterable = itertools.izip(itertools.count(1), iterable)
+
+        for num, line_encoded in iterable:
+            result = self._split_record(line_encoded)
+            if result:
+                id, title, res = result
+                yield id, title, res, num, lastmod
+        f.close()
 
     def load_board_idx(self):
         lastmod = ""