OSDN Git Service

Rearrange. changes are individual, do not effect other. improve performance.
authorAiwota Programmer <aiwotaprog@tetteke.tk>
Tue, 19 Sep 2006 18:01:57 +0000 (03:01 +0900)
committerAiwota Programmer <aiwotaprog@tetteke.tk>
Tue, 19 Sep 2006 18:01:57 +0000 (03:01 +0900)
src/FukuiNoNamari/board_data.py
src/FukuiNoNamari/board_window.py
src/FukuiNoNamari/cachefile.py
src/FukuiNoNamari/idxfile.py
src/FukuiNoNamari/misc.py

index 97a6bda..24b32a0 100644 (file)
@@ -69,7 +69,7 @@ class BoardData:
                 item["res"] = res
                 item["average"] = average
         else:
-            datalist[id] = {"num": num, "title": title,
+            datalist[id] = {"id": id, "num": num, "title": title,
                             "res": res, "lineCount": BOARD_DATA_INVALID_VALUE,
                             "lastModified": "", "average": average}
 
@@ -83,20 +83,17 @@ class BoardData:
             self._merge_new_thread(datalist, id, title, res, num, lastmod)
         self._get_subjecttxt(f)
 
-    def _add_idx(self, datalist, id, dic):
-        datalist[id] = dic
+    def _init_extra_data(self, dic):
         dic["num"] = 0
         dic["res"] = 0
         dic["average"] = 0
-        
+        return dic
+
     def load_idxfiles(self):
         datalist = {}
 
         def on_load_record(id, metadata_dic):
-            bbs_type_for_thread = self.bbs_type.clone_with_thread(id)
-            idxfile_path = misc.get_thread_idx_path(bbs_type_for_thread)
-            if os.path.exists(idxfile_path):
-                self._add_idx(datalist, id, metadata_dic)
+            datalist[id] = self._init_extra_data(metadata_dic)
 
         print "load_cache"
         cachefile.load_cache(self.bbs_type, on_load_record)
@@ -109,25 +106,40 @@ class BoardData:
 
     def _load_modified_idxfiles(self, datalist):
         basedir = misc.get_thread_idx_dir_path(self.bbs_type)
+        ext = ".idx"
+        exist_key_set = set()
         if os.path.isdir(basedir):
-            for idxfile_path in glob.glob(os.path.join(basedir, "*.idx")):
-                thread_id, ext = os.path.splitext(
-                    os.path.basename(idxfile_path))
-                idxlastModified = os.path.getmtime(idxfile_path)
-                bbs_type_for_thread = self.bbs_type.clone_with_thread(
-                    thread_id)
+            for idxfile_path in glob.glob(os.path.join(basedir, "*"+ext)):
+                basename = os.path.basename(idxfile_path)
+                thread_id = basename[:len(ext)*-1]
+                try:
+                    idxlastModified = os.path.getmtime(idxfile_path)
+                except OSError:
+                    continue
+                exist_key_set.add(thread_id)
                 if thread_id not in datalist:
-                    print "new"
+                    print "new", thread_id
+                    bbs_type_for_thread = self.bbs_type.clone_with_thread(
+                        thread_id)
                     dic = idxfile.load_idx(bbs_type_for_thread)
-                    #dic.pop("etag")
+                    dic["id"] = thread_id
                     dic["idxlastModified"] = idxlastModified
-                    self._add_idx(datalist, thread_id, dic)
+                    dic = self._init_extra_data(dic)
+                    datalist[thread_id] = dic
                 elif idxlastModified > datalist[thread_id]["idxlastModified"]:
-                    print "modified"
+                    print "modified", thread_id
+                    bbs_type_for_thread = self.bbs_type.clone_with_thread(
+                        thread_id)
                     datalist[thread_id]["idxlastModified"] = idxlastModified
                     dic = idxfile.load_idx(bbs_type_for_thread)
-                    for name in idxfile.metadata_namelist:
-                        datalist[thread_id][name] = dic[name]
+                    for key, value in dic.iteritems():
+                        datalist[thread_id][key] = value
+
+        # delete from datalist if idx file does not exist.
+        for key in datalist.keys():
+            if key not in exist_key_set:
+                del datalist[key]
+                print "del", key
 
     def _split_record(self, line_encoded):
         line = line_encoded.decode(self.bbs_type.encoding, "replace")
index f1aed31..e778d91 100644 (file)
@@ -25,6 +25,7 @@ import gobject
 import gconf
 import traceback
 import sys
+import itertools
 
 import board_data
 import uri_opener
@@ -130,25 +131,10 @@ class WinWrap(winwrapbase.WinWrapBase, board_data.BoardData):
         bbs_type_for_thread = self.bbs_type.clone_with_thread(thread)
         uri_opener.open_uri(bbs_type_for_thread.get_thread_uri(), update)
 
-    def update_datastore(self, datalist):
+    def update_datastore(self, new_list):
         print "reflesh datastore"
 
-        list_list = []
-        for id, dic in datalist.iteritems():
-            dic["id"] = id
-
-            # lastModified
-            httpdate = dic["lastModified"]
-            try:
-                secs = misc.httpdate_to_secs(httpdate)
-                dic["lastModified"] = secs
-            except ValueError:
-                dic["lastModified"] = 0
-
-            list_list.append(dic)
-
-        model = self.treeview.get_model()
-        model.set_list(list_list)
+        self.treeview.get_model().set_list(new_list)
 
         # redraw visible area after set list to model
         self.treeview.queue_draw()
@@ -179,16 +165,40 @@ class WinWrap(winwrapbase.WinWrapBase, board_data.BoardData):
 
     def load(self, update=False):
 
+        def modify_dict(item_dict):
+            # lastModified, httpdate to second
+            httpdate = item_dict["lastModified"]
+            try:
+                secs = misc.httpdate_to_secs(httpdate)
+            except ValueError:
+                item_dict["lastModified"] = 0
+            else:
+                item_dict["lastModified"] = secs
+            return item_dict
+
+        def set_id(thread_id, item_dict):
+            item_dict["id"] = thread_id
+            return item_dict
+
+        def conv_dictdict_to_listdict(dictdict):
+            key_iter = dictdict.iterkeys()
+            value_iter = dictdict.itervalues()
+            iterable = itertools.imap(set_id, key_iter, value_iter)
+            iterable = itertools.imap(modify_dict, iterable)
+            return [item_dict for item_dict in iterable]
+
         def load_local():
             datalist = self.load_idxfiles()
             self.merge_local_subjecttxt(datalist)
-            gobject.idle_add(self.update_datastore, datalist)
+            new_list = conv_dictdict_to_listdict(datalist)
+            gobject.idle_add(self.update_datastore, new_list)
 
         def get_remote():
             print "start get subject.txt"
             datalist = self.load_idxfiles()
             self.merge_remote_subjecttxt(datalist)
-            gobject.idle_add(self.update_datastore, datalist)
+            new_list = conv_dictdict_to_listdict(datalist)
+            gobject.idle_add(self.update_datastore, new_list)
 
         sbj_path = misc.get_board_subjecttxt_path(self.bbs_type)
         sbj_exists = os.path.exists(sbj_path)
index 53184d1..f913ab6 100644 (file)
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 
 import os
+import traceback
 
 import misc
 
-metadata_namelist = ["title", "lineCount", "lastModified", "idxlastModified"]
+metadata_namelist = ("title", "lineCount", "lastModified", "idxlastModified")
+recorded_data = ("id", "title", "lineCount", "lastModified", "idxlastModified")
+int_data = ("lineCount", "idxlastModified")
+
+def _empty_generator():
+    for name in metadata_namelist:
+        if name in int_data:
+            yield name, 0
+        else:
+            yield name, ""
+
+default_dict = dict([pair for pair in _empty_generator()])
+
+
+def _tabbed_to_key_value_generator(line):
+    for key_equal_value in line.split("\t"):
+        try:
+            index = key_equal_value.index("=")
+        except ValueError:
+            pass
+        else:
+            key = key_equal_value[:index]
+            if key in recorded_data:
+                value = key_equal_value[index+1:]
+                if key in int_data:
+                    try:
+                        value = int(value)
+                    except:
+                        value = 0
+                yield key, value
+
+def _dict_to_tabbed(dic):
+    line = ""
+    for name, value in dic.iteritems():
+
+        # save metadata only if its name exists in metadata_namelist
+        if name not in metadata_namelist:
+            continue
+
+        # no need to save an empty or invalid field
+        if name in int_data and value > 0:
+            line += name + "=" + str(value) + "\t"
+        elif name not in int_data and value:
+            line += name + "=" + value + "\t"
+
+    return line
 
 def load_cache(bbs_type, func):
     """ Loads metadata from .cache file
 
     loads metadata and invokes handler per one thread
 
-    bbs: bbs id
-
-    board: board id
-
     func: invoked at loading one thread's metadata
     should be of the form:
     def handler_example(id, metadata_dic):
@@ -37,51 +79,24 @@ def load_cache(bbs_type, func):
     the second metadata dic which key is in metadata_namelist
     """
 
-    # nothing to do if .cache file does not exist
     cachefile_path = misc.get_board_cache_path(bbs_type)
-    if not os.path.exists(cachefile_path):
-        return
-
-    f = open(cachefile_path, "r")
     try:
-        line = f.readline()
-        while line:
-            metadatas = line.split("\t")
-            dic = {}
-            for field in metadatas:
-                for name in ["id", "title", "lineCount",
-                             "lastModified", "idxlastModified"]:
-                    if field.startswith(name+"="):
-                        value = field[len(name)+1:]
-                        if name is "lineCount" or name is "idxlastModified":
-                            try:
-                                dic[name] = int(value)
-                            except:
-                                dic[name] = 0
-                        else:
-                            dic[name] = value
-            # invoke func only if id exists
+        for line in file(cachefile_path):
+
+            dic = default_dict.copy()
+            for key, value in _tabbed_to_key_value_generator(line.rstrip()):
+                dic[key] = value
+
+            # only if id exists
             if "id" in dic and dic["id"]:
-                # if metadata in metadata_namelist does not exist,
-                # set empty str or 0
-                for name in metadata_namelist:
-                    if name not in dic:
-                        if name is "lineCount" or name is "idxlastModified":
-                            dic[name] = 0
-                        else:
-                            dic[name] = ""
                 func(dic["id"], dic)
-            line = f.readline()
-    finally:
-        f.close()
+
+    except IOError:
+        traceback.print_exc()
 
 def save_cache(bbs_type, dic):
     """ Saves metadata list to .cache file
 
-    bbs: bbs id
-
-    board: board id
-
     dic: dictionary of thread id and metadata dictionary
     which key is in metadata_namelist
     """
@@ -101,17 +116,7 @@ def save_cache(bbs_type, dic):
         # save only if id is not empty
         if id:
             line = "id=" + id + "\t"
-            # save metadata only if exists in metadata_namelist
-            for name in metadata_namelist:
-                # save metadata only if exists in metadata_list
-                # and no need to save an empty or invalid field
-                if name in metadata_list:
-                    if name is "lineCount" or name is "idxlastModified":
-                        if metadata_list[name] > 0:
-                            line += name + "=" + str(metadata_list[name]) + "\t"
-                    else:
-                        if metadata_list[name]:
-                            line += name + "=" + metadata_list[name] + "\t"
+            line += _dict_to_tabbed(metadata_list)
             line += "\n"
             f.write(line)
     f.close()
index 14b58b9..b89ce87 100644 (file)
 
 import os
 import os.path
-from fileinput import FileInput
+import traceback
+
 import misc
 
-metadata_namelist = ["title", "lineCount", "lastModified", "etag"]
+metadata_namelist = ("title", "lineCount", "lastModified", "etag")
+int_data = ("lineCount")
 
-def load_idx(bbs_type):
-    """Loads index file of thread
-
-    bbs: bbs id
+def _empty_generator():
+    for name in metadata_namelist:
+        if name in int_data:
+            yield name, 0
+        else:
+            yield name, ""
 
-    board: board id
+default_dict = dict([pair for pair in _empty_generator()])
 
-    thread: thread id
+def load_idx(bbs_type):
+    """Loads index file of thread
 
     return dictionary which key is in metadata_namelist if idx file exist,
     otherwise return empty dic
     """
     idxfile_path = misc.get_thread_idx_path(bbs_type)
-    if not os.path.exists(idxfile_path):
-        return {"title":"","lineCount":0,"lastModified":"","etag":""}
-
-    datadic = {}
-    f = FileInput(idxfile_path)
-    for line in f:
-        for name in metadata_namelist:
-            if line.startswith(name+"="):
-                value = line[len(name)+1:].rstrip("\n")
-                if name is "lineCount":
-                    try:
-                        datadic[name] = int(value)
-                    except:
-                        datadic[name] = 0
-                else:
-                    datadic[name] = value
-                #print name, datadic[name]
-                break;
-    f.close()
-
-    # if datadic does not have key, insert empty entry.
-    for name in metadata_namelist:
-        if name not in datadic:
-            if name is "lineCount":
-                datadic[name] = 0
+    datadic = default_dict.copy()
+
+    try:
+        for line in file(idxfile_path):
+            key_equal_value = line.rstrip()
+            try:
+                index = key_equal_value.index("=")
+            except ValueError:
+                pass
             else:
-                datadic[name] = ""
+                key = key_equal_value[:index]
+                if key in metadata_namelist:
+                    value = key_equal_value[index+1:]
+                    if key in int_data:
+                        try:
+                            value = int(value)
+                        except:
+                            value = 0
+                    datadic[key] = value
+    except IOError:
+        traceback.print_exc()
+
     return datadic
 
 def save_idx(bbs_type, datadic):
     """Saves thread metadatas to a index file
 
-    bbs: bbs id
-
-    board: board id
-
-    thread: thread id
-
     datadic: dictionary which key is in metadata_namelist
 
     no need to save empty or non-existing metadata
@@ -84,13 +78,17 @@ def save_idx(bbs_type, datadic):
     if not os.path.isdir(basedir):
         os.makedirs(basedir)
 
-    f = open(idxfile_path, "w")
-    for name in metadata_namelist:
-        if name in datadic:
-            if name is "lineCount":
-                if datadic[name] > 0:
-                    f.write(name + "=" + str(datadic[name]) + "\n")
-            else:
-                if datadic[name]:
-                    f.write(name + "=" + datadic[name] + "\n")
+    f = file(idxfile_path, "w")
+    for name, value in datadic.iteritems():
+        
+        # save metadata only if its name exists in metadata_namelist
+        if name not in metadata_namelist:
+            continue
+
+        # no need to save an empty or invalid field
+        if name in int_data and value > 0:
+            f.write(name + "=" + str(value) + "\n")
+        elif name not in int_data and value:
+            f.write(name + "=" + value + "\n")
+
     f.close()
index d2ca0d5..25e5b62 100644 (file)
@@ -105,6 +105,9 @@ def get_board_cache_path(bbs_type):
 def httpdate_to_secs(httpdate):
     """Returns the seconds since the epoch"""
 
+    if not httpdate:
+        raise ValueError
+
     m = REG_EXPR_HTTPDATE.match(httpdate)
     if m:
         tm_wday = WDAY_DICT[m.group(1)]