From 157f358349c94fbecd0f605ac47fa6b95984f008 Mon Sep 17 00:00:00 2001
From: Aiwota Programmer <aiwotaprog@tetteke.tk>
Date: Fri, 22 Sep 2006 06:44:40 +0900
Subject: [PATCH] Re implement board_data._load_modified_idxfiles.

---
 src/FukuiNoNamari/board_data.py | 97 +++++++++++++++++++++++++++--------------
 1 file changed, 64 insertions(+), 33 deletions(-)

diff --git a/src/FukuiNoNamari/board_data.py b/src/FukuiNoNamari/board_data.py
index 2f4f500..b881f5e 100644
--- a/src/FukuiNoNamari/board_data.py
+++ b/src/FukuiNoNamari/board_data.py
@@ -147,7 +147,7 @@ class BoardData:
         iterable_line = itertools.imap(lambda x :len(x), iterable_line)
         iterable_line = accumulate(iterable_line)
         iterable_line = itertools.imap(
-            lambda value: float(value) / total / 2, iterable_line)
+            lambda value: float(value) / total / 5 * 2, iterable_line)
         iterable_line = self._progressing(iterable_line)
 
         # union
@@ -158,41 +158,72 @@ class BoardData:
         return dict([(dic["id"], dic) for dic in iterable])
 
     def _load_modified_idxfiles(self, datalist):
-        basedir = misc.get_thread_idx_dir_path(self.bbs_type)
         ext = ".idx"
-        exist_key_set = set()
-        if os.path.isdir(basedir):
-            for idxfile_path in glob.glob(os.path.join(basedir, "*"+ext)):
-                basename = os.path.basename(idxfile_path)
-                thread_id = basename[:len(ext)*-1]
-                try:
-                    idxlastModified = os.path.getmtime(idxfile_path)
-                except OSError:
-                    continue
-                exist_key_set.add(thread_id)
-                if thread_id not in datalist:
-                    print "new", thread_id
-                    bbs_type_for_thread = self.bbs_type.clone_with_thread(
-                        thread_id)
-                    dic = idxfile.load_idx(bbs_type_for_thread)
-                    dic["id"] = thread_id
-                    dic["idxlastModified"] = idxlastModified
-                    dic = self._init_extra_data(dic)
-                    datalist[thread_id] = dic
-                elif idxlastModified > datalist[thread_id]["idxlastModified"]:
-                    print "modified", thread_id
-                    bbs_type_for_thread = self.bbs_type.clone_with_thread(
-                        thread_id)
-                    datalist[thread_id]["idxlastModified"] = idxlastModified
-                    dic = idxfile.load_idx(bbs_type_for_thread)
-                    for key, value in dic.iteritems():
-                        datalist[thread_id][key] = value
+
+        def id_and_lastmod(file_path):
+            thread_id = os.path.basename(file_path)[:len(ext)*-1]
+            try:
+                idxlastModified = os.path.getmtime(file_path)
+                return thread_id, idxlastModified
+            except OSError:
+                pass
+
+        def _do_new_thread(thread_id, idxlastModified):
+            print "new", thread_id
+
+            dic = idxfile.load_idx(self.bbs_type.clone_with_thread(thread_id))
+            dic["id"] = thread_id
+            dic["idxlastModified"] = idxlastModified
+            dic = self._init_extra_data(dic)
+            datalist[thread_id] = dic
+            return thread_id, idxlastModified
+
+        def _do_modified_thread(thread_id, idxlastModified):
+            print "modified", thread_id
+
+            datalist[thread_id]["idxlastModified"] = idxlastModified
+            dic = idxfile.load_idx(self.bbs_type.clone_with_thread(thread_id))
+            for key, value in dic.iteritems():
+                datalist[thread_id][key] = value
+            return thread_id, idxlastModified
+
+        def new_or_modified_thread(thread_id, idxlastModified):
+            if thread_id not in datalist:
+                return _do_new_thread(thread_id, idxlastModified)
+            elif idxlastModified > datalist[thread_id]["idxlastModified"]:
+                return _do_modified_thread(thread_id, idxlastModified)
+            return thread_id, idxlastModified
+
+        basedir = misc.get_thread_idx_dir_path(self.bbs_type)
+
+        filelist = glob.glob(os.path.join(basedir, "*"+ext))
+        total = len(filelist)
+
+        iterable = filelist
+
+        # split
+        iterable, iterable_count = itertools.tee(iterable)
+
+        iterable_count = itertools.izip(itertools.count(1), iterable_count)
+        iterable_count = itertools.starmap(lambda x, y: x, iterable_count)
+        iterable_count = itertools.imap(
+            lambda x: float(x)/total/10 + 0.4, iterable_count)
+        iterable_count = self._progressing(iterable_count)
+
+        # union
+        iterable = itertools.imap(lambda x, y: x, iterable, iterable_count)
+
+        iterable = itertools.imap(id_and_lastmod, iterable)
+        iterable = itertools.ifilter(None, iterable)
+        iterable = itertools.starmap(new_or_modified_thread, iterable)
+        exist_key_set = frozenset([x for x, y in iterable])
 
         # delete from datalist if idx file does not exist.
-        for key in datalist.keys():
-            if key not in exist_key_set:
-                del datalist[key]
-                print "del", key
+        datalist_key_set = frozenset(datalist.iterkeys())
+        delete_key_set = datalist_key_set - exist_key_set
+        for key in delete_key_set:
+            del datalist[key]
+            print "del", key
 
     def _split_record(self, line_encoded):
         line = line_encoded.decode(self.bbs_type.encoding, "replace")
-- 
2.11.0