OSDN Git Service

Correct word index bug which excluded some words in error.
[neighbornote/NeighborNote.git] / src / cx / fbn / nevernote / threads / IndexRunner.java
index 80d4615..f9ff768 100644 (file)
 \r
 package cx.fbn.nevernote.threads;\r
 \r
+import java.io.File;\r
+import java.io.FileInputStream;\r
+import java.io.FileNotFoundException;\r
+import java.io.IOException;\r
+import java.io.InputStream;\r
+import java.util.List;\r
+import java.util.TreeSet;\r
 import java.util.concurrent.LinkedBlockingQueue;\r
 \r
+import org.apache.commons.lang.StringEscapeUtils;\r
+import org.apache.tika.exception.TikaException;\r
+import org.apache.tika.metadata.Metadata;\r
+import org.apache.tika.parser.ParseContext;\r
+import org.apache.tika.parser.microsoft.OfficeParser;\r
+import org.apache.tika.parser.microsoft.ooxml.OOXMLParser;\r
+import org.apache.tika.parser.odf.OpenDocumentParser;\r
+import org.apache.tika.parser.pdf.PDFParser;\r
+import org.apache.tika.parser.rtf.RTFParser;\r
+import org.apache.tika.sax.BodyContentHandler;\r
+import org.xml.sax.ContentHandler;\r
+import org.xml.sax.SAXException;\r
+\r
+import com.evernote.edam.type.Data;\r
 import com.evernote.edam.type.Note;\r
 import com.evernote.edam.type.Resource;\r
 import com.trolltech.qt.core.QByteArray;\r
+import com.trolltech.qt.core.QIODevice.OpenModeFlag;\r
 import com.trolltech.qt.core.QObject;\r
+import com.trolltech.qt.core.QTemporaryFile;\r
 import com.trolltech.qt.xml.QDomDocument;\r
 import com.trolltech.qt.xml.QDomElement;\r
 import com.trolltech.qt.xml.QDomNodeList;\r
 \r
 import cx.fbn.nevernote.Global;\r
+import cx.fbn.nevernote.signals.IndexSignal;\r
 import cx.fbn.nevernote.signals.NoteResourceSignal;\r
 import cx.fbn.nevernote.signals.NoteSignal;\r
 import cx.fbn.nevernote.sql.DatabaseConnection;\r
 import cx.fbn.nevernote.utilities.ApplicationLogger;\r
-import cx.fbn.nevernote.utilities.StringUtils;\r
 \r
-//public class IndexRunner implements QRunnable {\r
 public class IndexRunner extends QObject implements Runnable {\r
        \r
        private final ApplicationLogger         logger;\r
@@ -45,35 +67,33 @@ public class IndexRunner extends QObject implements Runnable {
        public volatile NoteSignal                      noteSignal;\r
        public volatile NoteResourceSignal      resourceSignal;\r
        private int                                                     indexType;\r
-       public final int                                        CONTENT=1; \r
-       public final int                                        RESOURCE=2;\r
-       private boolean                                         keepRunning;\r
-//     public volatile int                                     ID;\r
+       public final int                                        SCAN=1; \r
+       public final int                                        REINDEXALL=2;\r
+       public final int                                        REINDEXNOTE=3;\r
+       public boolean                                          keepRunning;\r
        private final QDomDocument                      doc;\r
-       private final int                                                       threadID;\r
        private static String                           regex = Global.getWordRegex();\r
        private final DatabaseConnection        conn;\r
        private volatile LinkedBlockingQueue<String> workQueue;\r
-//     private static int MAX_EMPTY_QUEUE_COUNT = 1;\r
        private static int MAX_QUEUED_WAITING = 1000;\r
-       \r
+       public boolean interrupt;\r
+       public boolean idle;\r
+       public boolean indexAttachmentsLocally = true;\r
+       public volatile IndexSignal                     signal;\r
+       private final TreeSet<String>           foundWords;\r
 \r
        \r
-       public IndexRunner(String logname) {\r
+       public IndexRunner(String logname, String u, String uid, String pswd, String cpswd) {\r
+               foundWords = new TreeSet<String>();\r
                logger = new ApplicationLogger(logname);\r
-               threadID = Global.indexThreadId;\r
-               conn = new DatabaseConnection(threadID);\r
-               noteSignal = new NoteSignal();\r
-               resourceSignal = new NoteResourceSignal();\r
-//             threadSignal = new ThreadSignal();\r
-               indexType = CONTENT;\r
+               conn = new DatabaseConnection(logger, u, uid, pswd, cpswd);\r
+               indexType = SCAN;\r
                guid = null;\r
                keepRunning = true;\r
                doc = new QDomDocument();\r
-               workQueue=new LinkedBlockingQueue<String>(MAX_QUEUED_WAITING);\r
+               workQueue=new LinkedBlockingQueue<String>(MAX_QUEUED_WAITING);  \r
        }\r
        \r
-       \r
        public void setIndexType(int t) {\r
                indexType = t;\r
        }\r
@@ -82,59 +102,74 @@ public class IndexRunner extends QObject implements Runnable {
        @Override\r
        public void run() {\r
                thread().setPriority(Thread.MIN_PRIORITY);\r
+               noteSignal = new NoteSignal();\r
+               resourceSignal = new NoteResourceSignal();\r
+               signal = new IndexSignal();\r
                logger.log(logger.EXTREME, "Starting index thread ");\r
                while (keepRunning) {\r
+                       idle=true;\r
                        try {\r
+                               //waitSeconds(1);\r
                                String work = workQueue.take();\r
-                               if (work.startsWith("CONTENT")) {\r
-                                       work = work.replace("CONTENT ", "");\r
-                                       guid = work;\r
-                                       indexType = CONTENT;\r
+                               idle=false;\r
+                               if (work.startsWith("SCAN")) {\r
+                                       guid=null;\r
+                                       interrupt = false;\r
+                                       indexType = SCAN;\r
+                               }\r
+                               if (work.startsWith("REINDEXALL")) {\r
+                                       guid = null;\r
+                                       indexType=REINDEXALL;\r
                                }\r
-                               if (work.startsWith("RESOURCE")) {\r
-                                       work = work.replace("RESOURCE ", "");\r
+                               if (work.startsWith("REINDEXNOTE")) {\r
+                                       work = work.replace("REINDEXNOTE ", "");\r
                                        guid = work;\r
-                                       indexType = RESOURCE;\r
+                                       indexType = REINDEXNOTE;\r
                                }\r
                                if (work.startsWith("STOP")) {\r
                                        keepRunning = false;\r
-                                       guid = work;\r
-                               }\r
-                               if (guid == null || guid.trim().equals("")) {\r
-                                       setIndexType(0);\r
-                                       resourceSignal.resourceIndexed.emit("null or empty guid");\r
+                                       guid = null;\r
                                }\r
                                logger.log(logger.EXTREME, "Type:" +indexType);\r
-                               if (indexType == CONTENT && keepRunning) {\r
-                                       logger.log(logger.MEDIUM, "Indexing note: "+guid);\r
-                                       indexNoteContent();\r
+                               if (indexType == SCAN && keepRunning) {\r
+                                       logger.log(logger.MEDIUM, "Scanning for unindexed notes & resources");\r
+                                       scanUnindexed();\r
                                        setIndexType(0);\r
                                }\r
-                               if (indexType == RESOURCE && keepRunning) {\r
-                                       logger.log(logger.MEDIUM, "Indexing resource: "+guid);\r
-                                       indexResource();\r
+                               if (indexType == REINDEXALL && keepRunning) {\r
+                                       logger.log(logger.MEDIUM, "Marking all for reindex");\r
+                                       reindexAll();\r
                                        setIndexType(0);\r
                                }\r
+                               if (indexType == REINDEXNOTE && keepRunning) {\r
+                                       reindexNote();\r
+                               }\r
                        } catch (InterruptedException e) {\r
-                               // TODO Auto-generated catch block\r
-                               e.printStackTrace();\r
+                               logger.log(logger.LOW, "Thread interrupted exception: " +e.getMessage());\r
                        }\r
                }\r
+               logger.log(logger.EXTREME, "Shutting down database");\r
+               conn.dbShutdown();\r
+               logger.log(logger.EXTREME, "Database shut down.  Exiting thread");\r
        }\r
        \r
        // Reindex a note\r
        public void indexNoteContent() {\r
+               foundWords.clear();\r
+               \r
                logger.log(logger.EXTREME, "Entering indexRunner.indexNoteContent()");\r
                \r
                logger.log(logger.EXTREME, "Getting note content");\r
                Note n = conn.getNoteTable().getNote(guid,true,false,true,true, true);\r
                String data = n.getContent();\r
+               data = conn.getNoteTable().getNoteContentNoUTFConversion(n.getGuid());\r
                \r
                logger.log(logger.EXTREME, "Removing any encrypted data");\r
-               data = removeEnCrypt(data);\r
+               data = removeEnCrypt(data.toString());\r
                logger.log(logger.EXTREME, "Removing xml markups");\r
-               String text = StringUtils.unescapeHTML(data.replaceAll("\\<.*?\\>", ""),0);\r
-               \r
+               String text =  removeTags(StringEscapeUtils.unescapeHtml(data) +" "+\r
+               n.getTitle());\r
+                               \r
                logger.log(logger.EXTREME, "Splitting words");\r
                String[] result = text.toString().split(regex);\r
                logger.log(logger.EXTREME, "Deleting existing words for note from index");\r
@@ -142,24 +177,9 @@ public class IndexRunner extends QObject implements Runnable {
                \r
                logger.log(logger.EXTREME, "Number of words found: " +result.length);\r
                for (int j=0; j<result.length && keepRunning; j++) {\r
-                       logger.log(logger.EXTREME, "Result word: " +result[j]);\r
-                       if (result[j].length() > 0) {\r
-                               if (Character.isLetterOrDigit(result[j].charAt(0))) {\r
-                                       int len = result[j].length();\r
-                                       StringBuffer buffer = new StringBuffer(result[j].toLowerCase());\r
-                                       logger.log(logger.EXTREME, "Processing " +buffer);\r
-                                       for (int k=len-1; k>=0 && keepRunning; k--) {\r
-                                               if (!Character.isLetterOrDigit(result[j].charAt(k)))\r
-                                                       buffer.deleteCharAt(k);\r
-                                               else\r
-                                                       k=-1;\r
-                                       }\r
-\r
-                                       if (buffer.length()>=Global.minimumWordCount) {\r
-                                               logger.log(logger.EXTREME, "Adding " +buffer);\r
-                                               conn.getWordsTable().addWordToNoteIndex(guid, buffer.toString(), "CONTENT", 100);\r
-                                       }\r
-                               }\r
+                       if (!result[j].trim().equals("")) {\r
+                               logger.log(logger.EXTREME, "Result word: " +result[j].trim());\r
+                               addToIndex(guid, result[j], "CONTENT");\r
                        }\r
                }\r
                // If we were interrupted, we will reindex this note next time\r
@@ -169,6 +189,22 @@ public class IndexRunner extends QObject implements Runnable {
                }\r
                logger.log(logger.EXTREME, "Leaving indexRunner.indexNoteContent()");\r
        }\r
+       \r
+       \r
+       private String removeTags(String text) {\r
+               StringBuffer buffer = new StringBuffer(text);\r
+               boolean inTag = false;\r
+               for (int i=buffer.length()-1; i>=0; i--) {\r
+                       if (buffer.charAt(i) == '>')\r
+                               inTag = true;\r
+                       if (buffer.charAt(i) == '<')\r
+                               inTag = false;\r
+                       if (inTag || buffer.charAt(i) == '<')\r
+                               buffer.deleteCharAt(i);\r
+               }\r
+               \r
+               return buffer.toString();\r
+       }\r
 \r
        \r
        public synchronized boolean addWork(String request) {\r
@@ -187,14 +223,16 @@ public class IndexRunner extends QObject implements Runnable {
                \r
                if (guid == null)\r
                        return;\r
-               \r
+               foundWords.clear();\r
                Resource r = conn.getNoteTable().noteResourceTable.getNoteResourceRecognition(guid);\r
                if (r == null || r.getRecognition() == null || r.getRecognition().getBody() == null || r.getRecognition().getBody().length == 0) \r
                        resourceBinary = new QByteArray(" ");\r
                else\r
                        resourceBinary = new QByteArray(r.getRecognition().getBody());\r
                \r
-               conn.getWordsTable().expungeFromWordIndex(guid, "RESOURCE");\r
+               conn.getWordsTable().expungeFromWordIndex(r.getNoteGuid(), "RESOURCE");\r
+               // This is due to an old bug & can be removed at some point in the future 11/23/2010\r
+               conn.getWordsTable().expungeFromWordIndex(guid, "RESOURCE");   \r
                        \r
                doc.setContent(resourceBinary);\r
                QDomElement docElem = doc.documentElement();\r
@@ -206,12 +244,273 @@ public class IndexRunner extends QObject implements Runnable {
                        String weight = new String(enmedia.attribute("w"));\r
                        String text = new String(enmedia.text()).toLowerCase();\r
                        if (!text.equals("")) {\r
-                               conn.getWordsTable().addWordToNoteIndex(guid, text, "RESOURCE", new Integer(weight));\r
+                               conn.getWordsTable().addWordToNoteIndex(r.getNoteGuid(), text, "RESOURCE", new Integer(weight));\r
                        }\r
                }\r
+               \r
+               if (Global.keepRunning && indexAttachmentsLocally) {\r
+                       indexResourceContent(guid);\r
+               }\r
+                               \r
                if (Global.keepRunning)\r
                        conn.getNoteTable().noteResourceTable.setIndexNeeded(guid,false);\r
        }\r
+       \r
+       private void indexResourceContent(String guid) {\r
+               Resource r = conn.getNoteTable().noteResourceTable.getNoteResource(guid, true);\r
+               if (r.getMime().equalsIgnoreCase("application/pdf")) {\r
+                       indexResourcePDF(r);\r
+                       return;\r
+               }\r
+               if (r.getMime().equalsIgnoreCase("application/docx") || \r
+                       r.getMime().equalsIgnoreCase("application/xlsx") || \r
+                       r.getMime().equalsIgnoreCase("application/pptx")) {\r
+                       indexResourceOOXML(r);\r
+                       return;\r
+               }\r
+               if (r.getMime().equalsIgnoreCase("application/vsd") ||\r
+                       r.getMime().equalsIgnoreCase("application/ppt") ||\r
+                       r.getMime().equalsIgnoreCase("application/xls") ||\r
+                       r.getMime().equalsIgnoreCase("application/msg") ||\r
+                       r.getMime().equalsIgnoreCase("application/doc")) {\r
+                               indexResourceOffice(r);\r
+                               return;\r
+               }\r
+               if (r.getMime().equalsIgnoreCase("application/rtf")) {\r
+                                       indexResourceRTF(r);\r
+                                       return;\r
+               }\r
+               if (r.getMime().equalsIgnoreCase("application/odf") ||\r
+                       r.getMime().equalsIgnoreCase("application/odt") ||\r
+                       r.getMime().equalsIgnoreCase("application/odp") ||\r
+                       r.getMime().equalsIgnoreCase("application/odg") ||\r
+                       r.getMime().equalsIgnoreCase("application/odb") ||\r
+                       r.getMime().equalsIgnoreCase("application/ods")) {\r
+                       indexResourceODF(r);\r
+                       return;\r
+               }\r
+       }\r
+\r
+\r
+       private void indexResourceRTF(Resource r) {\r
+\r
+               QTemporaryFile f = writeResource(r.getData());\r
+               if (!keepRunning) {\r
+                       return;\r
+               }\r
+               \r
+               InputStream input;\r
+               try {\r
+                       input = new FileInputStream(new File(f.fileName()));\r
+                       ContentHandler textHandler = new BodyContentHandler(-1);\r
+                       Metadata metadata = new Metadata();\r
+                       RTFParser parser = new RTFParser();     \r
+                       ParseContext context = new ParseContext();\r
+                       parser.parse(input, textHandler, metadata, context);\r
+                       String[] result = textHandler.toString().split(regex);\r
+                       for (int i=0; i<result.length && keepRunning; i++) {\r
+                               addToIndex(r.getNoteGuid(), result[i], "RESOURCE");\r
+                       }\r
+                       input.close();\r
+               \r
+                       f.close();\r
+               } catch (java.lang.ClassCastException e) {\r
+                       logger.log(logger.LOW, "Cast exception: " +e.getMessage());\r
+               } catch (FileNotFoundException e) {\r
+                       logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());\r
+               } catch (IOException e) {\r
+                       logger.log(logger.LOW, "IO  exception: " +e.getMessage());\r
+               } catch (SAXException e) {\r
+                       logger.log(logger.LOW, "SAX  exception: " +e.getMessage());\r
+               } catch (TikaException e) {\r
+                       logger.log(logger.LOW, "Tika  exception: " +e.getMessage());\r
+               } catch (Exception e) {\r
+                       logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());\r
+               } catch (java.lang.NoSuchMethodError e) {\r
+                       logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());\r
+               } catch (Error e) {\r
+                       logger.log(logger.LOW, "Unknown error: " +e.getMessage());\r
+               }\r
+       }\r
+\r
+       \r
+       private void indexResourceODF(Resource r) {\r
+\r
+               QTemporaryFile f = writeResource(r.getData());\r
+               if (!keepRunning) {\r
+                       return;\r
+               }\r
+               \r
+               InputStream input;\r
+               try {\r
+                       input = new FileInputStream(new File(f.fileName()));\r
+                       ContentHandler textHandler = new BodyContentHandler(-1);\r
+                       Metadata metadata = new Metadata();\r
+                       OpenDocumentParser parser = new OpenDocumentParser();   \r
+                       ParseContext context = new ParseContext();\r
+                       parser.parse(input, textHandler, metadata, context);\r
+                       String[] result = textHandler.toString().split(regex);\r
+                       for (int i=0; i<result.length && keepRunning; i++) {\r
+                               addToIndex(r.getNoteGuid(), result[i], "RESOURCE");\r
+                       }\r
+                       input.close();\r
+               \r
+                       f.close();\r
+               } catch (java.lang.ClassCastException e) {\r
+                       logger.log(logger.LOW, "Cast exception: " +e.getMessage());\r
+               } catch (FileNotFoundException e) {\r
+                       logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());\r
+               } catch (IOException e) {\r
+                       logger.log(logger.LOW, "IO  exception: " +e.getMessage());\r
+               } catch (SAXException e) {\r
+                       logger.log(logger.LOW, "SAX  exception: " +e.getMessage());\r
+               } catch (TikaException e) {\r
+                       logger.log(logger.LOW, "Tika  exception: " +e.getMessage());\r
+               } catch (Exception e) {\r
+                       logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());\r
+               } catch (java.lang.NoSuchMethodError e) {\r
+                       logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());\r
+               } catch (Error e) {\r
+                       logger.log(logger.LOW, "Unknown error: " +e.getMessage());\r
+               }\r
+       }\r
+\r
+       \r
+       private void indexResourceOffice(Resource r) {\r
+\r
+               QTemporaryFile f = writeResource(r.getData());\r
+               if (!keepRunning) {\r
+                       return;\r
+               }\r
+               \r
+               InputStream input;\r
+               try {\r
+                       input = new FileInputStream(new File(f.fileName()));\r
+                       ContentHandler textHandler = new BodyContentHandler(-1);\r
+                       Metadata metadata = new Metadata();\r
+                       OfficeParser parser = new OfficeParser();       \r
+                       ParseContext context = new ParseContext();\r
+                       parser.parse(input, textHandler, metadata, context);\r
+                       String[] result = textHandler.toString().split(regex);\r
+                       for (int i=0; i<result.length && keepRunning; i++) {\r
+                               addToIndex(r.getNoteGuid(), result[i], "RESOURCE");\r
+                       }\r
+                       input.close();\r
+               \r
+                       f.close();\r
+               } catch (java.lang.ClassCastException e) {\r
+                       logger.log(logger.LOW, "Cast exception: " +e.getMessage());\r
+               } catch (FileNotFoundException e) {\r
+                       logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());\r
+               } catch (IOException e) {\r
+                       logger.log(logger.LOW, "IO  exception: " +e.getMessage());\r
+               } catch (SAXException e) {\r
+                       logger.log(logger.LOW, "SAX  exception: " +e.getMessage());\r
+               } catch (TikaException e) {\r
+                       logger.log(logger.LOW, "Tika  exception: " +e.getMessage());\r
+               } catch (Exception e) {\r
+                       logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());\r
+               } catch (java.lang.NoSuchMethodError e) {\r
+                       logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());\r
+               } catch (Error e) {\r
+                       logger.log(logger.LOW, "Unknown error: " +e.getMessage());\r
+               }\r
+       }\r
+\r
+       \r
+       \r
+       private void indexResourcePDF(Resource r) {\r
+\r
+               QTemporaryFile f = writeResource(r.getData());\r
+               if (!keepRunning) {\r
+                       return;\r
+               }\r
+               \r
+               InputStream input;\r
+               try {                   \r
+                       input = new FileInputStream(new File(f.fileName()));\r
+                       ContentHandler textHandler = new BodyContentHandler(-1);\r
+                       Metadata metadata = new Metadata();\r
+                       PDFParser parser = new PDFParser();     \r
+                       ParseContext context = new ParseContext();\r
+                       parser.parse(input, textHandler, metadata, context);\r
+                       String[] result = textHandler.toString().split(regex);\r
+                       for (int i=0; i<result.length && keepRunning; i++) {\r
+                               addToIndex(r.getNoteGuid(), result[i], "RESOURCE");\r
+                       }\r
+                       input.close();\r
+               \r
+                       f.close();\r
+               } catch (java.lang.ClassCastException e) {\r
+                       logger.log(logger.LOW, "Cast exception: " +e.getMessage());\r
+               } catch (FileNotFoundException e) {\r
+                       logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());\r
+               } catch (IOException e) {\r
+                       logger.log(logger.LOW, "IO  exception: " +e.getMessage());\r
+               } catch (SAXException e) {\r
+                       logger.log(logger.LOW, "SAX  exception: " +e.getMessage());\r
+               } catch (TikaException e) {\r
+                       logger.log(logger.LOW, "Tika  exception: " +e.getMessage());\r
+               } catch (Exception e) {\r
+                       logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());\r
+               } catch (java.lang.NoSuchMethodError e) {\r
+                       logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());\r
+               } catch (Error e) {\r
+                       logger.log(logger.LOW, "Unknown error: " +e.getMessage());\r
+               }\r
+       }\r
+       \r
+       \r
+       private void indexResourceOOXML(Resource r) {\r
+\r
+               QTemporaryFile f = writeResource(r.getData());\r
+               if (!keepRunning) {\r
+                       return;\r
+               }\r
+               \r
+               InputStream input;\r
+               try {\r
+                       input = new FileInputStream(new File(f.fileName()));\r
+                       ContentHandler textHandler = new BodyContentHandler(-1);\r
+                       Metadata metadata = new Metadata();\r
+                       OOXMLParser parser = new OOXMLParser(); \r
+                       ParseContext context = new ParseContext();\r
+                       parser.parse(input, textHandler, metadata, context);\r
+                       String[] result = textHandler.toString().split(regex);\r
+                       for (int i=0; i<result.length && keepRunning; i++) {\r
+                               addToIndex(r.getNoteGuid(), result[i], "RESOURCE");\r
+                       }\r
+                       input.close();\r
+               \r
+                       f.close();\r
+               } catch (java.lang.ClassCastException e) {\r
+                       logger.log(logger.LOW, "Cast exception: " +e.getMessage());\r
+               } catch (FileNotFoundException e) {\r
+                       logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());\r
+               } catch (IOException e) {\r
+                       logger.log(logger.LOW, "IO  exception: " +e.getMessage());\r
+               } catch (SAXException e) {\r
+                       logger.log(logger.LOW, "SAX  exception: " +e.getMessage());\r
+               } catch (TikaException e) {\r
+                       logger.log(logger.LOW, "Tika  exception: " +e.getMessage());\r
+               } catch (Exception e) {\r
+                       logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());\r
+               } catch (java.lang.NoSuchMethodError e) {\r
+                       logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());\r
+               } catch (Error e) {\r
+                       logger.log(logger.LOW, "Unknown error: " +e.getMessage());\r
+               }\r
+       }\r
+       \r
+\r
+       \r
+       private QTemporaryFile writeResource(Data d) {\r
+               QTemporaryFile newFile = new QTemporaryFile();\r
+               newFile.open(OpenModeFlag.WriteOnly);\r
+               newFile.write(d.getBody());\r
+               newFile.close();\r
+               return newFile;\r
+       } \r
 \r
        \r
        private String removeEnCrypt(String content) {\r
@@ -231,7 +530,85 @@ public class IndexRunner extends QObject implements Runnable {
        }\r
 \r
        \r
+       private void addToIndex(String guid, String word, String type) {\r
+               if (foundWords.contains(word))\r
+                       return;\r
+               StringBuffer buffer = new StringBuffer(word.toLowerCase());\r
+               for (int i=buffer.length()-1; i>=0; i--) {\r
+                       if (!Character.isLetterOrDigit(buffer.charAt(i)))\r
+                               buffer.deleteCharAt(i);\r
+                       else\r
+                               break;\r
+               }\r
+               buffer = buffer.reverse();\r
+               for (int i=buffer.length()-1; i>=0; i--) {\r
+                       if (!Character.isLetterOrDigit(buffer.charAt(i)))\r
+                               buffer.deleteCharAt(i);\r
+                       else\r
+                               break;\r
+               }\r
+               buffer = buffer.reverse();\r
+               if (buffer.length() > 0) {\r
+                       // We have a good word, now let's trim off junk at the beginning or end\r
+                       if (!foundWords.contains(buffer.toString())) {\r
+                               foundWords.add(buffer.toString());\r
+                               foundWords.add(word);\r
+                               conn.getWordsTable().addWordToNoteIndex(guid, buffer.toString(), type, 100);\r
+                       }\r
+               }\r
+               return;\r
+       }\r
        \r
+       private void scanUnindexed() {\r
+               List<String> notes = conn.getNoteTable().getUnindexed();\r
+               guid = null;\r
+               boolean started = false;\r
+               if (notes.size() > 0) {\r
+                       signal.indexStarted.emit();\r
+                       started = true;\r
+               }\r
+               for (int i=0; i<notes.size() && !interrupt && keepRunning; i++) {\r
+                       guid = notes.get(i);\r
+                       if (guid != null && keepRunning) {\r
+                               //waitSeconds(1);\r
+                               indexNoteContent();\r
+                       }\r
+               }\r
+               \r
+               List<String> unindexedResources = conn.getNoteTable().noteResourceTable.getUnindexed();\r
+               if (unindexedResources.size() > 0 && !started) {\r
+                       signal.indexStarted.emit();\r
+                       started = true;\r
+               }\r
+               for (int i=0; i<unindexedResources.size()&& !interrupt && keepRunning; i++) {\r
+                       guid = unindexedResources.get(i);\r
+                       if (keepRunning) {\r
+                               //waitSeconds(1);\r
+                               indexResource();\r
+                       }\r
+               }\r
+               if (started && keepRunning && !interrupt) \r
+                       signal.indexFinished.emit();\r
+       }\r
+       \r
+       private void reindexNote() {\r
+               if (guid == null)\r
+                       return;\r
+               conn.getNoteTable().setIndexNeeded(guid, true);\r
+       }\r
        \r
+       private void reindexAll() {\r
+               conn.getNoteTable().reindexAllNotes();\r
+               conn.getNoteTable().noteResourceTable.reindexAll(); \r
+       }\r
 \r
+//     private void waitSeconds(int len) {\r
+//             QDateTime currentdate = new QDateTime(QDateTime.currentDateTime());\r
+//             QDateTime futuredate = new QDateTime(QDateTime.currentDateTime());\r
+//             \r
+//             while (keepRunning && (futuredate.toTime_t() - currentdate.toTime_t() >=len) ) {\r
+//                     Thread.yield();\r
+//                     futuredate = new QDateTime(QDateTime.currentDateTime());\r
+//             }\r
+//     }\r
 }\r