OSDN Git Service

Add connection throttling to try and reduce indexing & synchronizing impact.
[neighbornote/NeighborNote.git] / src / cx / fbn / nevernote / threads / IndexRunner.java
1 /*\r
2  * This file is part of NeverNote \r
3  * Copyright 2009 Randy Baumgarte\r
4  * \r
5  * This file may be licensed under the terms of of the\r
6  * GNU General Public License Version 2 (the ``GPL'').\r
7  *\r
8  * Software distributed under the License is distributed\r
9  * on an ``AS IS'' basis, WITHOUT WARRANTY OF ANY KIND, either\r
10  * express or implied. See the GPL for the specific language\r
11  * governing rights and limitations.\r
12  *\r
13  * You should have received a copy of the GPL along with this\r
14  * program. If not, go to http://www.gnu.org/licenses/gpl.html\r
15  * or write to the Free Software Foundation, Inc.,\r
16  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.\r
17  *\r
18 */\r
19 \r
20 package cx.fbn.nevernote.threads;\r
21 \r
22 import java.io.File;\r
23 import java.io.FileInputStream;\r
24 import java.io.FileNotFoundException;\r
25 import java.io.IOException;\r
26 import java.io.InputStream;\r
27 import java.util.List;\r
28 import java.util.TreeSet;\r
29 import java.util.concurrent.LinkedBlockingQueue;\r
30 import java.util.concurrent.locks.LockSupport;\r
31 \r
32 import org.apache.commons.lang.StringEscapeUtils;\r
33 import org.apache.tika.exception.TikaException;\r
34 import org.apache.tika.metadata.Metadata;\r
35 import org.apache.tika.parser.ParseContext;\r
36 import org.apache.tika.parser.microsoft.OfficeParser;\r
37 import org.apache.tika.parser.microsoft.ooxml.OOXMLParser;\r
38 import org.apache.tika.parser.odf.OpenDocumentParser;\r
39 import org.apache.tika.parser.pdf.PDFParser;\r
40 import org.apache.tika.parser.rtf.RTFParser;\r
41 import org.apache.tika.sax.BodyContentHandler;\r
42 import org.xml.sax.ContentHandler;\r
43 import org.xml.sax.SAXException;\r
44 \r
45 import com.evernote.edam.type.Data;\r
46 import com.evernote.edam.type.Note;\r
47 import com.evernote.edam.type.Resource;\r
48 import com.trolltech.qt.core.QByteArray;\r
49 import com.trolltech.qt.core.QIODevice.OpenModeFlag;\r
50 import com.trolltech.qt.core.QObject;\r
51 import com.trolltech.qt.core.QTemporaryFile;\r
52 import com.trolltech.qt.xml.QDomDocument;\r
53 import com.trolltech.qt.xml.QDomElement;\r
54 import com.trolltech.qt.xml.QDomNodeList;\r
55 \r
56 import cx.fbn.nevernote.Global;\r
57 import cx.fbn.nevernote.signals.IndexSignal;\r
58 import cx.fbn.nevernote.signals.NoteResourceSignal;\r
59 import cx.fbn.nevernote.signals.NoteSignal;\r
60 import cx.fbn.nevernote.sql.DatabaseConnection;\r
61 import cx.fbn.nevernote.utilities.ApplicationLogger;\r
62 \r
63 public class IndexRunner extends QObject implements Runnable {\r
64         \r
65         private final ApplicationLogger         logger;\r
66         private String                                          guid;\r
67         private QByteArray                                      resourceBinary;\r
68         public volatile NoteSignal                      noteSignal;\r
69         public volatile NoteResourceSignal      resourceSignal;\r
70         private int                                                     indexType;\r
71         public final int                                        SCAN=1; \r
72         public final int                                        REINDEXALL=2;\r
73         public final int                                        REINDEXNOTE=3;\r
74         public boolean                                          keepRunning;\r
75         private final QDomDocument                      doc;\r
76         private static String                           regex = Global.getWordRegex();\r
77         private final DatabaseConnection        conn;\r
78         private volatile LinkedBlockingQueue<String> workQueue;\r
79         private static int MAX_QUEUED_WAITING = 1000;\r
80         public boolean interrupt;\r
81         public boolean idle;\r
82         public boolean indexAttachmentsLocally = true;\r
83         public volatile IndexSignal                     signal;\r
84         private final TreeSet<String>           foundWords;\r
85 \r
86         \r
87         public IndexRunner(String logname, String u, String uid, String pswd, String cpswd) {\r
88                 foundWords = new TreeSet<String>();\r
89                 logger = new ApplicationLogger(logname);\r
90                 conn = new DatabaseConnection(logger, u, uid, pswd, cpswd, 300);\r
91                 indexType = SCAN;\r
92                 guid = null;\r
93                 keepRunning = true;\r
94                 doc = new QDomDocument();\r
95                 workQueue=new LinkedBlockingQueue<String>(MAX_QUEUED_WAITING);  \r
96         }\r
97         \r
98         public void setIndexType(int t) {\r
99                 indexType = t;\r
100         }\r
101         \r
102         \r
103         @Override\r
104         public void run() {\r
105                 thread().setPriority(Thread.MIN_PRIORITY);\r
106                 noteSignal = new NoteSignal();\r
107                 resourceSignal = new NoteResourceSignal();\r
108                 signal = new IndexSignal();\r
109                 logger.log(logger.EXTREME, "Starting index thread ");\r
110                 while (keepRunning) {\r
111                         idle=true;\r
112                         try {\r
113                                 waitSeconds(1);\r
114                                 String work = workQueue.take();\r
115                                 idle=false;\r
116                                 if (work.startsWith("SCAN")) {\r
117                                         guid=null;\r
118                                         interrupt = false;\r
119                                         indexType = SCAN;\r
120                                 }\r
121                                 if (work.startsWith("REINDEXALL")) {\r
122                                         guid = null;\r
123                                         indexType=REINDEXALL;\r
124                                 }\r
125                                 if (work.startsWith("REINDEXNOTE")) {\r
126                                         work = work.replace("REINDEXNOTE ", "");\r
127                                         guid = work;\r
128                                         indexType = REINDEXNOTE;\r
129                                 }\r
130                                 if (work.startsWith("STOP")) {\r
131                                         keepRunning = false;\r
132                                         guid = null;\r
133                                 }\r
134                                 logger.log(logger.EXTREME, "Type:" +indexType);\r
135                                 if (indexType == SCAN && keepRunning) {\r
136                                         logger.log(logger.MEDIUM, "Scanning for unindexed notes & resources");\r
137                                         scanUnindexed();\r
138                                         setIndexType(0);\r
139                                 }\r
140                                 if (indexType == REINDEXALL && keepRunning) {\r
141                                         logger.log(logger.MEDIUM, "Marking all for reindex");\r
142                                         reindexAll();\r
143                                         setIndexType(0);\r
144                                 }\r
145                                 if (indexType == REINDEXNOTE && keepRunning) {\r
146                                         reindexNote();\r
147                                 }\r
148                         } catch (InterruptedException e) {\r
149                                 logger.log(logger.LOW, "Thread interrupted exception: " +e.getMessage());\r
150                         }\r
151                 }\r
152                 logger.log(logger.EXTREME, "Shutting down database");\r
153                 conn.dbShutdown();\r
154                 logger.log(logger.EXTREME, "Database shut down.  Exiting thread");\r
155         }\r
156         \r
157         // Reindex a note\r
158         public void indexNoteContent() {\r
159                 foundWords.clear();\r
160                 \r
161                 logger.log(logger.EXTREME, "Entering indexRunner.indexNoteContent()");\r
162                 \r
163                 logger.log(logger.EXTREME, "Getting note content");\r
164                 Note n = conn.getNoteTable().getNote(guid,true,false,true,true, true);\r
165                 String data = n.getContent();\r
166                 data = conn.getNoteTable().getNoteContentNoUTFConversion(n.getGuid());\r
167                 \r
168                 logger.log(logger.EXTREME, "Removing any encrypted data");\r
169                 data = removeEnCrypt(data.toString());\r
170                 logger.log(logger.EXTREME, "Removing xml markups");\r
171                 String text =  removeTags(StringEscapeUtils.unescapeHtml(data) +" "+\r
172                 n.getTitle());\r
173                                 \r
174                 logger.log(logger.EXTREME, "Splitting words");\r
175                 String[] result = text.toString().split(regex);\r
176                 logger.log(logger.EXTREME, "Deleting existing words for note from index");\r
177                 conn.getWordsTable().expungeFromWordIndex(guid, "CONTENT");\r
178                 \r
179                 logger.log(logger.EXTREME, "Number of words found: " +result.length);\r
180                 for (int j=0; j<result.length && keepRunning; j++) {\r
181                         if (!result[j].trim().equals("")) {\r
182                                 logger.log(logger.EXTREME, "Result word: " +result[j].trim());\r
183                                 addToIndex(guid, result[j], "CONTENT");\r
184                         }\r
185                 }\r
186                 // If we were interrupted, we will reindex this note next time\r
187                 if (Global.keepRunning) {\r
188                         logger.log(logger.EXTREME, "Resetting note guid needed");\r
189                         conn.getNoteTable().setIndexNeeded(guid, false);\r
190                 }\r
191                 logger.log(logger.EXTREME, "Leaving indexRunner.indexNoteContent()");\r
192         }\r
193         \r
194         \r
195         private String removeTags(String text) {\r
196                 StringBuffer buffer = new StringBuffer(text);\r
197                 boolean inTag = false;\r
198                 for (int i=buffer.length()-1; i>=0; i--) {\r
199                         if (buffer.charAt(i) == '>')\r
200                                 inTag = true;\r
201                         if (buffer.charAt(i) == '<')\r
202                                 inTag = false;\r
203                         if (inTag || buffer.charAt(i) == '<')\r
204                                 buffer.deleteCharAt(i);\r
205                 }\r
206                 \r
207                 return buffer.toString();\r
208         }\r
209 \r
210         \r
211         public synchronized boolean addWork(String request) {\r
212                 if (workQueue.size() == 0) {\r
213                         workQueue.offer(request);\r
214                         return true;\r
215                 }\r
216                 return false;\r
217         }\r
218         \r
219         public synchronized int getWorkQueueSize() {\r
220                 return workQueue.size();\r
221         }\r
222         \r
223         public void indexResource() {\r
224                 \r
225                 if (guid == null)\r
226                         return;\r
227                 foundWords.clear();\r
228                 Resource r = conn.getNoteTable().noteResourceTable.getNoteResourceRecognition(guid);\r
229                 if (r == null || r.getRecognition() == null || r.getRecognition().getBody() == null || r.getRecognition().getBody().length == 0) \r
230                         resourceBinary = new QByteArray(" ");\r
231                 else\r
232                         resourceBinary = new QByteArray(r.getRecognition().getBody());\r
233                 \r
234                 conn.getWordsTable().expungeFromWordIndex(r.getNoteGuid(), "RESOURCE");\r
235                 // This is due to an old bug & can be removed at some point in the future 11/23/2010\r
236                 conn.getWordsTable().expungeFromWordIndex(guid, "RESOURCE");   \r
237                         \r
238                 doc.setContent(resourceBinary);\r
239                 QDomElement docElem = doc.documentElement();\r
240                         \r
241                 // look for text tags\r
242                 QDomNodeList anchors = docElem.elementsByTagName("t");\r
243                 for (int i=0; i<anchors.length() && keepRunning; i++) {\r
244                         QDomElement enmedia = anchors.at(i).toElement();\r
245                         String weight = new String(enmedia.attribute("w"));\r
246                         String text = new String(enmedia.text()).toLowerCase();\r
247                         if (!text.equals("")) {\r
248                                 conn.getWordsTable().addWordToNoteIndex(r.getNoteGuid(), text, "RESOURCE", new Integer(weight));\r
249                         }\r
250                 }\r
251                 \r
252                 if (Global.keepRunning && indexAttachmentsLocally) {\r
253                         indexResourceContent(guid);\r
254                 }\r
255                                 \r
256                 if (Global.keepRunning)\r
257                         conn.getNoteTable().noteResourceTable.setIndexNeeded(guid,false);\r
258         }\r
259         \r
260         private void indexResourceContent(String guid) {\r
261                 Resource r = conn.getNoteTable().noteResourceTable.getNoteResource(guid, true);\r
262                 if (r.getMime().equalsIgnoreCase("application/pdf")) {\r
263                         indexResourcePDF(r);\r
264                         return;\r
265                 }\r
266                 if (r.getMime().equalsIgnoreCase("application/docx") || \r
267                         r.getMime().equalsIgnoreCase("application/xlsx") || \r
268                         r.getMime().equalsIgnoreCase("application/pptx")) {\r
269                         indexResourceOOXML(r);\r
270                         return;\r
271                 }\r
272                 if (r.getMime().equalsIgnoreCase("application/vsd") ||\r
273                         r.getMime().equalsIgnoreCase("application/ppt") ||\r
274                         r.getMime().equalsIgnoreCase("application/xls") ||\r
275                         r.getMime().equalsIgnoreCase("application/msg") ||\r
276                         r.getMime().equalsIgnoreCase("application/doc")) {\r
277                                 indexResourceOffice(r);\r
278                                 return;\r
279                 }\r
280                 if (r.getMime().equalsIgnoreCase("application/rtf")) {\r
281                                         indexResourceRTF(r);\r
282                                         return;\r
283                 }\r
284                 if (r.getMime().equalsIgnoreCase("application/odf") ||\r
285                         r.getMime().equalsIgnoreCase("application/odt") ||\r
286                         r.getMime().equalsIgnoreCase("application/odp") ||\r
287                         r.getMime().equalsIgnoreCase("application/odg") ||\r
288                         r.getMime().equalsIgnoreCase("application/odb") ||\r
289                         r.getMime().equalsIgnoreCase("application/ods")) {\r
290                         indexResourceODF(r);\r
291                         return;\r
292                 }\r
293         }\r
294 \r
295 \r
296         private void indexResourceRTF(Resource r) {\r
297 \r
298                 QTemporaryFile f = writeResource(r.getData());\r
299                 if (!keepRunning) {\r
300                         return;\r
301                 }\r
302                 \r
303                 InputStream input;\r
304                 try {\r
305                         input = new FileInputStream(new File(f.fileName()));\r
306                         ContentHandler textHandler = new BodyContentHandler(-1);\r
307                         Metadata metadata = new Metadata();\r
308                         RTFParser parser = new RTFParser();     \r
309                         ParseContext context = new ParseContext();\r
310                         parser.parse(input, textHandler, metadata, context);\r
311                         String[] result = textHandler.toString().split(regex);\r
312                         for (int i=0; i<result.length && keepRunning; i++) {\r
313                                 addToIndex(r.getNoteGuid(), result[i], "RESOURCE");\r
314                         }\r
315                         input.close();\r
316                 \r
317                         f.close();\r
318                 } catch (java.lang.ClassCastException e) {\r
319                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());\r
320                 } catch (FileNotFoundException e) {\r
321                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());\r
322                 } catch (IOException e) {\r
323                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());\r
324                 } catch (SAXException e) {\r
325                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());\r
326                 } catch (TikaException e) {\r
327                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());\r
328                 } catch (Exception e) {\r
329                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());\r
330                 } catch (java.lang.NoSuchMethodError e) {\r
331                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());\r
332                 } catch (Error e) {\r
333                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());\r
334                 }\r
335         }\r
336 \r
337         \r
338         private void indexResourceODF(Resource r) {\r
339 \r
340                 QTemporaryFile f = writeResource(r.getData());\r
341                 if (!keepRunning) {\r
342                         return;\r
343                 }\r
344                 \r
345                 InputStream input;\r
346                 try {\r
347                         input = new FileInputStream(new File(f.fileName()));\r
348                         ContentHandler textHandler = new BodyContentHandler(-1);\r
349                         Metadata metadata = new Metadata();\r
350                         OpenDocumentParser parser = new OpenDocumentParser();   \r
351                         ParseContext context = new ParseContext();\r
352                         parser.parse(input, textHandler, metadata, context);\r
353                         String[] result = textHandler.toString().split(regex);\r
354                         for (int i=0; i<result.length && keepRunning; i++) {\r
355                                 addToIndex(r.getNoteGuid(), result[i], "RESOURCE");\r
356                         }\r
357                         input.close();\r
358                 \r
359                         f.close();\r
360                 } catch (java.lang.ClassCastException e) {\r
361                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());\r
362                 } catch (FileNotFoundException e) {\r
363                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());\r
364                 } catch (IOException e) {\r
365                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());\r
366                 } catch (SAXException e) {\r
367                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());\r
368                 } catch (TikaException e) {\r
369                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());\r
370                 } catch (Exception e) {\r
371                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());\r
372                 } catch (java.lang.NoSuchMethodError e) {\r
373                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());\r
374                 } catch (Error e) {\r
375                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());\r
376                 }\r
377         }\r
378 \r
379         \r
380         private void indexResourceOffice(Resource r) {\r
381 \r
382                 QTemporaryFile f = writeResource(r.getData());\r
383                 if (!keepRunning) {\r
384                         return;\r
385                 }\r
386                 \r
387                 InputStream input;\r
388                 try {\r
389                         input = new FileInputStream(new File(f.fileName()));\r
390                         ContentHandler textHandler = new BodyContentHandler(-1);\r
391                         Metadata metadata = new Metadata();\r
392                         OfficeParser parser = new OfficeParser();       \r
393                         ParseContext context = new ParseContext();\r
394                         parser.parse(input, textHandler, metadata, context);\r
395                         String[] result = textHandler.toString().split(regex);\r
396                         for (int i=0; i<result.length && keepRunning; i++) {\r
397                                 addToIndex(r.getNoteGuid(), result[i], "RESOURCE");\r
398                         }\r
399                         input.close();\r
400                 \r
401                         f.close();\r
402                 } catch (java.lang.ClassCastException e) {\r
403                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());\r
404                 } catch (FileNotFoundException e) {\r
405                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());\r
406                 } catch (IOException e) {\r
407                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());\r
408                 } catch (SAXException e) {\r
409                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());\r
410                 } catch (TikaException e) {\r
411                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());\r
412                 } catch (Exception e) {\r
413                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());\r
414                 } catch (java.lang.NoSuchMethodError e) {\r
415                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());\r
416                 } catch (Error e) {\r
417                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());\r
418                 }\r
419         }\r
420 \r
421         \r
422         \r
423         private void indexResourcePDF(Resource r) {\r
424 \r
425                 QTemporaryFile f = writeResource(r.getData());\r
426                 if (!keepRunning) {\r
427                         return;\r
428                 }\r
429                 \r
430                 InputStream input;\r
431                 try {                   \r
432                         input = new FileInputStream(new File(f.fileName()));\r
433                         ContentHandler textHandler = new BodyContentHandler(-1);\r
434                         Metadata metadata = new Metadata();\r
435                         PDFParser parser = new PDFParser();     \r
436                         ParseContext context = new ParseContext();\r
437                         parser.parse(input, textHandler, metadata, context);\r
438                         String[] result = textHandler.toString().split(regex);\r
439                         for (int i=0; i<result.length && keepRunning; i++) {\r
440                                 addToIndex(r.getNoteGuid(), result[i], "RESOURCE");\r
441                         }\r
442                         input.close();\r
443                 \r
444                         f.close();\r
445                 } catch (java.lang.ClassCastException e) {\r
446                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());\r
447                 } catch (FileNotFoundException e) {\r
448                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());\r
449                 } catch (IOException e) {\r
450                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());\r
451                 } catch (SAXException e) {\r
452                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());\r
453                 } catch (TikaException e) {\r
454                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());\r
455                 } catch (Exception e) {\r
456                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());\r
457                 } catch (java.lang.NoSuchMethodError e) {\r
458                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());\r
459                 } catch (Error e) {\r
460                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());\r
461                 }\r
462         }\r
463         \r
464         \r
465         private void indexResourceOOXML(Resource r) {\r
466 \r
467                 QTemporaryFile f = writeResource(r.getData());\r
468                 if (!keepRunning) {\r
469                         return;\r
470                 }\r
471                 \r
472                 InputStream input;\r
473                 try {\r
474                         input = new FileInputStream(new File(f.fileName()));\r
475                         ContentHandler textHandler = new BodyContentHandler(-1);\r
476                         Metadata metadata = new Metadata();\r
477                         OOXMLParser parser = new OOXMLParser(); \r
478                         ParseContext context = new ParseContext();\r
479                         parser.parse(input, textHandler, metadata, context);\r
480                         String[] result = textHandler.toString().split(regex);\r
481                         for (int i=0; i<result.length && keepRunning; i++) {\r
482                                 addToIndex(r.getNoteGuid(), result[i], "RESOURCE");\r
483                         }\r
484                         input.close();\r
485                 \r
486                         f.close();\r
487                 } catch (java.lang.ClassCastException e) {\r
488                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());\r
489                 } catch (FileNotFoundException e) {\r
490                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());\r
491                 } catch (IOException e) {\r
492                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());\r
493                 } catch (SAXException e) {\r
494                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());\r
495                 } catch (TikaException e) {\r
496                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());\r
497                 } catch (Exception e) {\r
498                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());\r
499                 } catch (java.lang.NoSuchMethodError e) {\r
500                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());\r
501                 } catch (Error e) {\r
502                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());\r
503                 }\r
504         }\r
505         \r
506 \r
507         \r
508         private QTemporaryFile writeResource(Data d) {\r
509                 QTemporaryFile newFile = new QTemporaryFile();\r
510                 newFile.open(OpenModeFlag.WriteOnly);\r
511                 newFile.write(d.getBody());\r
512                 newFile.close();\r
513                 return newFile;\r
514         } \r
515 \r
516         \r
517         private String removeEnCrypt(String content) {\r
518                 int index = content.indexOf("<en-crypt");\r
519                 int endPos;\r
520                 boolean tagFound = true;\r
521                 while (tagFound && keepRunning) {\r
522                         endPos = content.indexOf("</en-crypt>", index)+11;\r
523                         if (endPos > -1 && index > -1) {\r
524                                 content = content.substring(0,index)+content.substring(endPos);\r
525                                 index = content.indexOf("<en-crypt");\r
526                         } else {\r
527                                 tagFound = false;\r
528                         }\r
529                 }\r
530                 return content;\r
531         }\r
532 \r
533         \r
534         private void addToIndex(String guid, String word, String type) {\r
535                 if (foundWords.contains(word))\r
536                         return;\r
537                 StringBuffer buffer = new StringBuffer(word.toLowerCase());\r
538                 for (int i=buffer.length()-1; i>=0; i--) {\r
539                         if (!Character.isLetterOrDigit(buffer.charAt(i)))\r
540                                 buffer.deleteCharAt(i);\r
541                         else\r
542                                 break;\r
543                 }\r
544                 buffer = buffer.reverse();\r
545                 for (int i=buffer.length()-1; i>=0; i--) {\r
546                         if (!Character.isLetterOrDigit(buffer.charAt(i)))\r
547                                 buffer.deleteCharAt(i);\r
548                         else\r
549                                 break;\r
550                 }\r
551                 buffer = buffer.reverse();\r
552                 if (buffer.length() > 0) {\r
553                         // We have a good word, now let's trim off junk at the beginning or end\r
554                         if (!foundWords.contains(buffer.toString())) {\r
555                                 foundWords.add(buffer.toString());\r
556                                 foundWords.add(word);\r
557                                 conn.getWordsTable().addWordToNoteIndex(guid, buffer.toString(), type, 100);\r
558                         }\r
559                 }\r
560                 return;\r
561         }\r
562         \r
563         private void scanUnindexed() {\r
564                 List<String> notes = conn.getNoteTable().getUnindexed();\r
565                 guid = null;\r
566                 boolean started = false;\r
567                 if (notes.size() > 0) {\r
568                         signal.indexStarted.emit();\r
569                         started = true;\r
570                 }\r
571                 for (int i=0; i<notes.size() && !interrupt && keepRunning; i++) {\r
572                         guid = notes.get(i);\r
573                         if (guid != null && keepRunning) {\r
574                                 waitSeconds(1);\r
575                                 indexNoteContent();\r
576                         }\r
577                 }\r
578                 \r
579                 List<String> unindexedResources = conn.getNoteTable().noteResourceTable.getUnindexed();\r
580                 if (unindexedResources.size() > 0 && !started) {\r
581                         signal.indexStarted.emit();\r
582                         started = true;\r
583                 }\r
584                 for (int i=0; i<unindexedResources.size()&& !interrupt && keepRunning; i++) {\r
585                         guid = unindexedResources.get(i);\r
586                         if (keepRunning) {\r
587                                 waitSeconds(1);\r
588                                 indexResource();\r
589                         }\r
590                 }\r
591                 if (started && keepRunning && !interrupt) \r
592                         signal.indexFinished.emit();\r
593         }\r
594         \r
595         private void reindexNote() {\r
596                 if (guid == null)\r
597                         return;\r
598                 conn.getNoteTable().setIndexNeeded(guid, true);\r
599         }\r
600         \r
601         private void reindexAll() {\r
602                 conn.getNoteTable().reindexAllNotes();\r
603                 conn.getNoteTable().noteResourceTable.reindexAll(); \r
604         }\r
605 \r
606         private void waitSeconds(int len) {\r
607                 long starttime = 0; // variable declared\r
608                 //...\r
609                 // for the first time, remember the timestamp\r
610             starttime = System.currentTimeMillis();\r
611                 // the next timestamp we want to wake up\r
612                 starttime += (1000.0);\r
613                 // Wait until the desired next time arrives using nanosecond\r
614                 // accuracy timer (wait(time) isn't accurate enough on most platforms) \r
615                 LockSupport.parkNanos((Math.max(0, \r
616                     starttime - System.currentTimeMillis()) * 1000000));\r
617         }\r
618 }\r