OSDN Git Service

1ace4c02dd114e7bfcc52657d2a01c980ba0cc00
[neighbornote/NeighborNote.git] / src / cx / fbn / nevernote / threads / IndexRunner.java
1 /*\r
2  * This file is part of NeverNote \r
3  * Copyright 2009 Randy Baumgarte\r
4  * \r
5  * This file may be licensed under the terms of of the\r
6  * GNU General Public License Version 2 (the ``GPL'').\r
7  *\r
8  * Software distributed under the License is distributed\r
9  * on an ``AS IS'' basis, WITHOUT WARRANTY OF ANY KIND, either\r
10  * express or implied. See the GPL for the specific language\r
11  * governing rights and limitations.\r
12  *\r
13  * You should have received a copy of the GPL along with this\r
14  * program. If not, go to http://www.gnu.org/licenses/gpl.html\r
15  * or write to the Free Software Foundation, Inc.,\r
16  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.\r
17  *\r
18 */\r
19 \r
20 package cx.fbn.nevernote.threads;\r
21 \r
22 import java.io.File;\r
23 import java.io.FileInputStream;\r
24 import java.io.FileNotFoundException;\r
25 import java.io.IOException;\r
26 import java.io.InputStream;\r
27 import java.util.List;\r
28 import java.util.TreeSet;\r
29 import java.util.concurrent.LinkedBlockingQueue;\r
30 \r
31 import org.apache.commons.lang.StringEscapeUtils;\r
32 import org.apache.tika.exception.TikaException;\r
33 import org.apache.tika.metadata.Metadata;\r
34 import org.apache.tika.parser.ParseContext;\r
35 import org.apache.tika.parser.microsoft.OfficeParser;\r
36 import org.apache.tika.parser.microsoft.ooxml.OOXMLParser;\r
37 import org.apache.tika.parser.odf.OpenDocumentParser;\r
38 import org.apache.tika.parser.pdf.PDFParser;\r
39 import org.apache.tika.parser.rtf.RTFParser;\r
40 import org.apache.tika.sax.BodyContentHandler;\r
41 import org.xml.sax.ContentHandler;\r
42 import org.xml.sax.SAXException;\r
43 \r
44 import com.evernote.edam.type.Data;\r
45 import com.evernote.edam.type.Note;\r
46 import com.evernote.edam.type.Resource;\r
47 import com.trolltech.qt.core.QByteArray;\r
48 import com.trolltech.qt.core.QIODevice.OpenModeFlag;\r
49 import com.trolltech.qt.core.QObject;\r
50 import com.trolltech.qt.core.QTemporaryFile;\r
51 import com.trolltech.qt.xml.QDomDocument;\r
52 import com.trolltech.qt.xml.QDomElement;\r
53 import com.trolltech.qt.xml.QDomNodeList;\r
54 \r
55 import cx.fbn.nevernote.Global;\r
56 import cx.fbn.nevernote.signals.IndexSignal;\r
57 import cx.fbn.nevernote.signals.NoteResourceSignal;\r
58 import cx.fbn.nevernote.signals.NoteSignal;\r
59 import cx.fbn.nevernote.sql.DatabaseConnection;\r
60 import cx.fbn.nevernote.utilities.ApplicationLogger;\r
61 \r
62 public class IndexRunner extends QObject implements Runnable {\r
63         \r
64         private final ApplicationLogger         logger;\r
65         private String                                          guid;\r
66         private QByteArray                                      resourceBinary;\r
67         public volatile NoteSignal                      noteSignal;\r
68         public volatile NoteResourceSignal      resourceSignal;\r
69         private int                                                     indexType;\r
70         public final int                                        SCAN=1; \r
71         public final int                                        REINDEXALL=2;\r
72         public final int                                        REINDEXNOTE=3;\r
73         public boolean                                          keepRunning;\r
74         private final QDomDocument                      doc;\r
75         private static String                           regex = Global.getWordRegex();\r
76         private final DatabaseConnection        conn;\r
77         private volatile LinkedBlockingQueue<String> workQueue;\r
78         private static int MAX_QUEUED_WAITING = 1000;\r
79         public boolean interrupt;\r
80         public boolean idle;\r
81         public boolean indexAttachmentsLocally = true;\r
82         public volatile IndexSignal                     signal;\r
83         private final TreeSet<String>           foundWords;\r
84 \r
85         \r
86         public IndexRunner(String logname, String u, String uid, String pswd, String cpswd) {\r
87                 foundWords = new TreeSet<String>();\r
88                 logger = new ApplicationLogger(logname);\r
89                 conn = new DatabaseConnection(logger, u, uid, pswd, cpswd);\r
90                 indexType = SCAN;\r
91                 guid = null;\r
92                 keepRunning = true;\r
93                 doc = new QDomDocument();\r
94                 workQueue=new LinkedBlockingQueue<String>(MAX_QUEUED_WAITING);  \r
95         }\r
96         \r
97         public void setIndexType(int t) {\r
98                 indexType = t;\r
99         }\r
100         \r
101         \r
102         @Override\r
103         public void run() {\r
104                 thread().setPriority(Thread.MIN_PRIORITY);\r
105                 noteSignal = new NoteSignal();\r
106                 resourceSignal = new NoteResourceSignal();\r
107                 signal = new IndexSignal();\r
108                 logger.log(logger.EXTREME, "Starting index thread ");\r
109                 while (keepRunning) {\r
110                         idle=true;\r
111                         try {\r
112                                 String work = workQueue.take();\r
113                                 idle=false;\r
114                                 if (work.startsWith("SCAN")) {\r
115                                         guid=null;\r
116                                         interrupt = false;\r
117                                         indexType = SCAN;\r
118                                 }\r
119                                 if (work.startsWith("REINDEXALL")) {\r
120                                         guid = null;\r
121                                         indexType=REINDEXALL;\r
122                                 }\r
123                                 if (work.startsWith("REINDEXNOTE")) {\r
124                                         work = work.replace("REINDEXNOTE ", "");\r
125                                         guid = work;\r
126                                         indexType = REINDEXNOTE;\r
127                                 }\r
128                                 if (work.startsWith("STOP")) {\r
129                                         keepRunning = false;\r
130                                         guid = null;\r
131                                 }\r
132                                 logger.log(logger.EXTREME, "Type:" +indexType);\r
133                                 if (indexType == SCAN && keepRunning) {\r
134                                         logger.log(logger.MEDIUM, "Scanning for unindexed notes & resources");\r
135                                         scanUnindexed();\r
136                                         setIndexType(0);\r
137                                 }\r
138                                 if (indexType == REINDEXALL && keepRunning) {\r
139                                         logger.log(logger.MEDIUM, "Marking all for reindex");\r
140                                         reindexAll();\r
141                                         setIndexType(0);\r
142                                 }\r
143                                 if (indexType == REINDEXNOTE && keepRunning) {\r
144                                         reindexNote();\r
145                                 }\r
146                         } catch (InterruptedException e) {\r
147                                 logger.log(logger.LOW, "Thread interrupted exception: " +e.getMessage());\r
148                         }\r
149                 }\r
150                 logger.log(logger.EXTREME, "Shutting down database");\r
151                 conn.dbShutdown();\r
152                 logger.log(logger.EXTREME, "Database shut down.  Exiting thread");\r
153         }\r
154         \r
155         // Reindex a note\r
156         public void indexNoteContent() {\r
157                 foundWords.clear();\r
158                 \r
159                 logger.log(logger.EXTREME, "Entering indexRunner.indexNoteContent()");\r
160                 \r
161                 logger.log(logger.EXTREME, "Getting note content");\r
162                 Note n = conn.getNoteTable().getNote(guid,true,false,true,true, true);\r
163                 String data = n.getContent();\r
164                 data = conn.getNoteTable().getNoteContentNoUTFConversion(n.getGuid());\r
165                 \r
166                 logger.log(logger.EXTREME, "Removing any encrypted data");\r
167                 data = removeEnCrypt(data.toString());\r
168                 logger.log(logger.EXTREME, "Removing xml markups");\r
169                 String text =  removeTags(StringEscapeUtils.unescapeHtml(data) +" "+\r
170                 n.getTitle());\r
171                                 \r
172                 logger.log(logger.EXTREME, "Splitting words");\r
173                 String[] result = text.toString().split(regex);\r
174                 logger.log(logger.EXTREME, "Deleting existing words for note from index");\r
175                 conn.getWordsTable().expungeFromWordIndex(guid, "CONTENT");\r
176                 \r
177                 logger.log(logger.EXTREME, "Number of words found: " +result.length);\r
178                 for (int j=0; j<result.length && keepRunning; j++) {\r
179                         if (!result[j].trim().equals("")) {\r
180                                 logger.log(logger.EXTREME, "Result word: " +result[j].trim());\r
181                                 addToIndex(guid, result[j], "CONTENT");\r
182                         }\r
183                 }\r
184                 // If we were interrupted, we will reindex this note next time\r
185                 if (Global.keepRunning) {\r
186                         logger.log(logger.EXTREME, "Resetting note guid needed");\r
187                         conn.getNoteTable().setIndexNeeded(guid, false);\r
188                 }\r
189                 logger.log(logger.EXTREME, "Leaving indexRunner.indexNoteContent()");\r
190         }\r
191         \r
192         \r
193         private String removeTags(String text) {\r
194                 StringBuffer buffer = new StringBuffer(text);\r
195                 boolean inTag = false;\r
196                 for (int i=buffer.length()-1; i>=0; i--) {\r
197                         if (buffer.charAt(i) == '>')\r
198                                 inTag = true;\r
199                         if (buffer.charAt(i) == '<')\r
200                                 inTag = false;\r
201                         if (inTag || buffer.charAt(i) == '<')\r
202                                 buffer.deleteCharAt(i);\r
203                 }\r
204                 \r
205                 return buffer.toString();\r
206         }\r
207 \r
208         \r
209         public synchronized boolean addWork(String request) {\r
210                 if (workQueue.size() == 0) {\r
211                         workQueue.offer(request);\r
212                         return true;\r
213                 }\r
214                 return false;\r
215         }\r
216         \r
217         public synchronized int getWorkQueueSize() {\r
218                 return workQueue.size();\r
219         }\r
220         \r
221         public void indexResource() {\r
222                 \r
223                 if (guid == null)\r
224                         return;\r
225                 foundWords.clear();\r
226                 Resource r = conn.getNoteTable().noteResourceTable.getNoteResourceRecognition(guid);\r
227                 if (r == null || r.getRecognition() == null || r.getRecognition().getBody() == null || r.getRecognition().getBody().length == 0) \r
228                         resourceBinary = new QByteArray(" ");\r
229                 else\r
230                         resourceBinary = new QByteArray(r.getRecognition().getBody());\r
231                 \r
232                 conn.getWordsTable().expungeFromWordIndex(r.getNoteGuid(), "RESOURCE");\r
233                 // This is due to an old bug & can be removed at some point in the future 11/23/2010\r
234                 conn.getWordsTable().expungeFromWordIndex(guid, "RESOURCE");   \r
235                         \r
236                 doc.setContent(resourceBinary);\r
237                 QDomElement docElem = doc.documentElement();\r
238                         \r
239                 // look for text tags\r
240                 QDomNodeList anchors = docElem.elementsByTagName("t");\r
241                 for (int i=0; i<anchors.length() && keepRunning; i++) {\r
242                         QDomElement enmedia = anchors.at(i).toElement();\r
243                         String weight = new String(enmedia.attribute("w"));\r
244                         String text = new String(enmedia.text()).toLowerCase();\r
245                         if (!text.equals("")) {\r
246                                 conn.getWordsTable().addWordToNoteIndex(r.getNoteGuid(), text, "RESOURCE", new Integer(weight));\r
247                         }\r
248                 }\r
249                 \r
250                 if (Global.keepRunning && indexAttachmentsLocally) {\r
251                         indexResourceContent(guid);\r
252                 }\r
253                                 \r
254                 if (Global.keepRunning)\r
255                         conn.getNoteTable().noteResourceTable.setIndexNeeded(guid,false);\r
256         }\r
257         \r
258         private void indexResourceContent(String guid) {\r
259                 Resource r = conn.getNoteTable().noteResourceTable.getNoteResource(guid, true);\r
260                 if (r.getMime().equalsIgnoreCase("application/pdf")) {\r
261                         indexResourcePDF(r);\r
262                         return;\r
263                 }\r
264                 if (r.getMime().equalsIgnoreCase("application/docx") || \r
265                         r.getMime().equalsIgnoreCase("application/xlsx") || \r
266                         r.getMime().equalsIgnoreCase("application/pptx")) {\r
267                         indexResourceOOXML(r);\r
268                         return;\r
269                 }\r
270                 if (r.getMime().equalsIgnoreCase("application/vsd") ||\r
271                         r.getMime().equalsIgnoreCase("application/ppt") ||\r
272                         r.getMime().equalsIgnoreCase("application/xls") ||\r
273                         r.getMime().equalsIgnoreCase("application/msg") ||\r
274                         r.getMime().equalsIgnoreCase("application/doc")) {\r
275                                 indexResourceOffice(r);\r
276                                 return;\r
277                 }\r
278                 if (r.getMime().equalsIgnoreCase("application/rtf")) {\r
279                                         indexResourceRTF(r);\r
280                                         return;\r
281                 }\r
282                 if (r.getMime().equalsIgnoreCase("application/odf") ||\r
283                         r.getMime().equalsIgnoreCase("application/odt") ||\r
284                         r.getMime().equalsIgnoreCase("application/odp") ||\r
285                         r.getMime().equalsIgnoreCase("application/odg") ||\r
286                         r.getMime().equalsIgnoreCase("application/odb") ||\r
287                         r.getMime().equalsIgnoreCase("application/ods")) {\r
288                         indexResourceODF(r);\r
289                         return;\r
290                 }\r
291         }\r
292 \r
293 \r
294         private void indexResourceRTF(Resource r) {\r
295 \r
296                 QTemporaryFile f = writeResource(r.getData());\r
297                 if (!keepRunning) {\r
298                         return;\r
299                 }\r
300                 \r
301                 InputStream input;\r
302                 try {\r
303                         input = new FileInputStream(new File(f.fileName()));\r
304                         ContentHandler textHandler = new BodyContentHandler(-1);\r
305                         Metadata metadata = new Metadata();\r
306                         RTFParser parser = new RTFParser();     \r
307                         ParseContext context = new ParseContext();\r
308                         parser.parse(input, textHandler, metadata, context);\r
309                         String[] result = textHandler.toString().split(regex);\r
310                         for (int i=0; i<result.length && keepRunning; i++) {\r
311                                 addToIndex(r.getNoteGuid(), result[i], "RESOURCE");\r
312                         }\r
313                         input.close();\r
314                 \r
315                         f.close();\r
316                 } catch (java.lang.ClassCastException e) {\r
317                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());\r
318                 } catch (FileNotFoundException e) {\r
319                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());\r
320                 } catch (IOException e) {\r
321                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());\r
322                 } catch (SAXException e) {\r
323                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());\r
324                 } catch (TikaException e) {\r
325                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());\r
326                 } catch (Exception e) {\r
327                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());\r
328                 } catch (java.lang.NoSuchMethodError e) {\r
329                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());\r
330                 } catch (Error e) {\r
331                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());\r
332                 }\r
333         }\r
334 \r
335         \r
336         private void indexResourceODF(Resource r) {\r
337 \r
338                 QTemporaryFile f = writeResource(r.getData());\r
339                 if (!keepRunning) {\r
340                         return;\r
341                 }\r
342                 \r
343                 InputStream input;\r
344                 try {\r
345                         input = new FileInputStream(new File(f.fileName()));\r
346                         ContentHandler textHandler = new BodyContentHandler(-1);\r
347                         Metadata metadata = new Metadata();\r
348                         OpenDocumentParser parser = new OpenDocumentParser();   \r
349                         ParseContext context = new ParseContext();\r
350                         parser.parse(input, textHandler, metadata, context);\r
351                         String[] result = textHandler.toString().split(regex);\r
352                         for (int i=0; i<result.length && keepRunning; i++) {\r
353                                 addToIndex(r.getNoteGuid(), result[i], "RESOURCE");\r
354                         }\r
355                         input.close();\r
356                 \r
357                         f.close();\r
358                 } catch (java.lang.ClassCastException e) {\r
359                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());\r
360                 } catch (FileNotFoundException e) {\r
361                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());\r
362                 } catch (IOException e) {\r
363                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());\r
364                 } catch (SAXException e) {\r
365                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());\r
366                 } catch (TikaException e) {\r
367                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());\r
368                 } catch (Exception e) {\r
369                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());\r
370                 } catch (java.lang.NoSuchMethodError e) {\r
371                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());\r
372                 } catch (Error e) {\r
373                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());\r
374                 }\r
375         }\r
376 \r
377         \r
378         private void indexResourceOffice(Resource r) {\r
379 \r
380                 QTemporaryFile f = writeResource(r.getData());\r
381                 if (!keepRunning) {\r
382                         return;\r
383                 }\r
384                 \r
385                 InputStream input;\r
386                 try {\r
387                         input = new FileInputStream(new File(f.fileName()));\r
388                         ContentHandler textHandler = new BodyContentHandler(-1);\r
389                         Metadata metadata = new Metadata();\r
390                         OfficeParser parser = new OfficeParser();       \r
391                         ParseContext context = new ParseContext();\r
392                         parser.parse(input, textHandler, metadata, context);\r
393                         String[] result = textHandler.toString().split(regex);\r
394                         for (int i=0; i<result.length && keepRunning; i++) {\r
395                                 addToIndex(r.getNoteGuid(), result[i], "RESOURCE");\r
396                         }\r
397                         input.close();\r
398                 \r
399                         f.close();\r
400                 } catch (java.lang.ClassCastException e) {\r
401                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());\r
402                 } catch (FileNotFoundException e) {\r
403                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());\r
404                 } catch (IOException e) {\r
405                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());\r
406                 } catch (SAXException e) {\r
407                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());\r
408                 } catch (TikaException e) {\r
409                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());\r
410                 } catch (Exception e) {\r
411                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());\r
412                 } catch (java.lang.NoSuchMethodError e) {\r
413                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());\r
414                 } catch (Error e) {\r
415                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());\r
416                 }\r
417         }\r
418 \r
419         \r
420         \r
421         private void indexResourcePDF(Resource r) {\r
422 \r
423                 QTemporaryFile f = writeResource(r.getData());\r
424                 if (!keepRunning) {\r
425                         return;\r
426                 }\r
427                 \r
428                 InputStream input;\r
429                 try {                   \r
430                         input = new FileInputStream(new File(f.fileName()));\r
431                         ContentHandler textHandler = new BodyContentHandler(-1);\r
432                         Metadata metadata = new Metadata();\r
433                         PDFParser parser = new PDFParser();     \r
434                         ParseContext context = new ParseContext();\r
435                         parser.parse(input, textHandler, metadata, context);\r
436                         String[] result = textHandler.toString().split(regex);\r
437                         for (int i=0; i<result.length && keepRunning; i++) {\r
438                                 addToIndex(r.getNoteGuid(), result[i], "RESOURCE");\r
439                         }\r
440                         input.close();\r
441                 \r
442                         f.close();\r
443                 } catch (java.lang.ClassCastException e) {\r
444                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());\r
445                 } catch (FileNotFoundException e) {\r
446                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());\r
447                 } catch (IOException e) {\r
448                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());\r
449                 } catch (SAXException e) {\r
450                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());\r
451                 } catch (TikaException e) {\r
452                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());\r
453                 } catch (Exception e) {\r
454                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());\r
455                 } catch (java.lang.NoSuchMethodError e) {\r
456                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());\r
457                 } catch (Error e) {\r
458                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());\r
459                 }\r
460         }\r
461         \r
462         \r
463         private void indexResourceOOXML(Resource r) {\r
464 \r
465                 QTemporaryFile f = writeResource(r.getData());\r
466                 if (!keepRunning) {\r
467                         return;\r
468                 }\r
469                 \r
470                 InputStream input;\r
471                 try {\r
472                         input = new FileInputStream(new File(f.fileName()));\r
473                         ContentHandler textHandler = new BodyContentHandler(-1);\r
474                         Metadata metadata = new Metadata();\r
475                         OOXMLParser parser = new OOXMLParser(); \r
476                         ParseContext context = new ParseContext();\r
477                         parser.parse(input, textHandler, metadata, context);\r
478                         String[] result = textHandler.toString().split(regex);\r
479                         for (int i=0; i<result.length && keepRunning; i++) {\r
480                                 addToIndex(r.getNoteGuid(), result[i], "RESOURCE");\r
481                         }\r
482                         input.close();\r
483                 \r
484                         f.close();\r
485                 } catch (java.lang.ClassCastException e) {\r
486                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());\r
487                 } catch (FileNotFoundException e) {\r
488                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());\r
489                 } catch (IOException e) {\r
490                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());\r
491                 } catch (SAXException e) {\r
492                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());\r
493                 } catch (TikaException e) {\r
494                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());\r
495                 } catch (Exception e) {\r
496                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());\r
497                 } catch (java.lang.NoSuchMethodError e) {\r
498                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());\r
499                 } catch (Error e) {\r
500                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());\r
501                 }\r
502         }\r
503         \r
504 \r
505         \r
506         private QTemporaryFile writeResource(Data d) {\r
507                 QTemporaryFile newFile = new QTemporaryFile();\r
508                 newFile.open(OpenModeFlag.WriteOnly);\r
509                 newFile.write(d.getBody());\r
510                 newFile.close();\r
511                 return newFile;\r
512         } \r
513 \r
514         \r
515         private String removeEnCrypt(String content) {\r
516                 int index = content.indexOf("<en-crypt");\r
517                 int endPos;\r
518                 boolean tagFound = true;\r
519                 while (tagFound && keepRunning) {\r
520                         endPos = content.indexOf("</en-crypt>", index)+11;\r
521                         if (endPos > -1 && index > -1) {\r
522                                 content = content.substring(0,index)+content.substring(endPos);\r
523                                 index = content.indexOf("<en-crypt");\r
524                         } else {\r
525                                 tagFound = false;\r
526                         }\r
527                 }\r
528                 return content;\r
529         }\r
530 \r
531         \r
532         private void addToIndex(String guid, String word, String type) {\r
533                 if (!foundWords.contains(word))\r
534                         foundWords.add(word);\r
535                 StringBuffer buffer = new StringBuffer(word.toLowerCase());\r
536                 for (int i=buffer.length()-1; i>=0; i--) {\r
537                         if (!Character.isLetterOrDigit(buffer.charAt(i)))\r
538                                 buffer.deleteCharAt(i);\r
539                         else\r
540                                 break;\r
541                 }\r
542                 buffer = buffer.reverse();\r
543                 for (int i=buffer.length()-1; i>=0; i--) {\r
544                         if (!Character.isLetterOrDigit(buffer.charAt(i)))\r
545                                 buffer.deleteCharAt(i);\r
546                         else\r
547                                 break;\r
548                 }\r
549                 buffer = buffer.reverse();\r
550                 if (buffer.length() > 0) {\r
551                         // We have a good word, now let's trim off junk at the beginning or end\r
552                         if (!foundWords.contains(buffer.toString())) {\r
553                                 foundWords.add(buffer.toString());\r
554                                 conn.getWordsTable().addWordToNoteIndex(guid, buffer.toString(), type, 100);\r
555                         }\r
556                 }\r
557                 return;\r
558         }\r
559         \r
560         private void scanUnindexed() {\r
561                 List<String> notes = conn.getNoteTable().getUnindexed();\r
562                 guid = null;\r
563                 boolean started = false;\r
564                 if (notes.size() > 0) {\r
565                         signal.indexStarted.emit();\r
566                         started = true;\r
567                 }\r
568                 for (int i=0; i<notes.size() && !interrupt && keepRunning; i++) {\r
569                         guid = notes.get(i);\r
570                         if (guid != null && keepRunning) {\r
571                                 indexNoteContent();\r
572                         }\r
573                 }\r
574                 \r
575                 List<String> unindexedResources = conn.getNoteTable().noteResourceTable.getUnindexed();\r
576                 if (unindexedResources.size() > 0 && !started) {\r
577                         signal.indexStarted.emit();\r
578                         started = true;\r
579                 }\r
580                 for (int i=0; i<unindexedResources.size()&& !interrupt && keepRunning; i++) {\r
581                         guid = unindexedResources.get(i);\r
582                         if (keepRunning) {\r
583                                 indexResource();\r
584                         }\r
585                 }\r
586                 if (started && keepRunning && !interrupt) \r
587                         signal.indexFinished.emit();\r
588         }\r
589         \r
590         private void reindexNote() {\r
591                 if (guid == null)\r
592                         return;\r
593                 conn.getNoteTable().setIndexNeeded(guid, true);\r
594         }\r
595         \r
596         private void reindexAll() {\r
597                 conn.getNoteTable().reindexAllNotes();\r
598                 conn.getNoteTable().noteResourceTable.reindexAll(); \r
599         }\r
600 \r
601 }\r