OSDN Git Service

67a1c1cb6e8413d1483c7faa53e5878b4bdad820
[neighbornote/NeighborNote.git] / src / cx / fbn / nevernote / threads / IndexRunner.java
1 /*\r
2  * This file is part of NeverNote \r
3  * Copyright 2009 Randy Baumgarte\r
4  * \r
5  * This file may be licensed under the terms of of the\r
6  * GNU General Public License Version 2 (the ``GPL'').\r
7  *\r
8  * Software distributed under the License is distributed\r
9  * on an ``AS IS'' basis, WITHOUT WARRANTY OF ANY KIND, either\r
10  * express or implied. See the GPL for the specific language\r
11  * governing rights and limitations.\r
12  *\r
13  * You should have received a copy of the GPL along with this\r
14  * program. If not, go to http://www.gnu.org/licenses/gpl.html\r
15  * or write to the Free Software Foundation, Inc.,\r
16  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.\r
17  *\r
18 */\r
19 \r
20 package cx.fbn.nevernote.threads;\r
21 \r
22 import java.io.File;\r
23 import java.io.FileInputStream;\r
24 import java.io.FileNotFoundException;\r
25 import java.io.IOException;\r
26 import java.io.InputStream;\r
27 import java.util.List;\r
28 import java.util.TreeSet;\r
29 import java.util.concurrent.LinkedBlockingQueue;\r
30 import java.util.concurrent.locks.LockSupport;\r
31 \r
32 import org.apache.commons.lang.StringEscapeUtils;\r
33 import org.apache.tika.exception.TikaException;\r
34 import org.apache.tika.metadata.Metadata;\r
35 import org.apache.tika.parser.ParseContext;\r
36 import org.apache.tika.parser.microsoft.OfficeParser;\r
37 import org.apache.tika.parser.microsoft.ooxml.OOXMLParser;\r
38 import org.apache.tika.parser.odf.OpenDocumentParser;\r
39 import org.apache.tika.parser.pdf.PDFParser;\r
40 import org.apache.tika.parser.rtf.RTFParser;\r
41 import org.apache.tika.sax.BodyContentHandler;\r
42 import org.xml.sax.ContentHandler;\r
43 import org.xml.sax.SAXException;\r
44 \r
45 import com.evernote.edam.type.Data;\r
46 import com.evernote.edam.type.Note;\r
47 import com.evernote.edam.type.Resource;\r
48 import com.trolltech.qt.core.QByteArray;\r
49 import com.trolltech.qt.core.QIODevice.OpenModeFlag;\r
50 import com.trolltech.qt.core.QObject;\r
51 import com.trolltech.qt.core.QTemporaryFile;\r
52 import com.trolltech.qt.xml.QDomDocument;\r
53 import com.trolltech.qt.xml.QDomElement;\r
54 import com.trolltech.qt.xml.QDomNodeList;\r
55 \r
56 import cx.fbn.nevernote.Global;\r
57 import cx.fbn.nevernote.signals.IndexSignal;\r
58 import cx.fbn.nevernote.signals.NoteResourceSignal;\r
59 import cx.fbn.nevernote.signals.NoteSignal;\r
60 import cx.fbn.nevernote.sql.DatabaseConnection;\r
61 import cx.fbn.nevernote.utilities.ApplicationLogger;\r
62 \r
63 public class IndexRunner extends QObject implements Runnable {\r
64         \r
65         private final ApplicationLogger         logger;\r
66         private String                                          guid;\r
67         private QByteArray                                      resourceBinary;\r
68         public volatile NoteSignal                      noteSignal;\r
69         public volatile NoteResourceSignal      resourceSignal;\r
70         private int                                                     indexType;\r
71         public final int                                        SCAN=1; \r
72         public final int                                        REINDEXALL=2;\r
73         public final int                                        REINDEXNOTE=3;\r
74         public boolean                                          keepRunning;\r
75         private final QDomDocument                      doc;\r
76         private static String                           regex = Global.getWordRegex();\r
77         private final DatabaseConnection        conn;\r
78         private volatile LinkedBlockingQueue<String> workQueue;\r
79         private static int MAX_QUEUED_WAITING = 1000;\r
80         public boolean interrupt;\r
81         public boolean idle;\r
82         public boolean indexAttachmentsLocally = true;\r
83         public volatile IndexSignal                     signal;\r
84         private final TreeSet<String>           foundWords;\r
85         int uncommittedCount = 0;\r
86 \r
87         \r
88         public IndexRunner(String logname, String u, String i, String r, String uid, String pswd, String cpswd) {\r
89                 foundWords = new TreeSet<String>();\r
90                 logger = new ApplicationLogger(logname);\r
91                 conn = new DatabaseConnection(logger, u, i, r, uid, pswd, cpswd, 500);\r
92                 indexType = SCAN;\r
93                 guid = null;\r
94                 keepRunning = true;\r
95                 doc = new QDomDocument();\r
96                 workQueue=new LinkedBlockingQueue<String>(MAX_QUEUED_WAITING);  \r
97         }\r
98         \r
99         public void setIndexType(int t) {\r
100                 indexType = t;\r
101         }\r
102         \r
103         \r
104         @Override\r
105         public void run() {\r
106                 thread().setPriority(Thread.MIN_PRIORITY);\r
107                 noteSignal = new NoteSignal();\r
108                 resourceSignal = new NoteResourceSignal();\r
109                 signal = new IndexSignal();\r
110                 logger.log(logger.EXTREME, "Starting index thread ");\r
111                 while (keepRunning) {\r
112                         idle=true;\r
113                         try {\r
114                                 conn.commitTransaction();\r
115                                 uncommittedCount = 0;\r
116                                 String work = workQueue.take();\r
117                                 idle=false;\r
118                                 if (work.startsWith("SCAN")) {\r
119                                         guid=null;\r
120                                         interrupt = false;\r
121                                         indexType = SCAN;\r
122                                 }\r
123                                 if (work.startsWith("REINDEXALL")) {\r
124                                         guid = null;\r
125                                         indexType=REINDEXALL;\r
126                                 }\r
127                                 if (work.startsWith("REINDEXNOTE")) {\r
128                                         work = work.replace("REINDEXNOTE ", "");\r
129                                         guid = work;\r
130                                         indexType = REINDEXNOTE;\r
131                                 }\r
132                                 if (work.startsWith("STOP")) {\r
133                                         keepRunning = false;\r
134                                         guid = null;\r
135                                 }\r
136                                 logger.log(logger.EXTREME, "Type:" +indexType);\r
137                                 if (indexType == SCAN && keepRunning) {\r
138                                         logger.log(logger.MEDIUM, "Scanning for unindexed notes & resources");\r
139                                         scanUnindexed();\r
140                                         setIndexType(0);\r
141                                 }\r
142                                 if (indexType == REINDEXALL && keepRunning) {\r
143                                         logger.log(logger.MEDIUM, "Marking all for reindex");\r
144                                         reindexAll();\r
145                                         setIndexType(0);\r
146                                 }\r
147                                 if (indexType == REINDEXNOTE && keepRunning) {\r
148                                         reindexNote();\r
149                                 }\r
150                         } catch (InterruptedException e) {\r
151                                 logger.log(logger.LOW, "Thread interrupted exception: " +e.getMessage());\r
152                         }\r
153                 }\r
154                 logger.log(logger.EXTREME, "Shutting down database");\r
155                 conn.dbShutdown();\r
156                 logger.log(logger.EXTREME, "Database shut down.  Exiting thread");\r
157         }\r
158         \r
159         // Reindex a note\r
160         public void indexNoteContent() {\r
161                 foundWords.clear();\r
162                 \r
163                 logger.log(logger.EXTREME, "Entering indexRunner.indexNoteContent()");\r
164                 \r
165                 logger.log(logger.EXTREME, "Getting note content");\r
166                 Note n = conn.getNoteTable().getNote(guid,true,false,true,true, true);\r
167                 String data = n.getContent();\r
168                 data = conn.getNoteTable().getNoteContentNoUTFConversion(n.getGuid());\r
169                 \r
170                 logger.log(logger.EXTREME, "Removing any encrypted data");\r
171                 data = removeEnCrypt(data.toString());\r
172                 logger.log(logger.EXTREME, "Removing xml markups");\r
173                 String text =  removeTags(StringEscapeUtils.unescapeHtml(data) +" "+\r
174                 n.getTitle());\r
175                                 \r
176                 logger.log(logger.EXTREME, "Splitting words");\r
177                 String[] result = text.toString().split(regex);\r
178                 conn.commitTransaction();\r
179                 conn.beginTransaction();\r
180                 logger.log(logger.EXTREME, "Deleting existing words for note from index");\r
181                 conn.getWordsTable().expungeFromWordIndex(guid, "CONTENT");\r
182                 \r
183                 logger.log(logger.EXTREME, "Number of words found: " +result.length);\r
184                 for (int j=0; j<result.length && keepRunning; j++) {\r
185                         if (interrupt) {\r
186                                 processInterrupt();\r
187                         }\r
188                         if (!result[j].trim().equals("")) {\r
189                                 logger.log(logger.EXTREME, "Result word: " +result[j].trim());\r
190                                 addToIndex(guid, result[j], "CONTENT");\r
191                         }\r
192                 }\r
193                 // If we were interrupted, we will reindex this note next time\r
194                 if (Global.keepRunning) {\r
195                         logger.log(logger.EXTREME, "Resetting note guid needed");\r
196                         conn.getNoteTable().setIndexNeeded(guid, false);\r
197                 } \r
198                 conn.commitTransaction();\r
199                 uncommittedCount = 0;\r
200                 logger.log(logger.EXTREME, "Leaving indexRunner.indexNoteContent()");\r
201         }\r
202         \r
203         \r
204         private String removeTags(String text) {\r
205                 StringBuffer buffer = new StringBuffer(text);\r
206                 boolean inTag = false;\r
207                 for (int i=buffer.length()-1; i>=0; i--) {\r
208                         if (buffer.charAt(i) == '>')\r
209                                 inTag = true;\r
210                         if (buffer.charAt(i) == '<')\r
211                                 inTag = false;\r
212                         if (inTag || buffer.charAt(i) == '<')\r
213                                 buffer.deleteCharAt(i);\r
214                 }\r
215                 \r
216                 return buffer.toString();\r
217         }\r
218 \r
219         \r
220         public synchronized boolean addWork(String request) {\r
221                 if (workQueue.size() == 0) {\r
222                         workQueue.offer(request);\r
223                         return true;\r
224                 }\r
225                 return false;\r
226         }\r
227         \r
228         public synchronized int getWorkQueueSize() {\r
229                 return workQueue.size();\r
230         }\r
231         \r
232         public void indexResource() {\r
233                 \r
234                 if (guid == null)\r
235                         return;\r
236                 foundWords.clear();\r
237                 Resource r = conn.getNoteTable().noteResourceTable.getNoteResourceRecognition(guid);\r
238                 if (r == null || r.getRecognition() == null || r.getRecognition().getBody() == null || r.getRecognition().getBody().length == 0) \r
239                         resourceBinary = new QByteArray(" ");\r
240                 else\r
241                         resourceBinary = new QByteArray(r.getRecognition().getBody());\r
242                 \r
243                 conn.commitTransaction();\r
244                 conn.beginTransaction();\r
245                 conn.getWordsTable().expungeFromWordIndex(r.getNoteGuid(), "RESOURCE");\r
246                 // This is due to an old bug & can be removed at some point in the future 11/23/2010\r
247                 conn.getWordsTable().expungeFromWordIndex(guid, "RESOURCE");   \r
248                 conn.commitTransaction();\r
249                 uncommittedCount = 0;\r
250                 conn.beginTransaction();\r
251                         \r
252                 doc.setContent(resourceBinary);\r
253                 QDomElement docElem = doc.documentElement();\r
254                         \r
255                 // look for text tags\r
256                 QDomNodeList anchors = docElem.elementsByTagName("t");\r
257                 for (int i=0; i<anchors.length() && keepRunning; i++) {\r
258                         if (interrupt) {\r
259                                 if (interrupt) {\r
260                                         processInterrupt();\r
261                                 }\r
262                         }\r
263                         QDomElement enmedia = anchors.at(i).toElement();\r
264                         String weight = new String(enmedia.attribute("w"));\r
265                         String text = new String(enmedia.text()).toLowerCase();\r
266                         if (!text.equals("")) {\r
267                                 conn.getWordsTable().addWordToNoteIndex(r.getNoteGuid(), text, "RESOURCE", new Integer(weight));\r
268                                 uncommittedCount++;\r
269                                 if (uncommittedCount > 100) {\r
270                                         conn.commitTransaction();\r
271                                         uncommittedCount=0;\r
272                                 }\r
273                         }\r
274                 }\r
275                 \r
276                 if (Global.keepRunning && indexAttachmentsLocally) {\r
277                         conn.commitTransaction();\r
278                         uncommittedCount = 0;\r
279                         conn.beginTransaction();\r
280                         indexResourceContent(guid);\r
281                 }\r
282                                 \r
283                 if (Global.keepRunning)\r
284                         conn.getNoteTable().noteResourceTable.setIndexNeeded(guid,false);\r
285                 conn.commitTransaction();\r
286                 uncommittedCount = 0;\r
287         }\r
288         \r
289         private void indexResourceContent(String guid) {\r
290                 Resource r = conn.getNoteTable().noteResourceTable.getNoteResource(guid, true);\r
291                 if (r.getMime().equalsIgnoreCase("application/pdf")) {\r
292                         indexResourcePDF(r);\r
293                         return;\r
294                 }\r
295                 if (r.getMime().equalsIgnoreCase("application/docx") || \r
296                         r.getMime().equalsIgnoreCase("application/xlsx") || \r
297                         r.getMime().equalsIgnoreCase("application/pptx")) {\r
298                         indexResourceOOXML(r);\r
299                         return;\r
300                 }\r
301                 if (r.getMime().equalsIgnoreCase("application/vsd") ||\r
302                         r.getMime().equalsIgnoreCase("application/ppt") ||\r
303                         r.getMime().equalsIgnoreCase("application/xls") ||\r
304                         r.getMime().equalsIgnoreCase("application/msg") ||\r
305                         r.getMime().equalsIgnoreCase("application/doc")) {\r
306                                 indexResourceOffice(r);\r
307                                 return;\r
308                 }\r
309                 if (r.getMime().equalsIgnoreCase("application/rtf")) {\r
310                                         indexResourceRTF(r);\r
311                                         return;\r
312                 }\r
313                 if (r.getMime().equalsIgnoreCase("application/odf") ||\r
314                         r.getMime().equalsIgnoreCase("application/odt") ||\r
315                         r.getMime().equalsIgnoreCase("application/odp") ||\r
316                         r.getMime().equalsIgnoreCase("application/odg") ||\r
317                         r.getMime().equalsIgnoreCase("application/odb") ||\r
318                         r.getMime().equalsIgnoreCase("application/ods")) {\r
319                         indexResourceODF(r);\r
320                         return;\r
321                 }\r
322         }\r
323 \r
324 \r
325         private void indexResourceRTF(Resource r) {\r
326 \r
327                 QTemporaryFile f = writeResource(r.getData());\r
328                 if (!keepRunning) {\r
329                         return;\r
330                 }\r
331                 \r
332                 InputStream input;\r
333                 try {\r
334                         input = new FileInputStream(new File(f.fileName()));\r
335                         ContentHandler textHandler = new BodyContentHandler(-1);\r
336                         Metadata metadata = new Metadata();\r
337                         RTFParser parser = new RTFParser();     \r
338                         ParseContext context = new ParseContext();\r
339                         parser.parse(input, textHandler, metadata, context);\r
340                         String[] result = textHandler.toString().split(regex);\r
341                         for (int i=0; i<result.length && keepRunning; i++) {\r
342                                 addToIndex(r.getNoteGuid(), result[i], "RESOURCE");\r
343                         }\r
344                         input.close();\r
345                 \r
346                         f.close();\r
347                 } catch (java.lang.ClassCastException e) {\r
348                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());\r
349                 } catch (FileNotFoundException e) {\r
350                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());\r
351                 } catch (IOException e) {\r
352                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());\r
353                 } catch (SAXException e) {\r
354                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());\r
355                 } catch (TikaException e) {\r
356                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());\r
357                 } catch (Exception e) {\r
358                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());\r
359                 } catch (java.lang.NoSuchMethodError e) {\r
360                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());\r
361                 } catch (Error e) {\r
362                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());\r
363                 }\r
364         }\r
365 \r
366         \r
367         private void indexResourceODF(Resource r) {\r
368 \r
369                 QTemporaryFile f = writeResource(r.getData());\r
370                 if (!keepRunning) {\r
371                         return;\r
372                 }\r
373                 \r
374                 InputStream input;\r
375                 try {\r
376                         input = new FileInputStream(new File(f.fileName()));\r
377                         ContentHandler textHandler = new BodyContentHandler(-1);\r
378                         Metadata metadata = new Metadata();\r
379                         OpenDocumentParser parser = new OpenDocumentParser();   \r
380                         ParseContext context = new ParseContext();\r
381                         parser.parse(input, textHandler, metadata, context);\r
382                         String[] result = textHandler.toString().split(regex);\r
383                         for (int i=0; i<result.length && keepRunning; i++) {\r
384                                 if (interrupt) {\r
385                                         processInterrupt();\r
386                                 }\r
387                                 addToIndex(r.getNoteGuid(), result[i], "RESOURCE");\r
388                         }\r
389                         input.close();\r
390                 \r
391                         f.close();\r
392                 } catch (java.lang.ClassCastException e) {\r
393                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());\r
394                 } catch (FileNotFoundException e) {\r
395                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());\r
396                 } catch (IOException e) {\r
397                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());\r
398                 } catch (SAXException e) {\r
399                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());\r
400                 } catch (TikaException e) {\r
401                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());\r
402                 } catch (Exception e) {\r
403                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());\r
404                 } catch (java.lang.NoSuchMethodError e) {\r
405                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());\r
406                 } catch (Error e) {\r
407                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());\r
408                 }\r
409         }\r
410 \r
411         \r
412         private void indexResourceOffice(Resource r) {\r
413 \r
414                 QTemporaryFile f = writeResource(r.getData());\r
415                 if (!keepRunning) {\r
416                         return;\r
417                 }\r
418                 \r
419                 InputStream input;\r
420                 try {\r
421                         input = new FileInputStream(new File(f.fileName()));\r
422                         ContentHandler textHandler = new BodyContentHandler(-1);\r
423                         Metadata metadata = new Metadata();\r
424                         OfficeParser parser = new OfficeParser();       \r
425                         ParseContext context = new ParseContext();\r
426                         parser.parse(input, textHandler, metadata, context);\r
427                         String[] result = textHandler.toString().split(regex);\r
428                         for (int i=0; i<result.length && keepRunning; i++) {\r
429                                 if (interrupt) {\r
430                                         processInterrupt();\r
431                                 }\r
432                                 addToIndex(r.getNoteGuid(), result[i], "RESOURCE");\r
433                         }\r
434                         input.close();\r
435                 \r
436                         f.close();\r
437                 } catch (java.lang.ClassCastException e) {\r
438                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());\r
439                 } catch (FileNotFoundException e) {\r
440                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());\r
441                 } catch (IOException e) {\r
442                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());\r
443                 } catch (SAXException e) {\r
444                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());\r
445                 } catch (TikaException e) {\r
446                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());\r
447                 } catch (Exception e) {\r
448                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());\r
449                 } catch (java.lang.NoSuchMethodError e) {\r
450                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());\r
451                 } catch (Error e) {\r
452                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());\r
453                 }\r
454         }\r
455 \r
456         \r
457         \r
458         private void indexResourcePDF(Resource r) {\r
459 \r
460                 QTemporaryFile f = writeResource(r.getData());\r
461                 if (!keepRunning) {\r
462                         return;\r
463                 }\r
464                 \r
465                 InputStream input;\r
466                 try {                   \r
467                         input = new FileInputStream(new File(f.fileName()));\r
468                         ContentHandler textHandler = new BodyContentHandler(-1);\r
469                         Metadata metadata = new Metadata();\r
470                         PDFParser parser = new PDFParser();     \r
471                         ParseContext context = new ParseContext();\r
472                         parser.parse(input, textHandler, metadata, context);\r
473                         String[] result = textHandler.toString().split(regex);\r
474                         for (int i=0; i<result.length && keepRunning; i++) {\r
475                                 if (interrupt) {\r
476                                         processInterrupt();\r
477                                 }\r
478                                 addToIndex(r.getNoteGuid(), result[i], "RESOURCE");\r
479                         }\r
480                         input.close();\r
481                 \r
482                         f.close();\r
483                 } catch (java.lang.ClassCastException e) {\r
484                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());\r
485                 } catch (FileNotFoundException e) {\r
486                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());\r
487                 } catch (IOException e) {\r
488                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());\r
489                 } catch (SAXException e) {\r
490                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());\r
491                 } catch (TikaException e) {\r
492                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());\r
493                 } catch (Exception e) {\r
494                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());\r
495                 } catch (java.lang.NoSuchMethodError e) {\r
496                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());\r
497                 } catch (Error e) {\r
498                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());\r
499                 }\r
500         }\r
501         \r
502         \r
503         private void indexResourceOOXML(Resource r) {\r
504 \r
505                 QTemporaryFile f = writeResource(r.getData());\r
506                 if (!keepRunning) {\r
507                         return;\r
508                 }\r
509                 \r
510                 InputStream input;\r
511                 try {\r
512                         input = new FileInputStream(new File(f.fileName()));\r
513                         ContentHandler textHandler = new BodyContentHandler(-1);\r
514                         Metadata metadata = new Metadata();\r
515                         OOXMLParser parser = new OOXMLParser(); \r
516                         ParseContext context = new ParseContext();\r
517                         parser.parse(input, textHandler, metadata, context);\r
518                         String[] result = textHandler.toString().split(regex);\r
519                         for (int i=0; i<result.length && keepRunning; i++) {\r
520                                 if (interrupt) {\r
521                                         processInterrupt();\r
522                                 }\r
523                                 addToIndex(r.getNoteGuid(), result[i], "RESOURCE");\r
524                         }\r
525                         input.close();\r
526                 \r
527                         f.close();\r
528                 } catch (java.lang.ClassCastException e) {\r
529                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());\r
530                 } catch (FileNotFoundException e) {\r
531                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());\r
532                 } catch (IOException e) {\r
533                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());\r
534                 } catch (SAXException e) {\r
535                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());\r
536                 } catch (TikaException e) {\r
537                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());\r
538                 } catch (Exception e) {\r
539                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());\r
540                 } catch (java.lang.NoSuchMethodError e) {\r
541                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());\r
542                 } catch (Error e) {\r
543                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());\r
544                 }\r
545         }\r
546         \r
547 \r
548         \r
549         private QTemporaryFile writeResource(Data d) {\r
550                 QTemporaryFile newFile = new QTemporaryFile();\r
551                 newFile.open(OpenModeFlag.WriteOnly);\r
552                 newFile.write(d.getBody());\r
553                 newFile.close();\r
554                 return newFile;\r
555         } \r
556 \r
557         \r
558         private String removeEnCrypt(String content) {\r
559                 int index = content.indexOf("<en-crypt");\r
560                 int endPos;\r
561                 boolean tagFound = true;\r
562                 while (tagFound && keepRunning) {\r
563                         if (interrupt) {\r
564                                 processInterrupt();\r
565                         }\r
566                         endPos = content.indexOf("</en-crypt>", index)+11;\r
567                         if (endPos > -1 && index > -1) {\r
568                                 content = content.substring(0,index)+content.substring(endPos);\r
569                                 index = content.indexOf("<en-crypt");\r
570                         } else {\r
571                                 tagFound = false;\r
572                         }\r
573                 }\r
574                 return content;\r
575         }\r
576 \r
577         \r
578         private void addToIndex(String guid, String word, String type) {\r
579                 if (foundWords.contains(word))\r
580                         return;\r
581                 StringBuffer buffer = new StringBuffer(word.toLowerCase());\r
582                 for (int i=buffer.length()-1; i>=0; i--) {\r
583                         if (!Character.isLetterOrDigit(buffer.charAt(i)))\r
584                                 buffer.deleteCharAt(i);\r
585                         else\r
586                                 break;\r
587                 }\r
588                 buffer = buffer.reverse();\r
589                 for (int i=buffer.length()-1; i>=0; i--) {\r
590                         if (!Character.isLetterOrDigit(buffer.charAt(i)))\r
591                                 buffer.deleteCharAt(i);\r
592                         else\r
593                                 break;\r
594                 }\r
595                 buffer = buffer.reverse();\r
596                 if (buffer.length() > 0) {\r
597                         // We have a good word, now let's trim off junk at the beginning or end\r
598                         if (!foundWords.contains(buffer.toString())) {\r
599                                 foundWords.add(buffer.toString());\r
600                                 foundWords.add(word);\r
601                                 conn.getWordsTable().addWordToNoteIndex(guid, buffer.toString(), type, 100);\r
602                                 uncommittedCount++;\r
603                                 if (uncommittedCount > 100) {\r
604                                         conn.commitTransaction();\r
605                                         uncommittedCount=0;\r
606                                 }\r
607                         }\r
608                 }\r
609                 return;\r
610         }\r
611         \r
612         private void scanUnindexed() {\r
613                 List<String> notes = conn.getNoteTable().getUnindexed();\r
614                 guid = null;\r
615                 boolean started = false;\r
616                 if (notes.size() > 0) {\r
617                         signal.indexStarted.emit();\r
618                         started = true;\r
619                 }\r
620                 for (int i=0; i<notes.size() && keepRunning; i++) {\r
621                         if (interrupt) {\r
622                                 processInterrupt();\r
623                         }\r
624                         guid = notes.get(i);\r
625                         if (guid != null && keepRunning) {\r
626                                 indexNoteContent();\r
627                         }\r
628                 }\r
629                 \r
630                 List<String> unindexedResources = conn.getNoteTable().noteResourceTable.getUnindexed();\r
631                 if (unindexedResources.size() > 0 && !started) {\r
632                         signal.indexStarted.emit();\r
633                         started = true;\r
634                 }\r
635                 for (int i=0; i<unindexedResources.size()&& keepRunning; i++) {\r
636                         if (interrupt) {\r
637                                 processInterrupt();\r
638                         }\r
639                         guid = unindexedResources.get(i);\r
640                         if (keepRunning) {\r
641                                 indexResource();\r
642                         }\r
643                 }\r
644                 if (started && keepRunning) \r
645                         signal.indexFinished.emit();\r
646         }\r
647         \r
648         private void reindexNote() {\r
649                 if (guid == null)\r
650                         return;\r
651                 conn.getNoteTable().setIndexNeeded(guid, true);\r
652         }\r
653         \r
654         private void reindexAll() {\r
655                 conn.getNoteTable().reindexAllNotes();\r
656                 conn.getNoteTable().noteResourceTable.reindexAll(); \r
657         }\r
658 \r
659         private void waitSeconds(int len) {\r
660                 long starttime = 0; // variable declared\r
661                 //...\r
662                 // for the first time, remember the timestamp\r
663             starttime = System.currentTimeMillis();\r
664                 // the next timestamp we want to wake up\r
665                 starttime += (1000.0);\r
666                 // Wait until the desired next time arrives using nanosecond\r
667                 // accuracy timer (wait(time) isn't accurate enough on most platforms) \r
668                 LockSupport.parkNanos((Math.max(0, \r
669                     starttime - System.currentTimeMillis()) * 1000000));\r
670         }\r
671         \r
672         private void processInterrupt() {\r
673                 conn.commitTransaction();\r
674                 waitSeconds(1);\r
675                 uncommittedCount = 0;\r
676                 conn.beginTransaction();\r
677                 interrupt = false;\r
678         }\r
679         \r
680 }\r