OSDN Git Service

c6a8dc3ef9132327f3f11d4ca868e45fa8a0f37e
[neighbornote/NeighborNote.git] / src / cx / fbn / nevernote / threads / IndexRunner.java
1 /*
2  * This file is part of NixNote/NeighborNote 
3  * Copyright 2009 Randy Baumgarte
4  * Copyright 2013 Yuki Takahashi
5  * 
6  * This file may be licensed under the terms of of the
7  * GNU General Public License Version 2 (the ``GPL'').
8  *
9  * Software distributed under the License is distributed
10  * on an ``AS IS'' basis, WITHOUT WARRANTY OF ANY KIND, either
11  * express or implied. See the GPL for the specific language
12  * governing rights and limitations.
13  *
14  * You should have received a copy of the GPL along with this
15  * program. If not, go to http://www.gnu.org/licenses/gpl.html
16  * or write to the Free Software Foundation, Inc.,
17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18  *
19 */
20
21 package cx.fbn.nevernote.threads;
22
23 import java.io.File;
24 import java.io.FileInputStream;
25 import java.io.FileNotFoundException;
26 import java.io.IOException;
27 import java.io.InputStream;
28 import java.util.List;
29 import java.util.TreeSet;
30 import java.util.concurrent.LinkedBlockingQueue;
31 import java.util.concurrent.locks.LockSupport;
32
33 import org.apache.commons.lang3.StringEscapeUtils;
34 import org.apache.tika.exception.TikaException;
35 import org.apache.tika.metadata.Metadata;
36 import org.apache.tika.parser.ParseContext;
37 import org.apache.tika.parser.microsoft.OfficeParser;
38 import org.apache.tika.parser.microsoft.ooxml.OOXMLParser;
39 import org.apache.tika.parser.odf.OpenDocumentParser;
40 import org.apache.tika.parser.pdf.PDFParser;
41 import org.apache.tika.parser.rtf.RTFParser;
42 import org.apache.tika.sax.BodyContentHandler;
43 import org.xml.sax.ContentHandler;
44 import org.xml.sax.SAXException;
45
46 import com.evernote.edam.type.Data;
47 import com.evernote.edam.type.Note;
48 import com.evernote.edam.type.Resource;
49 import com.trolltech.qt.core.QByteArray;
50 import com.trolltech.qt.core.QIODevice.OpenModeFlag;
51 import com.trolltech.qt.core.QObject;
52 import com.trolltech.qt.core.QTemporaryFile;
53 import com.trolltech.qt.xml.QDomDocument;
54 import com.trolltech.qt.xml.QDomElement;
55 import com.trolltech.qt.xml.QDomNodeList;
56
57 import cx.fbn.nevernote.Global;
58 import cx.fbn.nevernote.signals.IndexSignal;
59 import cx.fbn.nevernote.signals.NoteResourceSignal;
60 import cx.fbn.nevernote.signals.NoteSignal;
61 import cx.fbn.nevernote.sql.DatabaseConnection;
62 import cx.fbn.nevernote.utilities.ApplicationLogger;
63
64 public class IndexRunner extends QObject implements Runnable {
65         
66         private final ApplicationLogger         logger;
67         private String                                          guid;
68         private QByteArray                                      resourceBinary;
69         public volatile NoteSignal                      noteSignal;
70         public volatile NoteResourceSignal      resourceSignal;
71         private int                                                     indexType;
72         public final int                                        SCAN=1; 
73         public final int                                        REINDEXALL=2;
74         public final int                                        REINDEXNOTE=3;
75         public boolean                                          keepRunning;
76         private final QDomDocument                      doc;
77         private static String                           regex = Global.getWordRegex();
78         public String                                           specialIndexCharacters = "";
79         public boolean                                          indexNoteBody = true;
80         public boolean                                          indexNoteTitle = true;
81         public boolean                                          indexImageRecognition = true;
82         private final DatabaseConnection        conn;
83         private volatile LinkedBlockingQueue<String> workQueue;
84         private static int MAX_QUEUED_WAITING = 1000;
85         public boolean interrupt;
86         public boolean idle;
87         public boolean indexAttachmentsLocally = true;
88         public volatile IndexSignal                     signal;
89         private final TreeSet<String>           foundWords;
90         int uncommittedCount = 0;
91
92         // ICHANGED String bを追加
93         public IndexRunner(String logname, String u, String i, String r, String b, String uid, String pswd, String cpswd) {
94                 foundWords = new TreeSet<String>();
95                 logger = new ApplicationLogger(logname);
96                 // ICHANGED bを追加
97                 conn = new DatabaseConnection(logger, u, i, r, b, uid, pswd, cpswd, 500);
98                 indexType = SCAN;
99                 guid = null;
100                 keepRunning = true;
101                 doc = new QDomDocument();
102                 workQueue=new LinkedBlockingQueue<String>(MAX_QUEUED_WAITING);  
103         }
104         
105         public void setIndexType(int t) {
106                 indexType = t;
107         }
108         
109         
110         @Override
111         public void run() {
112                 thread().setPriority(Thread.MIN_PRIORITY);
113                 noteSignal = new NoteSignal();
114                 resourceSignal = new NoteResourceSignal();
115                 signal = new IndexSignal();
116                 logger.log(logger.EXTREME, "Starting index thread ");
117                 while (keepRunning) {
118                         idle=true;
119                         try {
120                                 conn.commitTransaction();
121                                 uncommittedCount = 0;
122                                 String work = workQueue.take();
123                                 idle=false;
124                                 if (work.startsWith("SCAN")) {
125                                         guid=null;
126                                         interrupt = false;
127                                         indexType = SCAN;
128                                 }
129                                 if (work.startsWith("REINDEXALL")) {
130                                         guid = null;
131                                         indexType=REINDEXALL;
132                                 }
133                                 if (work.startsWith("REINDEXNOTE")) {
134                                         work = work.replace("REINDEXNOTE ", "");
135                                         guid = work;
136                                         indexType = REINDEXNOTE;
137                                 }
138                                 if (work.startsWith("STOP")) {
139                                         keepRunning = false;
140                                         guid = null;
141                                 }
142                                 logger.log(logger.EXTREME, "Type:" +indexType);
143                                 if (indexType == SCAN && keepRunning) {
144                                         logger.log(logger.MEDIUM, "Scanning for unindexed notes & resources");
145                                         scanUnindexed();
146                                         setIndexType(0);
147                                 }
148                                 if (indexType == REINDEXALL && keepRunning) {
149                                         logger.log(logger.MEDIUM, "Marking all for reindex");
150                                         reindexAll();
151                                         setIndexType(0);
152                                 }
153                                 if (indexType == REINDEXNOTE && keepRunning) {
154                                         reindexNote();
155                                 }
156                         } catch (InterruptedException e) {
157                                 logger.log(logger.LOW, "Thread interrupted exception: " +e.getMessage());
158                         }
159                 }
160                 logger.log(logger.EXTREME, "Shutting down database");
161                 conn.dbShutdown();
162                 logger.log(logger.EXTREME, "Database shut down.  Exiting thread");
163         }
164         
165         // Reindex a note
166         public void indexNoteContent() {
167                 foundWords.clear();
168                 
169                 logger.log(logger.EXTREME, "Entering indexRunner.indexNoteContent()");
170                 
171                 logger.log(logger.EXTREME, "Getting note content");
172                 Note n = conn.getNoteTable().getNote(guid,true,false,true,true, true);
173                 String data;
174                 if (indexNoteBody) {
175                         data = n.getContent();
176                         data = conn.getNoteTable().getNoteContentNoUTFConversion(n.getGuid());
177                 
178                         logger.log(logger.EXTREME, "Removing any encrypted data");
179                         data = removeEnCrypt(data.toString());
180                         logger.log(logger.EXTREME, "Removing xml markups");
181                 } else
182                         data = "";
183                 String text;
184                 if (indexNoteTitle)
185                         text =  removeTags(StringEscapeUtils.unescapeHtml4(data) +" "+ n.getTitle());
186                 else
187                         text = removeTags(StringEscapeUtils.unescapeHtml4(data));
188                                 
189                 logger.log(logger.EXTREME, "Splitting words");
190                 String[] result = text.toString().split(regex);
191                 conn.commitTransaction();
192                 conn.beginTransaction();
193                 logger.log(logger.EXTREME, "Deleting existing words for note from index");
194                 conn.getWordsTable().expungeFromWordIndex(guid, "CONTENT");
195                 
196                 logger.log(logger.EXTREME, "Number of words found: " +result.length);
197                 for (int j=0; j<result.length && keepRunning; j++) {
198                         if (interrupt) {
199                                 processInterrupt();
200                         }
201                         if (!result[j].trim().equals("")) {
202                                 logger.log(logger.EXTREME, "Result word: " +result[j].trim());
203                                 addToIndex(guid, result[j], "CONTENT");
204                         }
205                 }
206                 
207                 // Add tags
208                 for (int j=0; j<n.getTagNamesSize(); j++) {
209                         if (n.getTagNames() != null && n.getTagNames().get(j) != null && !n.getTagNames().get(j).trim().equals(""))
210                                 addToIndex(guid, n.getTagNames().get(j), "CONTENT");
211                 }
212                 
213                 // If we were interrupted, we will reindex this note next time
214                 if (Global.keepRunning) {
215                         logger.log(logger.EXTREME, "Resetting note guid needed");
216                         conn.getNoteTable().setIndexNeeded(guid, false);
217                 } 
218                 conn.commitTransaction();
219                 uncommittedCount = 0;
220                 logger.log(logger.EXTREME, "Leaving indexRunner.indexNoteContent()");
221         }
222         
223         
224         private String removeTags(String text) {
225                 StringBuffer buffer = new StringBuffer(text);
226                 boolean inTag = false;
227                 for (int i=buffer.length()-1; i>=0; i--) {
228                         if (buffer.charAt(i) == '>')
229                                 inTag = true;
230                         if (buffer.charAt(i) == '<')
231                                 inTag = false;
232                         if (inTag || buffer.charAt(i) == '<')
233                                 buffer.deleteCharAt(i);
234                 }
235                 
236                 return buffer.toString();
237         }
238
239         
240         public synchronized boolean addWork(String request) {
241                 if (workQueue.size() == 0) {
242                         workQueue.offer(request);
243                         return true;
244                 }
245                 return false;
246         }
247         
248         public synchronized int getWorkQueueSize() {
249                 return workQueue.size();
250         }
251         
252         public void indexResource() {
253                 
254                 if (guid == null)
255                         return;
256                 foundWords.clear();
257                 Resource r = conn.getNoteTable().noteResourceTable.getNoteResourceRecognition(guid);
258                 if (!indexImageRecognition || 
259                                 r == null || r.getRecognition() == null || 
260                                 r.getRecognition().getBody() == null || 
261                                 r.getRecognition().getBody().length == 0) 
262                         resourceBinary = new QByteArray(" ");
263                 else
264                         resourceBinary = new QByteArray(r.getRecognition().getBody());
265                 
266                 conn.commitTransaction();
267                 conn.beginTransaction();
268                 conn.getWordsTable().expungeFromWordIndex(r.getNoteGuid(), "RESOURCE");
269                 // This is due to an old bug & can be removed at some point in the future 11/23/2010
270                 conn.getWordsTable().expungeFromWordIndex(guid, "RESOURCE");   
271                 conn.commitTransaction();
272                 uncommittedCount = 0;
273                 conn.beginTransaction();
274                         
275                 doc.setContent(resourceBinary);
276                 QDomElement docElem = doc.documentElement();
277                         
278                 // look for text tags
279                 QDomNodeList anchors = docElem.elementsByTagName("t");
280                 for (int i=0; i<anchors.length() && keepRunning; i++) {
281                         if (interrupt) {
282                                 if (interrupt) {
283                                         processInterrupt();
284                                 }
285                         }
286                         QDomElement enmedia = anchors.at(i).toElement();
287                         String weight = new String(enmedia.attribute("w"));
288                         String text = new String(enmedia.text()).toLowerCase();
289                         if (!text.equals("")) {
290                                 conn.getWordsTable().addWordToNoteIndex(r.getNoteGuid(), text, "RESOURCE", new Integer(weight));
291                                 uncommittedCount++;
292                                 if (uncommittedCount > 100) {
293                                         conn.commitTransaction();
294                                         uncommittedCount=0;
295                                 }
296                         }
297                 }
298                 
299                 if (Global.keepRunning && indexAttachmentsLocally) {
300                         conn.commitTransaction();
301                         uncommittedCount = 0;
302                         conn.beginTransaction();
303                         indexResourceContent(guid);
304                 }
305                                 
306                 if (Global.keepRunning)
307                         conn.getNoteTable().noteResourceTable.setIndexNeeded(guid,false);
308                 conn.commitTransaction();
309                 uncommittedCount = 0;
310         }
311         
312         private void indexResourceContent(String guid) {
313                 Resource r = conn.getNoteTable().noteResourceTable.getNoteResource(guid, true);
314                 if (r != null && r.getMime() != null) {
315                         if (r.getMime().equalsIgnoreCase("application/pdf")) {
316                                 indexResourcePDF(r);
317                                 return;
318                         }
319                         if (r.getMime().equalsIgnoreCase("application/docx") || 
320                                 r.getMime().equalsIgnoreCase("application/xlsx") || 
321                                 r.getMime().equalsIgnoreCase("application/pptx")) {
322                                 indexResourceOOXML(r);
323                                 return;
324                         }
325                         if (r.getMime().equalsIgnoreCase("application/vsd") ||
326                                         r.getMime().equalsIgnoreCase("application/ppt") ||
327                                         r.getMime().equalsIgnoreCase("application/xls") ||
328                                         r.getMime().equalsIgnoreCase("application/msg") ||
329                                         r.getMime().equalsIgnoreCase("application/doc")) {
330                                 indexResourceOffice(r);
331                                 return;
332                         }
333                         if (r.getMime().equalsIgnoreCase("application/rtf")) {
334                                         indexResourceRTF(r);
335                                         return;
336                         }
337                         if (r.getMime().equalsIgnoreCase("application/odf") ||
338                                 r.getMime().equalsIgnoreCase("application/odt") ||
339                                 r.getMime().equalsIgnoreCase("application/odp") ||
340                                 r.getMime().equalsIgnoreCase("application/odg") ||
341                                 r.getMime().equalsIgnoreCase("application/odb") ||
342                                 r.getMime().equalsIgnoreCase("application/ods")) {
343                                 indexResourceODF(r);
344                                 return;
345                         }
346                 }
347         }
348
349
350         private void indexResourceRTF(Resource r) {
351
352                 Data d = r.getData();
353                 for (int i=0; i<20 && d.getSize() == 0; i++)
354                         d = r.getData();
355                 if (d.getSize()== 0)
356                         return;
357
358                 QTemporaryFile f = writeResource(d);
359                 if (!keepRunning) {
360                         return;
361                 }
362                 
363                 InputStream input;
364                 try {
365                         input = new FileInputStream(new File(f.fileName()));
366                         ContentHandler textHandler = new BodyContentHandler(-1);
367                         Metadata metadata = new Metadata();
368                         RTFParser parser = new RTFParser();     
369                         ParseContext context = new ParseContext();
370                         parser.parse(input, textHandler, metadata, context);
371                         String[] result = textHandler.toString().split(regex);
372                         for (int i=0; i<result.length && keepRunning; i++) {
373                                 addToIndex(r.getNoteGuid(), result[i], "RESOURCE");
374                         }
375                         input.close();
376                 
377                         f.close();
378                 } catch (java.lang.ClassCastException e) {
379                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());
380                 } catch (FileNotFoundException e) {
381                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());
382                 } catch (IOException e) {
383                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());
384                 } catch (SAXException e) {
385                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());
386                 } catch (TikaException e) {
387                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());
388                 } catch (Exception e) {
389                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());
390                 } catch (java.lang.NoSuchMethodError e) {
391                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());
392                 } catch (Error e) {
393                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());
394                 }
395         }
396
397         
398         private void indexResourceODF(Resource r) {
399
400                 Data d = r.getData();
401                 for (int i=0; i<20 && d.getSize() == 0; i++)
402                         d = r.getData();
403                 if (d.getSize()== 0)
404                         return;
405                 QTemporaryFile f = writeResource(d);
406                 if (!keepRunning) {
407                         return;
408                 }
409                 
410                 InputStream input;
411                 try {
412                         input = new FileInputStream(new File(f.fileName()));
413                         ContentHandler textHandler = new BodyContentHandler(-1);
414                         Metadata metadata = new Metadata();
415                         OpenDocumentParser parser = new OpenDocumentParser();   
416                         ParseContext context = new ParseContext();
417                         parser.parse(input, textHandler, metadata, context);
418                         String[] result = textHandler.toString().split(regex);
419                         for (int i=0; i<result.length && keepRunning; i++) {
420                                 if (interrupt) {
421                                         processInterrupt();
422                                 }
423                                 addToIndex(r.getNoteGuid(), result[i], "RESOURCE");
424                         }
425                         input.close();
426                 
427                         f.close();
428                 } catch (java.lang.ClassCastException e) {
429                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());
430                 } catch (FileNotFoundException e) {
431                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());
432                 } catch (IOException e) {
433                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());
434                 } catch (SAXException e) {
435                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());
436                 } catch (TikaException e) {
437                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());
438                 } catch (Exception e) {
439                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());
440                 } catch (java.lang.NoSuchMethodError e) {
441                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());
442                 } catch (Error e) {
443                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());
444                 }
445         }
446
447         
448         private void indexResourceOffice(Resource r) {
449
450                 Data d = r.getData();
451                 for (int i=0; i<20 && d.getSize() == 0; i++)
452                         d = r.getData();
453                 if (d.getSize()== 0)
454                         return;
455                 QTemporaryFile f = writeResource(d);
456                 if (!keepRunning) {
457                         return;
458                 }
459                 
460                 InputStream input;
461                 try {
462                         input = new FileInputStream(new File(f.fileName()));
463                         ContentHandler textHandler = new BodyContentHandler(-1);
464                         Metadata metadata = new Metadata();
465                         OfficeParser parser = new OfficeParser();       
466                         ParseContext context = new ParseContext();
467                         parser.parse(input, textHandler, metadata, context);
468                         String[] result = textHandler.toString().split(regex);
469                         for (int i=0; i<result.length && keepRunning; i++) {
470                                 if (interrupt) {
471                                         processInterrupt();
472                                 }
473                                 addToIndex(r.getNoteGuid(), result[i], "RESOURCE");
474                         }
475                         input.close();
476                 
477                         f.close();
478                 } catch (java.lang.ClassCastException e) {
479                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());
480                 } catch (FileNotFoundException e) {
481                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());
482                 } catch (IOException e) {
483                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());
484                 } catch (SAXException e) {
485                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());
486                 } catch (TikaException e) {
487                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());
488                 } catch (Exception e) {
489                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());
490                 } catch (java.lang.NoSuchMethodError e) {
491                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());
492                 } catch (Error e) {
493                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());
494                 }
495         }
496
497         
498         
499         private void indexResourcePDF(Resource r) {
500
501                 Data d = r.getData();
502                 for (int i=0; i<20 && d.getSize() == 0; i++)
503                         d = r.getData();
504                 if (d.getSize()== 0)
505                         return;
506                 QTemporaryFile f = writeResource(d);
507                 if (!keepRunning) {
508                         return;
509                 }
510                 
511                 InputStream input;
512                 try {                   
513                         input = new FileInputStream(new File(f.fileName()));
514                         ContentHandler textHandler = new BodyContentHandler(-1);
515                         Metadata metadata = new Metadata();
516                         PDFParser parser = new PDFParser();     
517                         ParseContext context = new ParseContext();
518                         parser.parse(input, textHandler, metadata, context);
519                         String[] result = textHandler.toString().split(regex);
520                         for (int i=0; i<result.length && keepRunning; i++) {
521                                 if (interrupt) {
522                                         processInterrupt();
523                                 }
524                                 addToIndex(r.getNoteGuid(), result[i], "RESOURCE");
525                         }
526                         input.close();
527                 
528                         f.close();
529                 } catch (java.lang.ClassCastException e) {
530                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());
531                 } catch (FileNotFoundException e) {
532                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());
533                 } catch (IOException e) {
534                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());
535                 } catch (SAXException e) {
536                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());
537                 } catch (TikaException e) {
538                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());
539                 } catch (Exception e) {
540                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());
541                 } catch (java.lang.NoSuchMethodError e) {
542                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());
543                 } catch (Error e) {
544                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());
545                 }
546         }
547         
548         
549         private void indexResourceOOXML(Resource r) {
550
551                 Data d = r.getData();
552                 for (int i=0; i<20 && d.getSize() == 0; i++)
553                         d = r.getData();
554                 if (d.getSize()== 0)
555                         return;
556                 QTemporaryFile f = writeResource(d);
557                 if (!keepRunning) {
558                         return;
559                 }
560                 
561                 InputStream input;
562                 try {
563                         input = new FileInputStream(new File(f.fileName()));
564                         ContentHandler textHandler = new BodyContentHandler(-1);
565                         Metadata metadata = new Metadata();
566                         OOXMLParser parser = new OOXMLParser(); 
567                         ParseContext context = new ParseContext();
568                         parser.parse(input, textHandler, metadata, context);
569                         String[] result = textHandler.toString().split(regex);
570                         for (int i=0; i<result.length && keepRunning; i++) {
571                                 if (interrupt) {
572                                         processInterrupt();
573                                 }
574                                 addToIndex(r.getNoteGuid(), result[i], "RESOURCE");
575                         }
576                         input.close();
577                 
578                         f.close();
579                 } catch (java.lang.ClassCastException e) {
580                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());
581                 } catch (FileNotFoundException e) {
582                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());
583                 } catch (IOException e) {
584                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());
585                 } catch (SAXException e) {
586                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());
587                 } catch (TikaException e) {
588                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());
589                 } catch (Exception e) {
590                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());
591                 } catch (java.lang.NoSuchMethodError e) {
592                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());
593                 } catch (Error e) {
594                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());              }
595         }
596         
597
598         
599         private QTemporaryFile writeResource(Data d) {
600                 QTemporaryFile newFile = new QTemporaryFile();
601                 newFile.open(OpenModeFlag.WriteOnly);
602                 newFile.write(d.getBody());
603                 newFile.close();
604                 return newFile;
605         } 
606
607         
608         private String removeEnCrypt(String content) {
609                 int index = content.indexOf("<en-crypt");
610                 int endPos;
611                 boolean tagFound = true;
612                 while (tagFound && keepRunning) {
613                         if (interrupt) {
614                                 processInterrupt();
615                         }
616                         endPos = content.indexOf("</en-crypt>", index)+11;
617                         if (endPos > -1 && index > -1) {
618                                 content = content.substring(0,index)+content.substring(endPos);
619                                 index = content.indexOf("<en-crypt");
620                         } else {
621                                 tagFound = false;
622                         }
623                 }
624                 return content;
625         }
626
627         
628         private void addToIndex(String guid, String word, String type) {
629                 if (foundWords.contains(word))
630                         return;
631                 StringBuffer buffer = new StringBuffer(word.toLowerCase());
632                 for (int i=buffer.length()-1; i>=0; i--) {
633                         if (!Character.isLetterOrDigit(buffer.charAt(i)) && specialIndexCharacters.indexOf(buffer.charAt(i)) == -1)
634                                 buffer.deleteCharAt(i);
635                         else
636                                 break;
637                 }
638                 buffer = buffer.reverse();
639                 for (int i=buffer.length()-1; i>=0; i--) {
640                         if (!Character.isLetterOrDigit(buffer.charAt(i)))
641                                 buffer.deleteCharAt(i);
642                         else
643                                 break;
644                 }
645                 buffer = buffer.reverse();
646                 if (buffer.length() > 0) {
647                         // We have a good word, now let's trim off junk at the beginning or end
648                         if (!foundWords.contains(buffer.toString())) {
649                                 foundWords.add(buffer.toString());
650                                 foundWords.add(word);
651                                 conn.getWordsTable().addWordToNoteIndex(guid, buffer.toString(), type, 100);
652                                 uncommittedCount++;
653                                 if (uncommittedCount > 100) {
654                                         conn.commitTransaction();
655                                         uncommittedCount=0;
656                                 }
657                         }
658                 }
659                 return;
660         }
661         
662         private void scanUnindexed() {
663                 List<String> notes = conn.getNoteTable().getUnindexed();
664                 guid = null;
665                 boolean started = false;
666                 if (notes.size() > 0) {
667                         signal.indexStarted.emit();
668                         started = true;
669                 }
670                 for (int i=0; i<notes.size() && keepRunning; i++) {
671                         if (interrupt) {
672                                 processInterrupt();
673                         }
674                         guid = notes.get(i);
675                         if (guid != null && keepRunning) {
676                                 indexNoteContent();
677                         }
678                 }
679                 
680                 List<String> unindexedResources = conn.getNoteTable().noteResourceTable.getUnindexed();
681                 if (unindexedResources.size() > 0 && !started) {
682                         signal.indexStarted.emit();
683                         started = true;
684                 }
685                 for (int i=0; i<unindexedResources.size()&& keepRunning; i++) {
686                         if (interrupt) {
687                                 processInterrupt();
688                         }
689                         guid = unindexedResources.get(i);
690                         if (keepRunning) {
691                                 indexResource();
692                         }
693                 }
694                 
695                 // Cleanup stuff that was deleted at some point
696                 List<String> guids = conn.getWordsTable().getGuidList();
697                 logger.log(logger.LOW, "GUIDS in index: " +guids.size());
698                 for (int i=0; i<guids.size() && keepRunning; i++) {
699                         if (!conn.getNoteTable().exists(guids.get(i))) {
700                                 logger.log(logger.LOW, "Old GUID found: " +guids.get(i));
701                                 conn.getWordsTable().expunge(guids.get(i));
702                         }
703                 }
704                 
705                 if (started && keepRunning) 
706                         signal.indexFinished.emit();
707         }
708         
709         private void reindexNote() {
710                 if (guid == null)
711                         return;
712                 conn.getNoteTable().setIndexNeeded(guid, true);
713         }
714         
715         private void reindexAll() {
716                 conn.getNoteTable().reindexAllNotes();
717                 conn.getNoteTable().noteResourceTable.reindexAll(); 
718         }
719
720         private void waitSeconds(int len) {
721                 long starttime = 0; // variable declared
722                 //...
723                 // for the first time, remember the timestamp
724             starttime = System.currentTimeMillis();
725                 // the next timestamp we want to wake up
726                 starttime += (1000.0);
727                 // Wait until the desired next time arrives using nanosecond
728                 // accuracy timer (wait(time) isn't accurate enough on most platforms) 
729                 LockSupport.parkNanos((Math.max(0, 
730                     starttime - System.currentTimeMillis()) * 1000000));
731         }
732         
733         private void processInterrupt() {
734                 conn.commitTransaction();
735                 waitSeconds(1);
736                 uncommittedCount = 0;
737                 conn.beginTransaction();
738                 interrupt = false;
739         }
740         
741 }