src/cx/fbn/nevernote/threads/IndexRunner.java

   1 /*
   2  * This file is part of NixNote/NeighborNote
   3  * Copyright 2009 Randy Baumgarte
   4  * Copyright 2013 Yuki Takahashi
   5  *
   6  * This file may be licensed under the terms of of the
   7  * GNU General Public License Version 2 (the ``GPL'').
   8  *
   9  * Software distributed under the License is distributed
  10  * on an ``AS IS'' basis, WITHOUT WARRANTY OF ANY KIND, either
  11  * express or implied. See the GPL for the specific language
  12  * governing rights and limitations.
  13  *
  14  * You should have received a copy of the GPL along with this
  15  * program. If not, go to http://www.gnu.org/licenses/gpl.html
  16  * or write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  *
  19 */
  20
  21 package cx.fbn.nevernote.threads;
  22
  23 import java.io.File;
  24 import java.io.FileInputStream;
  25 import java.io.FileNotFoundException;
  26 import java.io.IOException;
  27 import java.io.InputStream;
  28 import java.util.List;
  29 import java.util.TreeSet;
  30 import java.util.concurrent.LinkedBlockingQueue;
  31 import java.util.concurrent.locks.LockSupport;
  32
  33 import org.apache.commons.lang3.StringEscapeUtils;
  34 import org.apache.tika.exception.TikaException;
  35 import org.apache.tika.metadata.Metadata;
  36 import org.apache.tika.parser.ParseContext;
  37 import org.apache.tika.parser.microsoft.OfficeParser;
  38 import org.apache.tika.parser.microsoft.ooxml.OOXMLParser;
  39 import org.apache.tika.parser.odf.OpenDocumentParser;
  40 import org.apache.tika.parser.pdf.PDFParser;
  41 import org.apache.tika.parser.rtf.RTFParser;
  42 import org.apache.tika.sax.BodyContentHandler;
  43 import org.xml.sax.ContentHandler;
  44 import org.xml.sax.SAXException;
  45
  46 import com.evernote.edam.type.Data;
  47 import com.evernote.edam.type.Note;
  48 import com.evernote.edam.type.Resource;
  49 import com.trolltech.qt.core.QByteArray;
  50 import com.trolltech.qt.core.QIODevice.OpenModeFlag;
  51 import com.trolltech.qt.core.QObject;
  52 import com.trolltech.qt.core.QTemporaryFile;
  53 import com.trolltech.qt.xml.QDomDocument;
  54 import com.trolltech.qt.xml.QDomElement;
  55 import com.trolltech.qt.xml.QDomNodeList;
  56
  57 import cx.fbn.nevernote.Global;
  58 import cx.fbn.nevernote.signals.IndexSignal;
  59 import cx.fbn.nevernote.signals.NoteResourceSignal;
  60 import cx.fbn.nevernote.signals.NoteSignal;
  61 import cx.fbn.nevernote.sql.DatabaseConnection;
  62 import cx.fbn.nevernote.utilities.ApplicationLogger;
  63
  64 public class IndexRunner extends QObject implements Runnable {
  65
  66         private final ApplicationLogger         logger;
  67         private String                                          guid;
  68         private QByteArray                                      resourceBinary;
  69         public volatile NoteSignal                      noteSignal;
  70         public volatile NoteResourceSignal      resourceSignal;
  71         private int                                                     indexType;
  72         public final int                                        SCAN=1;
  73         public final int                                        REINDEXALL=2;
  74         public final int                                        REINDEXNOTE=3;
  75         public boolean                                          keepRunning;
  76         private final QDomDocument                      doc;
  77         private static String                           regex = Global.getWordRegex();
  78         public String                                           specialIndexCharacters = "";
  79         public boolean                                          indexNoteBody = true;
  80         public boolean                                          indexNoteTitle = true;
  81         public boolean                                          indexImageRecognition = true;
  82         private final DatabaseConnection        conn;
  83         private volatile LinkedBlockingQueue<String> workQueue;
  84         private static int MAX_QUEUED_WAITING = 1000;
  85         public boolean interrupt;
  86         public boolean idle;
  87         public boolean indexAttachmentsLocally = true;
  88         public volatile IndexSignal                     signal;
  89         private final TreeSet<String>           foundWords;
  90         int uncommittedCount = 0;
  91
  92         // ICHANGED String bを追加
  93         public IndexRunner(String logname, String u, String i, String r, String b, String uid, String pswd, String cpswd) {
  94                 foundWords = new TreeSet<String>();
  95                 logger = new ApplicationLogger(logname);
  96                 // ICHANGED bを追加
  97                 conn = new DatabaseConnection(logger, u, i, r, b, uid, pswd, cpswd, 500);
  98                 indexType = SCAN;
  99                 guid = null;
 100                 keepRunning = true;
 101                 doc = new QDomDocument();
 102                 workQueue=new LinkedBlockingQueue<String>(MAX_QUEUED_WAITING);
 103         }
 104
 105         public void setIndexType(int t) {
 106                 indexType = t;
 107         }
 108
 109
 110         @Override
 111         public void run() {
 112                 thread().setPriority(Thread.MIN_PRIORITY);
 113                 noteSignal = new NoteSignal();
 114                 resourceSignal = new NoteResourceSignal();
 115                 signal = new IndexSignal();
 116                 logger.log(logger.EXTREME, "Starting index thread ");
 117                 while (keepRunning) {
 118                         idle=true;
 119                         try {
 120                                 conn.commitTransaction();
 121                                 uncommittedCount = 0;
 122                                 String work = workQueue.take();
 123                                 idle=false;
 124                                 if (work.startsWith("SCAN")) {
 125                                         guid=null;
 126                                         interrupt = false;
 127                                         indexType = SCAN;
 128                                 }
 129                                 if (work.startsWith("REINDEXALL")) {
 130                                         guid = null;
 131                                         indexType=REINDEXALL;
 132                                 }
 133                                 if (work.startsWith("REINDEXNOTE")) {
 134                                         work = work.replace("REINDEXNOTE ", "");
 135                                         guid = work;
 136                                         indexType = REINDEXNOTE;
 137                                 }
 138                                 if (work.startsWith("STOP")) {
 139                                         keepRunning = false;
 140                                         guid = null;
 141                                 }
 142                                 logger.log(logger.EXTREME, "Type:" +indexType);
 143                                 if (indexType == SCAN && keepRunning) {
 144                                         logger.log(logger.MEDIUM, "Scanning for unindexed notes & resources");
 145                                         scanUnindexed();
 146                                         setIndexType(0);
 147                                 }
 148                                 if (indexType == REINDEXALL && keepRunning) {
 149                                         logger.log(logger.MEDIUM, "Marking all for reindex");
 150                                         reindexAll();
 151                                         setIndexType(0);
 152                                 }
 153                                 if (indexType == REINDEXNOTE && keepRunning) {
 154                                         reindexNote();
 155                                 }
 156                         } catch (InterruptedException e) {
 157                                 logger.log(logger.LOW, "Thread interrupted exception: " +e.getMessage());
 158                         }
 159                 }
 160                 logger.log(logger.EXTREME, "Shutting down database");
 161                 conn.dbShutdown();
 162                 logger.log(logger.EXTREME, "Database shut down.  Exiting thread");
 163         }
 164
 165         // Reindex a note
 166         public void indexNoteContent() {
 167                 foundWords.clear();
 168
 169                 logger.log(logger.EXTREME, "Entering indexRunner.indexNoteContent()");
 170
 171                 logger.log(logger.EXTREME, "Getting note content");
 172                 Note n = conn.getNoteTable().getNote(guid,true,false,true,true, true);
 173                 String data;
 174                 if (indexNoteBody) {
 175                         data = n.getContent();
 176                         data = conn.getNoteTable().getNoteContentNoUTFConversion(n.getGuid());
 177
 178                         logger.log(logger.EXTREME, "Removing any encrypted data");
 179                         data = removeEnCrypt(data.toString());
 180                         logger.log(logger.EXTREME, "Removing xml markups");
 181                 } else
 182                         data = "";
 183                 String text;
 184                 if (indexNoteTitle)
 185                         text =  removeTags(StringEscapeUtils.unescapeHtml4(data) +" "+ n.getTitle());
 186                 else
 187                         text = removeTags(StringEscapeUtils.unescapeHtml4(data));
 188
 189                 logger.log(logger.EXTREME, "Splitting words");
 190                 String[] result = text.toString().split(regex);
 191                 conn.commitTransaction();
 192                 conn.beginTransaction();
 193                 logger.log(logger.EXTREME, "Deleting existing words for note from index");
 194                 conn.getWordsTable().expungeFromWordIndex(guid, "CONTENT");
 195
 196                 logger.log(logger.EXTREME, "Number of words found: " +result.length);
 197                 for (int j=0; j<result.length && keepRunning; j++) {
 198                         if (interrupt) {
 199                                 processInterrupt();
 200                         }
 201                         if (!result[j].trim().equals("")) {
 202                                 logger.log(logger.EXTREME, "Result word: " +result[j].trim());
 203                                 addToIndex(guid, result[j], "CONTENT");
 204                         }
 205                 }
 206
 207                 // Add tags
 208                 for (int j=0; j<n.getTagNamesSize(); j++) {
 209                         if (n.getTagNames() != null && n.getTagNames().get(j) != null && !n.getTagNames().get(j).trim().equals(""))
 210                                 addToIndex(guid, n.getTagNames().get(j), "CONTENT");
 211                 }
 212
 213                 // If we were interrupted, we will reindex this note next time
 214                 if (Global.keepRunning) {
 215                         logger.log(logger.EXTREME, "Resetting note guid needed");
 216                         conn.getNoteTable().setIndexNeeded(guid, false);
 217                 }
 218                 conn.commitTransaction();
 219                 uncommittedCount = 0;
 220                 logger.log(logger.EXTREME, "Leaving indexRunner.indexNoteContent()");
 221         }
 222
 223
 224         private String removeTags(String text) {
 225                 StringBuffer buffer = new StringBuffer(text);
 226                 boolean inTag = false;
 227                 for (int i=buffer.length()-1; i>=0; i--) {
 228                         if (buffer.charAt(i) == '>')
 229                                 inTag = true;
 230                         if (buffer.charAt(i) == '<')
 231                                 inTag = false;
 232                         if (inTag || buffer.charAt(i) == '<')
 233                                 buffer.deleteCharAt(i);
 234                 }
 235
 236                 return buffer.toString();
 237         }
 238
 239
 240         public synchronized boolean addWork(String request) {
 241                 if (workQueue.size() == 0) {
 242                         workQueue.offer(request);
 243                         return true;
 244                 }
 245                 return false;
 246         }
 247
 248         public synchronized int getWorkQueueSize() {
 249                 return workQueue.size();
 250         }
 251
 252         public void indexResource() {
 253
 254                 if (guid == null)
 255                         return;
 256                 foundWords.clear();
 257                 Resource r = conn.getNoteTable().noteResourceTable.getNoteResourceRecognition(guid);
 258                 if (!indexImageRecognition ||
 259                                 r == null || r.getRecognition() == null ||
 260                                 r.getRecognition().getBody() == null ||
 261                                 r.getRecognition().getBody().length == 0)
 262                         resourceBinary = new QByteArray(" ");
 263                 else
 264                         resourceBinary = new QByteArray(r.getRecognition().getBody());
 265
 266                 conn.commitTransaction();
 267                 conn.beginTransaction();
 268                 conn.getWordsTable().expungeFromWordIndex(r.getNoteGuid(), "RESOURCE");
 269                 // This is due to an old bug & can be removed at some point in the future 11/23/2010
 270                 conn.getWordsTable().expungeFromWordIndex(guid, "RESOURCE");
 271                 conn.commitTransaction();
 272                 uncommittedCount = 0;
 273                 conn.beginTransaction();
 274
 275                 doc.setContent(resourceBinary);
 276                 QDomElement docElem = doc.documentElement();
 277
 278                 // look for text tags
 279                 QDomNodeList anchors = docElem.elementsByTagName("t");
 280                 for (int i=0; i<anchors.length() && keepRunning; i++) {
 281                         if (interrupt) {
 282                                 if (interrupt) {
 283                                         processInterrupt();
 284                                 }
 285                         }
 286                         QDomElement enmedia = anchors.at(i).toElement();
 287                         String weight = new String(enmedia.attribute("w"));
 288                         String text = new String(enmedia.text()).toLowerCase();
 289                         if (!text.equals("")) {
 290                                 conn.getWordsTable().addWordToNoteIndex(r.getNoteGuid(), text, "RESOURCE", new Integer(weight));
 291                                 uncommittedCount++;
 292                                 if (uncommittedCount > 100) {
 293                                         conn.commitTransaction();
 294                                         uncommittedCount=0;
 295                                 }
 296                         }
 297                 }
 298
 299                 if (Global.keepRunning && indexAttachmentsLocally) {
 300                         conn.commitTransaction();
 301                         uncommittedCount = 0;
 302                         conn.beginTransaction();
 303                         indexResourceContent(guid);
 304                 }
 305
 306                 if (Global.keepRunning)
 307                         conn.getNoteTable().noteResourceTable.setIndexNeeded(guid,false);
 308                 conn.commitTransaction();
 309                 uncommittedCount = 0;
 310         }
 311
 312         private void indexResourceContent(String guid) {
 313                 Resource r = conn.getNoteTable().noteResourceTable.getNoteResource(guid, true);
 314                 if (r != null && r.getMime() != null) {
 315                         if (r.getMime().equalsIgnoreCase("application/pdf")) {
 316                                 indexResourcePDF(r);
 317                                 return;
 318                         }
 319                         if (r.getMime().equalsIgnoreCase("application/docx") ||
 320                                 r.getMime().equalsIgnoreCase("application/xlsx") ||
 321                                 r.getMime().equalsIgnoreCase("application/pptx")) {
 322                                 indexResourceOOXML(r);
 323                                 return;
 324                         }
 325                         if (r.getMime().equalsIgnoreCase("application/vsd") ||
 326                                         r.getMime().equalsIgnoreCase("application/ppt") ||
 327                                         r.getMime().equalsIgnoreCase("application/xls") ||
 328                                         r.getMime().equalsIgnoreCase("application/msg") ||
 329                                         r.getMime().equalsIgnoreCase("application/doc")) {
 330                                 indexResourceOffice(r);
 331                                 return;
 332                         }
 333                         if (r.getMime().equalsIgnoreCase("application/rtf")) {
 334                                         indexResourceRTF(r);
 335                                         return;
 336                         }
 337                         if (r.getMime().equalsIgnoreCase("application/odf") ||
 338                                 r.getMime().equalsIgnoreCase("application/odt") ||
 339                                 r.getMime().equalsIgnoreCase("application/odp") ||
 340                                 r.getMime().equalsIgnoreCase("application/odg") ||
 341                                 r.getMime().equalsIgnoreCase("application/odb") ||
 342                                 r.getMime().equalsIgnoreCase("application/ods")) {
 343                                 indexResourceODF(r);
 344                                 return;
 345                         }
 346                 }
 347         }
 348
 349
 350         private void indexResourceRTF(Resource r) {
 351
 352                 Data d = r.getData();
 353                 for (int i=0; i<20 && d.getSize() == 0; i++)
 354                         d = r.getData();
 355                 if (d.getSize()== 0)
 356                         return;
 357
 358                 QTemporaryFile f = writeResource(d);
 359                 if (!keepRunning) {
 360                         return;
 361                 }
 362
 363                 InputStream input;
 364                 try {
 365                         input = new FileInputStream(new File(f.fileName()));
 366                         ContentHandler textHandler = new BodyContentHandler(-1);
 367                         Metadata metadata = new Metadata();
 368                         RTFParser parser = new RTFParser();
 369                         ParseContext context = new ParseContext();
 370                         parser.parse(input, textHandler, metadata, context);
 371                         String[] result = textHandler.toString().split(regex);
 372                         for (int i=0; i<result.length && keepRunning; i++) {
 373                                 addToIndex(r.getNoteGuid(), result[i], "RESOURCE");
 374                         }
 375                         input.close();
 376
 377                         f.close();
 378                 } catch (java.lang.ClassCastException e) {
 379                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());
 380                 } catch (FileNotFoundException e) {
 381                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());
 382                 } catch (IOException e) {
 383                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());
 384                 } catch (SAXException e) {
 385                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());
 386                 } catch (TikaException e) {
 387                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());
 388                 } catch (Exception e) {
 389                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());
 390                 } catch (java.lang.NoSuchMethodError e) {
 391                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());
 392                 } catch (Error e) {
 393                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());
 394                 }
 395         }
 396
 397
 398         private void indexResourceODF(Resource r) {
 399
 400                 Data d = r.getData();
 401                 for (int i=0; i<20 && d.getSize() == 0; i++)
 402                         d = r.getData();
 403                 if (d.getSize()== 0)
 404                         return;
 405                 QTemporaryFile f = writeResource(d);
 406                 if (!keepRunning) {
 407                         return;
 408                 }
 409
 410                 InputStream input;
 411                 try {
 412                         input = new FileInputStream(new File(f.fileName()));
 413                         ContentHandler textHandler = new BodyContentHandler(-1);
 414                         Metadata metadata = new Metadata();
 415                         OpenDocumentParser parser = new OpenDocumentParser();
 416                         ParseContext context = new ParseContext();
 417                         parser.parse(input, textHandler, metadata, context);
 418                         String[] result = textHandler.toString().split(regex);
 419                         for (int i=0; i<result.length && keepRunning; i++) {
 420                                 if (interrupt) {
 421                                         processInterrupt();
 422                                 }
 423                                 addToIndex(r.getNoteGuid(), result[i], "RESOURCE");
 424                         }
 425                         input.close();
 426
 427                         f.close();
 428                 } catch (java.lang.ClassCastException e) {
 429                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());
 430                 } catch (FileNotFoundException e) {
 431                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());
 432                 } catch (IOException e) {
 433                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());
 434                 } catch (SAXException e) {
 435                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());
 436                 } catch (TikaException e) {
 437                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());
 438                 } catch (Exception e) {
 439                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());
 440                 } catch (java.lang.NoSuchMethodError e) {
 441                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());
 442                 } catch (Error e) {
 443                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());
 444                 }
 445         }
 446
 447
 448         private void indexResourceOffice(Resource r) {
 449
 450                 Data d = r.getData();
 451                 for (int i=0; i<20 && d.getSize() == 0; i++)
 452                         d = r.getData();
 453                 if (d.getSize()== 0)
 454                         return;
 455                 QTemporaryFile f = writeResource(d);
 456                 if (!keepRunning) {
 457                         return;
 458                 }
 459
 460                 InputStream input;
 461                 try {
 462                         input = new FileInputStream(new File(f.fileName()));
 463                         ContentHandler textHandler = new BodyContentHandler(-1);
 464                         Metadata metadata = new Metadata();
 465                         OfficeParser parser = new OfficeParser();
 466                         ParseContext context = new ParseContext();
 467                         parser.parse(input, textHandler, metadata, context);
 468                         String[] result = textHandler.toString().split(regex);
 469                         for (int i=0; i<result.length && keepRunning; i++) {
 470                                 if (interrupt) {
 471                                         processInterrupt();
 472                                 }
 473                                 addToIndex(r.getNoteGuid(), result[i], "RESOURCE");
 474                         }
 475                         input.close();
 476
 477                         f.close();
 478                 } catch (java.lang.ClassCastException e) {
 479                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());
 480                 } catch (FileNotFoundException e) {
 481                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());
 482                 } catch (IOException e) {
 483                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());
 484                 } catch (SAXException e) {
 485                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());
 486                 } catch (TikaException e) {
 487                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());
 488                 } catch (Exception e) {
 489                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());
 490                 } catch (java.lang.NoSuchMethodError e) {
 491                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());
 492                 } catch (Error e) {
 493                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());
 494                 }
 495         }
 496
 497
 498
 499         private void indexResourcePDF(Resource r) {
 500
 501                 Data d = r.getData();
 502                 for (int i=0; i<20 && d.getSize() == 0; i++)
 503                         d = r.getData();
 504                 if (d.getSize()== 0)
 505                         return;
 506                 QTemporaryFile f = writeResource(d);
 507                 if (!keepRunning) {
 508                         return;
 509                 }
 510
 511                 InputStream input;
 512                 try {
 513                         input = new FileInputStream(new File(f.fileName()));
 514                         ContentHandler textHandler = new BodyContentHandler(-1);
 515                         Metadata metadata = new Metadata();
 516                         PDFParser parser = new PDFParser();
 517                         ParseContext context = new ParseContext();
 518                         parser.parse(input, textHandler, metadata, context);
 519                         String[] result = textHandler.toString().split(regex);
 520                         for (int i=0; i<result.length && keepRunning; i++) {
 521                                 if (interrupt) {
 522                                         processInterrupt();
 523                                 }
 524                                 addToIndex(r.getNoteGuid(), result[i], "RESOURCE");
 525                         }
 526                         input.close();
 527
 528                         f.close();
 529                 } catch (java.lang.ClassCastException e) {
 530                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());
 531                 } catch (FileNotFoundException e) {
 532                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());
 533                 } catch (IOException e) {
 534                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());
 535                 } catch (SAXException e) {
 536                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());
 537                 } catch (TikaException e) {
 538                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());
 539                 } catch (Exception e) {
 540                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());
 541                 } catch (java.lang.NoSuchMethodError e) {
 542                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());
 543                 } catch (Error e) {
 544                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());
 545                 }
 546         }
 547
 548
 549         private void indexResourceOOXML(Resource r) {
 550
 551                 Data d = r.getData();
 552                 for (int i=0; i<20 && d.getSize() == 0; i++)
 553                         d = r.getData();
 554                 if (d.getSize()== 0)
 555                         return;
 556                 QTemporaryFile f = writeResource(d);
 557                 if (!keepRunning) {
 558                         return;
 559                 }
 560
 561                 InputStream input;
 562                 try {
 563                         input = new FileInputStream(new File(f.fileName()));
 564                         ContentHandler textHandler = new BodyContentHandler(-1);
 565                         Metadata metadata = new Metadata();
 566                         OOXMLParser parser = new OOXMLParser();
 567                         ParseContext context = new ParseContext();
 568                         parser.parse(input, textHandler, metadata, context);
 569                         String[] result = textHandler.toString().split(regex);
 570                         for (int i=0; i<result.length && keepRunning; i++) {
 571                                 if (interrupt) {
 572                                         processInterrupt();
 573                                 }
 574                                 addToIndex(r.getNoteGuid(), result[i], "RESOURCE");
 575                         }
 576                         input.close();
 577
 578                         f.close();
 579                 } catch (java.lang.ClassCastException e) {
 580                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());
 581                 } catch (FileNotFoundException e) {
 582                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());
 583                 } catch (IOException e) {
 584                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());
 585                 } catch (SAXException e) {
 586                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());
 587                 } catch (TikaException e) {
 588                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());
 589                 } catch (Exception e) {
 590                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());
 591                 } catch (java.lang.NoSuchMethodError e) {
 592                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());
 593                 } catch (Error e) {
 594                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());              }
 595         }
 596
 597
 598
 599         private QTemporaryFile writeResource(Data d) {
 600                 QTemporaryFile newFile = new QTemporaryFile();
 601                 newFile.open(OpenModeFlag.WriteOnly);
 602                 newFile.write(d.getBody());
 603                 newFile.close();
 604                 return newFile;
 605         }
 606
 607
 608         private String removeEnCrypt(String content) {
 609                 int index = content.indexOf("<en-crypt");
 610                 int endPos;
 611                 boolean tagFound = true;
 612                 while (tagFound && keepRunning) {
 613                         if (interrupt) {
 614                                 processInterrupt();
 615                         }
 616                         endPos = content.indexOf("</en-crypt>", index)+11;
 617                         if (endPos > -1 && index > -1) {
 618                                 content = content.substring(0,index)+content.substring(endPos);
 619                                 index = content.indexOf("<en-crypt");
 620                         } else {
 621                                 tagFound = false;
 622                         }
 623                 }
 624                 return content;
 625         }
 626
 627
 628         private void addToIndex(String guid, String word, String type) {
 629                 if (foundWords.contains(word))
 630                         return;
 631                 StringBuffer buffer = new StringBuffer(word.toLowerCase());
 632                 for (int i=buffer.length()-1; i>=0; i--) {
 633                         if (!Character.isLetterOrDigit(buffer.charAt(i)) && specialIndexCharacters.indexOf(buffer.charAt(i)) == -1)
 634                                 buffer.deleteCharAt(i);
 635                         else
 636                                 break;
 637                 }
 638                 buffer = buffer.reverse();
 639                 for (int i=buffer.length()-1; i>=0; i--) {
 640                         if (!Character.isLetterOrDigit(buffer.charAt(i)))
 641                                 buffer.deleteCharAt(i);
 642                         else
 643                                 break;
 644                 }
 645                 buffer = buffer.reverse();
 646                 if (buffer.length() > 0) {
 647                         // We have a good word, now let's trim off junk at the beginning or end
 648                         if (!foundWords.contains(buffer.toString())) {
 649                                 foundWords.add(buffer.toString());
 650                                 foundWords.add(word);
 651                                 conn.getWordsTable().addWordToNoteIndex(guid, buffer.toString(), type, 100);
 652                                 uncommittedCount++;
 653                                 if (uncommittedCount > 100) {
 654                                         conn.commitTransaction();
 655                                         uncommittedCount=0;
 656                                 }
 657                         }
 658                 }
 659                 return;
 660         }
 661
 662         private void scanUnindexed() {
 663                 List<String> notes = conn.getNoteTable().getUnindexed();
 664                 guid = null;
 665                 boolean started = false;
 666                 if (notes.size() > 0) {
 667                         signal.indexStarted.emit();
 668                         started = true;
 669                 }
 670                 for (int i=0; i<notes.size() && keepRunning; i++) {
 671                         if (interrupt) {
 672                                 processInterrupt();
 673                         }
 674                         guid = notes.get(i);
 675                         if (guid != null && keepRunning) {
 676                                 indexNoteContent();
 677                         }
 678                 }
 679
 680                 List<String> unindexedResources = conn.getNoteTable().noteResourceTable.getUnindexed();
 681                 if (unindexedResources.size() > 0 && !started) {
 682                         signal.indexStarted.emit();
 683                         started = true;
 684                 }
 685                 for (int i=0; i<unindexedResources.size()&& keepRunning; i++) {
 686                         if (interrupt) {
 687                                 processInterrupt();
 688                         }
 689                         guid = unindexedResources.get(i);
 690                         if (keepRunning) {
 691                                 indexResource();
 692                         }
 693                 }
 694
 695                 // Cleanup stuff that was deleted at some point
 696                 List<String> guids = conn.getWordsTable().getGuidList();
 697                 logger.log(logger.LOW, "GUIDS in index: " +guids.size());
 698                 for (int i=0; i<guids.size() && keepRunning; i++) {
 699                         if (!conn.getNoteTable().exists(guids.get(i))) {
 700                                 logger.log(logger.LOW, "Old GUID found: " +guids.get(i));
 701                                 conn.getWordsTable().expunge(guids.get(i));
 702                         }
 703                 }
 704
 705                 if (started && keepRunning)
 706                         signal.indexFinished.emit();
 707         }
 708
 709         private void reindexNote() {
 710                 if (guid == null)
 711                         return;
 712                 conn.getNoteTable().setIndexNeeded(guid, true);
 713         }
 714
 715         private void reindexAll() {
 716                 conn.getNoteTable().reindexAllNotes();
 717                 conn.getNoteTable().noteResourceTable.reindexAll();
 718         }
 719
 720         private void waitSeconds(int len) {
 721                 long starttime = 0; // variable declared
 722                 //...
 723                 // for the first time, remember the timestamp
 724             starttime = System.currentTimeMillis();
 725                 // the next timestamp we want to wake up
 726                 starttime += (1000.0);
 727                 // Wait until the desired next time arrives using nanosecond
 728                 // accuracy timer (wait(time) isn't accurate enough on most platforms)
 729                 LockSupport.parkNanos((Math.max(0,
 730                     starttime - System.currentTimeMillis()) * 1000000));
 731         }
 732
 733         private void processInterrupt() {
 734                 conn.commitTransaction();
 735                 waitSeconds(1);
 736                 uncommittedCount = 0;
 737                 conn.beginTransaction();
 738                 interrupt = false;
 739         }
 740
 741 }