src/cx/fbn/nevernote/threads/IndexRunner.java

   1 /*
   2  * This file is part of NixNote/NeighborNote
   3  * Copyright 2009 Randy Baumgarte
   4  * Copyright 2013 Yuki Takahashi
   5  *
   6  * This file may be licensed under the terms of of the
   7  * GNU General Public License Version 2 (the ``GPL'').
   8  *
   9  * Software distributed under the License is distributed
  10  * on an ``AS IS'' basis, WITHOUT WARRANTY OF ANY KIND, either
  11  * express or implied. See the GPL for the specific language
  12  * governing rights and limitations.
  13  *
  14  * You should have received a copy of the GPL along with this
  15  * program. If not, go to http://www.gnu.org/licenses/gpl.html
  16  * or write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  *
  19 */
  20
  21 package cx.fbn.nevernote.threads;
  22
  23 import java.io.File;
  24 import java.io.FileInputStream;
  25 import java.io.FileNotFoundException;
  26 import java.io.IOException;
  27 import java.io.InputStream;
  28 import java.util.List;
  29 import java.util.TreeSet;
  30 import java.util.concurrent.LinkedBlockingQueue;
  31 import java.util.concurrent.locks.LockSupport;
  32
  33 import org.apache.tika.exception.TikaException;
  34 import org.apache.tika.metadata.Metadata;
  35 import org.apache.tika.parser.ParseContext;
  36 import org.apache.tika.parser.microsoft.OfficeParser;
  37 import org.apache.tika.parser.microsoft.ooxml.OOXMLParser;
  38 import org.apache.tika.parser.odf.OpenDocumentParser;
  39 import org.apache.tika.parser.pdf.PDFParser;
  40 import org.apache.tika.parser.rtf.RTFParser;
  41 import org.apache.tika.sax.BodyContentHandler;
  42 import org.xml.sax.ContentHandler;
  43 import org.xml.sax.SAXException;
  44
  45 import com.evernote.edam.type.Data;
  46 import com.evernote.edam.type.Resource;
  47 import com.trolltech.qt.core.QByteArray;
  48 import com.trolltech.qt.core.QIODevice.OpenModeFlag;
  49 import com.trolltech.qt.core.QObject;
  50 import com.trolltech.qt.core.QTemporaryFile;
  51 import com.trolltech.qt.xml.QDomDocument;
  52 import com.trolltech.qt.xml.QDomElement;
  53 import com.trolltech.qt.xml.QDomNodeList;
  54
  55 import cx.fbn.nevernote.Global;
  56 import cx.fbn.nevernote.signals.IndexSignal;
  57 import cx.fbn.nevernote.signals.NoteResourceSignal;
  58 import cx.fbn.nevernote.signals.NoteSignal;
  59 import cx.fbn.nevernote.sql.DatabaseConnection;
  60 import cx.fbn.nevernote.utilities.ApplicationLogger;
  61
  62 public class IndexRunner extends QObject implements Runnable {
  63
  64         private final ApplicationLogger         logger;
  65         private String                                          guid;
  66         private QByteArray                                      resourceBinary;
  67         public volatile NoteSignal                      noteSignal;
  68         public volatile NoteResourceSignal      resourceSignal;
  69         private int                                                     indexType;
  70         public final int                                        SCAN=1;
  71         public final int                                        REINDEXALL=2;
  72         public final int                                        REINDEXNOTE=3;
  73         public boolean                                          keepRunning;
  74         private final QDomDocument                      doc;
  75 //      private static String                           regex = Global.getWordRegex();
  76 //      public String                                           specialIndexCharacters = "";
  77 //      public boolean                                          indexNoteBody = true;
  78 //      public boolean                                          indexNoteTitle = true;
  79         public boolean                                          indexImageRecognition = true;
  80         private final DatabaseConnection        conn;
  81         private volatile LinkedBlockingQueue<String> workQueue;
  82         private static int MAX_QUEUED_WAITING = 1000;
  83         public boolean interrupt;
  84         public boolean idle;
  85         public boolean indexAttachmentsLocally = true;
  86         public volatile IndexSignal                     signal;
  87         private final TreeSet<String>           foundWords;
  88         int uncommittedCount = 0;
  89
  90         // ICHANGED String bを追加
  91         public IndexRunner(String logname, String u, String i, String r, String b, String uid, String pswd, String cpswd) {
  92                 foundWords = new TreeSet<String>();
  93                 logger = new ApplicationLogger(logname);
  94                 // ICHANGED bを追加
  95                 conn = new DatabaseConnection(logger, u, i, r, b, uid, pswd, cpswd, 500);
  96                 indexType = SCAN;
  97                 guid = null;
  98                 keepRunning = true;
  99                 doc = new QDomDocument();
 100                 workQueue=new LinkedBlockingQueue<String>(MAX_QUEUED_WAITING);
 101         }
 102
 103         public void setIndexType(int t) {
 104                 indexType = t;
 105         }
 106
 107
 108         @Override
 109         public void run() {
 110                 thread().setPriority(Thread.MIN_PRIORITY);
 111                 noteSignal = new NoteSignal();
 112                 resourceSignal = new NoteResourceSignal();
 113                 signal = new IndexSignal();
 114                 logger.log(logger.EXTREME, "Starting index thread ");
 115                 while (keepRunning) {
 116                         idle=true;
 117                         try {
 118                                 conn.commitTransaction();
 119                                 uncommittedCount = 0;
 120                                 String work = workQueue.take();
 121                                 idle=false;
 122                                 if (work.startsWith("SCAN")) {
 123                                         guid=null;
 124                                         interrupt = false;
 125                                         indexType = SCAN;
 126                                 }
 127                                 if (work.startsWith("REINDEXALL")) {
 128                                         guid = null;
 129                                         indexType=REINDEXALL;
 130                                 }
 131                                 if (work.startsWith("REINDEXNOTE")) {
 132                                         work = work.replace("REINDEXNOTE ", "");
 133                                         guid = work;
 134                                         indexType = REINDEXNOTE;
 135                                 }
 136                                 if (work.startsWith("STOP")) {
 137                                         keepRunning = false;
 138                                         guid = null;
 139                                 }
 140                                 logger.log(logger.EXTREME, "Type:" +indexType);
 141                                 if (indexType == SCAN && keepRunning) {
 142                                         logger.log(logger.MEDIUM, "Scanning for unindexed notes & resources");
 143                                         scanUnindexed();
 144                                         setIndexType(0);
 145                                 }
 146                                 if (indexType == REINDEXALL && keepRunning) {
 147                                         logger.log(logger.MEDIUM, "Marking all for reindex");
 148                                         reindexAll();
 149                                         setIndexType(0);
 150                                 }
 151                                 if (indexType == REINDEXNOTE && keepRunning) {
 152                                         reindexNote();
 153                                 }
 154                         } catch (InterruptedException e) {
 155                                 logger.log(logger.LOW, "Thread interrupted exception: " +e.getMessage());
 156                         }
 157                 }
 158                 logger.log(logger.EXTREME, "Shutting down database");
 159                 conn.dbShutdown();
 160                 logger.log(logger.EXTREME, "Database shut down.  Exiting thread");
 161         }
 162
 163         // Reindex a note
 164 //      public void indexNoteContent() {
 165 //              foundWords.clear();
 166 //
 167 //              logger.log(logger.EXTREME, "Entering indexRunner.indexNoteContent()");
 168 //
 169 //              logger.log(logger.EXTREME, "Getting note content");
 170 //              Note n = conn.getNoteTable().getNote(guid,true,false,true,true, true);
 171 //              String data;
 172 //              if (indexNoteBody) {
 173 //                      data = n.getContent();
 174 //                      data = conn.getNoteTable().getNoteContentNoUTFConversion(n.getGuid());
 175 //
 176 //                      logger.log(logger.EXTREME, "Removing any encrypted data");
 177 //                      data = removeEnCrypt(data.toString());
 178 //                      logger.log(logger.EXTREME, "Removing xml markups");
 179 //              } else
 180 //                      data = "";
 181 //              String text;
 182 //              if (indexNoteTitle)
 183 //                      text =  removeTags(StringEscapeUtils.unescapeHtml4(data) +" "+ n.getTitle());
 184 //              else
 185 //                      text = removeTags(StringEscapeUtils.unescapeHtml4(data));
 186 //
 187 //              logger.log(logger.EXTREME, "Splitting words");
 188 //              String[] result = text.toString().split(regex);
 189 //              conn.commitTransaction();
 190 //              conn.beginTransaction();
 191 //              logger.log(logger.EXTREME, "Deleting existing words for note from index");
 192 //              conn.getWordsTable().expungeFromWordIndex(guid, "CONTENT");
 193 //
 194 //              logger.log(logger.EXTREME, "Number of words found: " +result.length);
 195 //              for (int j=0; j<result.length && keepRunning; j++) {
 196 //                      if (interrupt) {
 197 //                              processInterrupt();
 198 //                      }
 199 //                      if (!result[j].trim().equals("")) {
 200 //                              logger.log(logger.EXTREME, "Result word: " +result[j].trim());
 201 //                              addToIndex(guid, result[j], "CONTENT");
 202 //                      }
 203 //              }
 204 //
 205 //              // Add tags
 206 //              for (int j=0; j<n.getTagNamesSize(); j++) {
 207 //                      if (n.getTagNames() != null && n.getTagNames().get(j) != null && !n.getTagNames().get(j).trim().equals(""))
 208 //                              addToIndex(guid, n.getTagNames().get(j), "CONTENT");
 209 //              }
 210 //
 211 //              // If we were interrupted, we will reindex this note next time
 212 //              if (Global.keepRunning) {
 213 //                      logger.log(logger.EXTREME, "Resetting note guid needed");
 214 //                      conn.getNoteTable().setIndexNeeded(guid, false);
 215 //              }
 216 //              conn.commitTransaction();
 217 //              uncommittedCount = 0;
 218 //              logger.log(logger.EXTREME, "Leaving indexRunner.indexNoteContent()");
 219 //      }
 220
 221
 222         private String removeTags(String text) {
 223                 StringBuffer buffer = new StringBuffer(text);
 224                 boolean inTag = false;
 225                 for (int i=buffer.length()-1; i>=0; i--) {
 226                         if (buffer.charAt(i) == '>')
 227                                 inTag = true;
 228                         if (buffer.charAt(i) == '<')
 229                                 inTag = false;
 230                         if (inTag || buffer.charAt(i) == '<')
 231                                 buffer.deleteCharAt(i);
 232                 }
 233
 234                 return buffer.toString();
 235         }
 236
 237
 238         public synchronized boolean addWork(String request) {
 239                 if (workQueue.size() == 0) {
 240                         workQueue.offer(request);
 241                         return true;
 242                 }
 243                 return false;
 244         }
 245
 246         public synchronized int getWorkQueueSize() {
 247                 return workQueue.size();
 248         }
 249
 250         public void indexResource() {
 251
 252                 if (guid == null)
 253                         return;
 254                 foundWords.clear();
 255                 Resource r = conn.getNoteTable().noteResourceTable.getNoteResourceRecognition(guid);
 256                 if (!indexImageRecognition ||
 257                                 r == null || r.getRecognition() == null ||
 258                                 r.getRecognition().getBody() == null ||
 259                                 r.getRecognition().getBody().length == 0)
 260                         resourceBinary = new QByteArray(" ");
 261                 else
 262                         resourceBinary = new QByteArray(r.getRecognition().getBody());
 263
 264                 conn.commitTransaction();
 265                 conn.beginTransaction();
 266                 conn.getWordsTable().expungeFromWordIndex(r.getNoteGuid(), "RESOURCE");
 267                 // This is due to an old bug & can be removed at some point in the future 11/23/2010
 268                 conn.getWordsTable().expungeFromWordIndex(guid, "RESOURCE");
 269                 conn.commitTransaction();
 270                 uncommittedCount = 0;
 271                 conn.beginTransaction();
 272
 273                 doc.setContent(resourceBinary);
 274                 QDomElement docElem = doc.documentElement();
 275
 276                 // look for text tags
 277                 QDomNodeList anchors = docElem.elementsByTagName("t");
 278                 for (int i=0; i<anchors.length() && keepRunning; i++) {
 279                         if (interrupt) {
 280                                 if (interrupt) {
 281                                         processInterrupt();
 282                                 }
 283                         }
 284                         QDomElement enmedia = anchors.at(i).toElement();
 285                         String weight = new String(enmedia.attribute("w"));
 286                         String text = new String(enmedia.text()).toLowerCase();
 287                         if (!text.equals("")) {
 288                                 conn.getWordsTable().addWordToNoteIndex(r.getNoteGuid(), text, "RESOURCE", new Integer(weight));
 289                                 uncommittedCount++;
 290                                 if (uncommittedCount > 100) {
 291                                         conn.commitTransaction();
 292                                         uncommittedCount=0;
 293                                 }
 294                         }
 295                 }
 296
 297                 if (Global.keepRunning && indexAttachmentsLocally) {
 298                         conn.commitTransaction();
 299                         uncommittedCount = 0;
 300                         conn.beginTransaction();
 301                         indexResourceContent(guid);
 302                 }
 303
 304                 if (Global.keepRunning)
 305                         conn.getNoteTable().noteResourceTable.setIndexNeeded(guid,false);
 306                 conn.commitTransaction();
 307                 uncommittedCount = 0;
 308         }
 309
 310         private void indexResourceContent(String guid) {
 311                 Resource r = conn.getNoteTable().noteResourceTable.getNoteResource(guid, true);
 312                 if (r != null && r.getMime() != null) {
 313                         if (r.getMime().equalsIgnoreCase("application/pdf")) {
 314                                 indexResourcePDF(r);
 315                                 return;
 316                         }
 317                         if (r.getMime().equalsIgnoreCase("application/docx") ||
 318                                 r.getMime().equalsIgnoreCase("application/xlsx") ||
 319                                 r.getMime().equalsIgnoreCase("application/pptx")) {
 320                                 indexResourceOOXML(r);
 321                                 return;
 322                         }
 323                         if (r.getMime().equalsIgnoreCase("application/vsd") ||
 324                                         r.getMime().equalsIgnoreCase("application/ppt") ||
 325                                         r.getMime().equalsIgnoreCase("application/xls") ||
 326                                         r.getMime().equalsIgnoreCase("application/msg") ||
 327                                         r.getMime().equalsIgnoreCase("application/doc")) {
 328                                 indexResourceOffice(r);
 329                                 return;
 330                         }
 331                         if (r.getMime().equalsIgnoreCase("application/rtf")) {
 332                                         indexResourceRTF(r);
 333                                         return;
 334                         }
 335                         if (r.getMime().equalsIgnoreCase("application/odf") ||
 336                                 r.getMime().equalsIgnoreCase("application/odt") ||
 337                                 r.getMime().equalsIgnoreCase("application/odp") ||
 338                                 r.getMime().equalsIgnoreCase("application/odg") ||
 339                                 r.getMime().equalsIgnoreCase("application/odb") ||
 340                                 r.getMime().equalsIgnoreCase("application/ods")) {
 341                                 indexResourceODF(r);
 342                                 return;
 343                         }
 344                 }
 345         }
 346
 347
 348         private void indexResourceRTF(Resource r) {
 349
 350                 Data d = r.getData();
 351                 for (int i=0; i<20 && d.getSize() == 0; i++)
 352                         d = r.getData();
 353                 if (d.getSize()== 0)
 354                         return;
 355
 356                 QTemporaryFile f = writeResource(d);
 357                 if (!keepRunning) {
 358                         return;
 359                 }
 360
 361                 InputStream input;
 362                 try {
 363                         input = new FileInputStream(new File(f.fileName()));
 364                         ContentHandler textHandler = new BodyContentHandler(-1);
 365                         Metadata metadata = new Metadata();
 366                         RTFParser parser = new RTFParser();
 367                         ParseContext context = new ParseContext();
 368                         parser.parse(input, textHandler, metadata, context);
 369 //                      String[] result = textHandler.toString().split(regex);
 370 //                      for (int i=0; i<result.length && keepRunning; i++) {
 371 //                              addToIndex(r.getNoteGuid(), result[i], "RESOURCE");
 372 //                      }
 373                         updateResourceText(r.getGuid(), textHandler.toString());
 374                         input.close();
 375
 376                         f.close();
 377                 } catch (java.lang.ClassCastException e) {
 378                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());
 379                 } catch (FileNotFoundException e) {
 380                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());
 381                 } catch (IOException e) {
 382                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());
 383                 } catch (SAXException e) {
 384                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());
 385                 } catch (TikaException e) {
 386                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());
 387                 } catch (Exception e) {
 388                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());
 389                 } catch (java.lang.NoSuchMethodError e) {
 390                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());
 391                 } catch (Error e) {
 392                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());
 393                 }
 394         }
 395
 396
 397         private void indexResourceODF(Resource r) {
 398
 399                 Data d = r.getData();
 400                 for (int i=0; i<20 && d.getSize() == 0; i++)
 401                         d = r.getData();
 402                 if (d.getSize()== 0)
 403                         return;
 404                 QTemporaryFile f = writeResource(d);
 405                 if (!keepRunning) {
 406                         return;
 407                 }
 408
 409                 InputStream input;
 410                 try {
 411                         input = new FileInputStream(new File(f.fileName()));
 412                         ContentHandler textHandler = new BodyContentHandler(-1);
 413                         Metadata metadata = new Metadata();
 414                         OpenDocumentParser parser = new OpenDocumentParser();
 415                         ParseContext context = new ParseContext();
 416                         parser.parse(input, textHandler, metadata, context);
 417 //                      String[] result = textHandler.toString().split(regex);
 418 //                      for (int i=0; i<result.length && keepRunning; i++) {
 419 //                              if (interrupt) {
 420 //                                      processInterrupt();
 421 //                              }
 422 //                              addToIndex(r.getNoteGuid(), result[i], "RESOURCE");
 423 //                      }
 424                         updateResourceText(r.getGuid(), textHandler.toString());
 425                         input.close();
 426
 427                         f.close();
 428                 } catch (java.lang.ClassCastException e) {
 429                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());
 430                 } catch (FileNotFoundException e) {
 431                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());
 432                 } catch (IOException e) {
 433                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());
 434                 } catch (SAXException e) {
 435                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());
 436                 } catch (TikaException e) {
 437                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());
 438                 } catch (Exception e) {
 439                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());
 440                 } catch (java.lang.NoSuchMethodError e) {
 441                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());
 442                 } catch (Error e) {
 443                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());
 444                 }
 445         }
 446
 447
 448         private void indexResourceOffice(Resource r) {
 449
 450                 Data d = r.getData();
 451                 for (int i=0; i<20 && d.getSize() == 0; i++)
 452                         d = r.getData();
 453                 if (d.getSize()== 0)
 454                         return;
 455                 QTemporaryFile f = writeResource(d);
 456                 if (!keepRunning) {
 457                         return;
 458                 }
 459
 460                 InputStream input;
 461                 try {
 462                         input = new FileInputStream(new File(f.fileName()));
 463                         ContentHandler textHandler = new BodyContentHandler(-1);
 464                         Metadata metadata = new Metadata();
 465                         OfficeParser parser = new OfficeParser();
 466                         ParseContext context = new ParseContext();
 467                         parser.parse(input, textHandler, metadata, context);
 468 //                      String[] result = textHandler.toString().split(regex);
 469 //                      for (int i=0; i<result.length && keepRunning; i++) {
 470 //                              if (interrupt) {
 471 //                                      processInterrupt();
 472 //                              }
 473 //                              addToIndex(r.getNoteGuid(), result[i], "RESOURCE");
 474 //                      }
 475                         updateResourceText(r.getGuid(), textHandler.toString());
 476                         input.close();
 477
 478                         f.close();
 479                 } catch (java.lang.ClassCastException e) {
 480                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());
 481                 } catch (FileNotFoundException e) {
 482                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());
 483                 } catch (IOException e) {
 484                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());
 485                 } catch (SAXException e) {
 486                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());
 487                 } catch (TikaException e) {
 488                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());
 489                 } catch (Exception e) {
 490                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());
 491                 } catch (java.lang.NoSuchMethodError e) {
 492                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());
 493                 } catch (Error e) {
 494                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());
 495                 }
 496         }
 497
 498
 499
 500         private void indexResourcePDF(Resource r) {
 501
 502                 Data d = r.getData();
 503                 for (int i=0; i<20 && d.getSize() == 0; i++)
 504                         d = r.getData();
 505                 if (d.getSize()== 0)
 506                         return;
 507                 QTemporaryFile f = writeResource(d);
 508                 if (!keepRunning) {
 509                         return;
 510                 }
 511
 512                 InputStream input;
 513                 try {
 514                         input = new FileInputStream(new File(f.fileName()));
 515                         ContentHandler textHandler = new BodyContentHandler(-1);
 516                         Metadata metadata = new Metadata();
 517                         PDFParser parser = new PDFParser();
 518                         ParseContext context = new ParseContext();
 519                         parser.parse(input, textHandler, metadata, context);
 520 //                      String[] result = textHandler.toString().split(regex);
 521 //                      for (int i=0; i<result.length && keepRunning; i++) {
 522 //                              if (interrupt) {
 523 //                                      processInterrupt();
 524 //                              }
 525 //                              addToIndex(r.getNoteGuid(), result[i], "RESOURCE");
 526 //                      }
 527                         updateResourceText(r.getGuid(), textHandler.toString());
 528                         input.close();
 529
 530                         f.close();
 531                 } catch (java.lang.ClassCastException e) {
 532                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());
 533                 } catch (FileNotFoundException e) {
 534                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());
 535                 } catch (IOException e) {
 536                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());
 537                 } catch (SAXException e) {
 538                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());
 539                 } catch (TikaException e) {
 540                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());
 541                 } catch (Exception e) {
 542                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());
 543                 } catch (java.lang.NoSuchMethodError e) {
 544                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());
 545                 } catch (Error e) {
 546                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());
 547                 }
 548         }
 549
 550
 551         private void indexResourceOOXML(Resource r) {
 552
 553                 Data d = r.getData();
 554                 for (int i=0; i<20 && d.getSize() == 0; i++)
 555                         d = r.getData();
 556                 if (d.getSize()== 0)
 557                         return;
 558                 QTemporaryFile f = writeResource(d);
 559                 if (!keepRunning) {
 560                         return;
 561                 }
 562
 563                 InputStream input;
 564                 try {
 565                         input = new FileInputStream(new File(f.fileName()));
 566                         ContentHandler textHandler = new BodyContentHandler(-1);
 567                         Metadata metadata = new Metadata();
 568                         OOXMLParser parser = new OOXMLParser();
 569                         ParseContext context = new ParseContext();
 570                         parser.parse(input, textHandler, metadata, context);
 571 //                      String[] result = textHandler.toString().split(regex);
 572 //                      for (int i=0; i<result.length && keepRunning; i++) {
 573 //                              if (interrupt) {
 574 //                                      processInterrupt();
 575 //                              }
 576 //                              addToIndex(r.getNoteGuid(), result[i], "RESOURCE");
 577 //                      }
 578                         updateResourceText(r.getGuid(), textHandler.toString());
 579                         input.close();
 580
 581                         f.close();
 582                 } catch (java.lang.ClassCastException e) {
 583                         logger.log(logger.LOW, "Cast exception: " +e.getMessage());
 584                 } catch (FileNotFoundException e) {
 585                         logger.log(logger.LOW, "FileNotFound  exception: " +e.getMessage());
 586                 } catch (IOException e) {
 587                         logger.log(logger.LOW, "IO  exception: " +e.getMessage());
 588                 } catch (SAXException e) {
 589                         logger.log(logger.LOW, "SAX  exception: " +e.getMessage());
 590                 } catch (TikaException e) {
 591                         logger.log(logger.LOW, "Tika  exception: " +e.getMessage());
 592                 } catch (Exception e) {
 593                         logger.log(logger.LOW, "Unknown  exception: " +e.getMessage());
 594                 } catch (java.lang.NoSuchMethodError e) {
 595                         logger.log(logger.LOW, "NoSuchMethod error: " +e.getMessage());
 596                 } catch (Error e) {
 597                         logger.log(logger.LOW, "Unknown error: " +e.getMessage());              }
 598         }
 599
 600
 601
 602         private QTemporaryFile writeResource(Data d) {
 603                 QTemporaryFile newFile = new QTemporaryFile();
 604                 newFile.open(OpenModeFlag.WriteOnly);
 605                 newFile.write(d.getBody());
 606                 newFile.close();
 607                 return newFile;
 608         }
 609
 610
 611         private String removeEnCrypt(String content) {
 612                 int index = content.indexOf("<en-crypt");
 613                 int endPos;
 614                 boolean tagFound = true;
 615                 while (tagFound && keepRunning) {
 616                         if (interrupt) {
 617                                 processInterrupt();
 618                         }
 619                         endPos = content.indexOf("</en-crypt>", index)+11;
 620                         if (endPos > -1 && index > -1) {
 621                                 content = content.substring(0,index)+content.substring(endPos);
 622                                 index = content.indexOf("<en-crypt");
 623                         } else {
 624                                 tagFound = false;
 625                         }
 626                 }
 627                 return content;
 628         }
 629
 630
 631 //      private void addToIndex(String guid, String word, String type) {
 632 //              if (foundWords.contains(word))
 633 //                      return;
 634 //              StringBuffer buffer = new StringBuffer(word.toLowerCase());
 635 //              for (int i=buffer.length()-1; i>=0; i--) {
 636 //                      if (!Character.isLetterOrDigit(buffer.charAt(i)) && specialIndexCharacters.indexOf(buffer.charAt(i)) == -1)
 637 //                              buffer.deleteCharAt(i);
 638 //                      else
 639 //                              break;
 640 //              }
 641 //              buffer = buffer.reverse();
 642 //              for (int i=buffer.length()-1; i>=0; i--) {
 643 //                      if (!Character.isLetterOrDigit(buffer.charAt(i)))
 644 //                              buffer.deleteCharAt(i);
 645 //                      else
 646 //                              break;
 647 //              }
 648 //              buffer = buffer.reverse();
 649 //              if (buffer.length() > 0) {
 650 //                      // We have a good word, now let's trim off junk at the beginning or end
 651 //                      if (!foundWords.contains(buffer.toString())) {
 652 //                              foundWords.add(buffer.toString());
 653 //                              foundWords.add(word);
 654 //                              conn.getWordsTable().addWordToNoteIndex(guid, buffer.toString(), type, 100);
 655 //                              uncommittedCount++;
 656 //                              if (uncommittedCount > 100) {
 657 //                                      conn.commitTransaction();
 658 //                                      uncommittedCount=0;
 659 //                              }
 660 //                      }
 661 //              }
 662 //              return;
 663 //      }
 664
 665         // ノートリソーステーブルのリソーステキストに追加
 666         private void updateResourceText(String guid, String text) {
 667                 conn.getNoteTable().noteResourceTable.updateResourceText(guid, text);
 668         }
 669
 670         private void scanUnindexed() {
 671 //              List<String> notes = conn.getNoteTable().getUnindexed();
 672                 guid = null;
 673                 boolean started = false;
 674 //              if (notes.size() > 0) {
 675 //                      signal.indexStarted.emit();
 676 //                      started = true;
 677 //              }
 678 //              for (int i=0; i<notes.size() && keepRunning; i++) {
 679 //                      if (interrupt) {
 680 //                              processInterrupt();
 681 //                      }
 682 //                      guid = notes.get(i);
 683 //                      if (guid != null && keepRunning) {
 684 //                              indexNoteContent();
 685 //                      }
 686 //              }
 687
 688                 List<String> unindexedResources = conn.getNoteTable().noteResourceTable.getUnindexed();
 689                 if (unindexedResources.size() > 0 && !started) {
 690                         signal.indexStarted.emit();
 691                         started = true;
 692                 }
 693                 for (int i=0; i<unindexedResources.size()&& keepRunning; i++) {
 694                         if (interrupt) {
 695                                 processInterrupt();
 696                         }
 697                         guid = unindexedResources.get(i);
 698                         if (keepRunning) {
 699                                 indexResource();
 700                         }
 701                 }
 702
 703                 // Cleanup stuff that was deleted at some point
 704                 List<String> guids = conn.getWordsTable().getGuidList();
 705                 logger.log(logger.LOW, "GUIDS in index: " +guids.size());
 706                 for (int i=0; i<guids.size() && keepRunning; i++) {
 707                         if (!conn.getNoteTable().exists(guids.get(i))) {
 708                                 logger.log(logger.LOW, "Old GUID found: " +guids.get(i));
 709                                 conn.getWordsTable().expunge(guids.get(i));
 710                         }
 711                 }
 712
 713                 if (started && keepRunning)
 714                         signal.indexFinished.emit();
 715         }
 716
 717         private void reindexNote() {
 718                 if (guid == null)
 719                         return;
 720                 conn.getNoteTable().setIndexNeeded(guid, true);
 721         }
 722
 723         private void reindexAll() {
 724                 conn.getNoteTable().reindexAllNotes();
 725                 conn.getNoteTable().noteResourceTable.reindexAll();
 726         }
 727
 728         private void waitSeconds(int len) {
 729                 long starttime = 0; // variable declared
 730                 //...
 731                 // for the first time, remember the timestamp
 732             starttime = System.currentTimeMillis();
 733                 // the next timestamp we want to wake up
 734                 starttime += (1000.0);
 735                 // Wait until the desired next time arrives using nanosecond
 736                 // accuracy timer (wait(time) isn't accurate enough on most platforms)
 737                 LockSupport.parkNanos((Math.max(0,
 738                     starttime - System.currentTimeMillis()) * 1000000));
 739         }
 740
 741         private void processInterrupt() {
 742                 conn.commitTransaction();
 743                 waitSeconds(1);
 744                 uncommittedCount = 0;
 745                 conn.beginTransaction();
 746                 interrupt = false;
 747         }
 748
 749 }