<classpathentry kind="src" path="src"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
<classpathentry kind="lib" path="lib/evernote-api-1.25.0.jar"/>
- <classpathentry kind="lib" path="lib/h2-1.3.158.jar"/>
<classpathentry kind="lib" path="lib/libthrift.jar"/>
<classpathentry kind="lib" path="lib/log4j-1.2.14.jar"/>
<classpathentry kind="lib" path="lib/commons-lang3-3.0.jar"/>
<classpathentry kind="lib" path="lib/httpcore-4.1.jar"/>
<classpathentry kind="lib" path="lib/httpmime-4.1.1.jar"/>
<classpathentry kind="lib" path="lib/pdfbox-app-1.6.0.jar"/>
- <classpathentry kind="lib" path="C:/pleiades/scribe-1.3.0.jar"/>
- <classpathentry kind="lib" path="C:/pleiades/qtjambi-win32-lgpl-4.5.2_01/qtjambi-4.5.2_01.jar"/>
- <classpathentry kind="lib" path="C:/pleiades/qtjambi-win32-lgpl-4.5.2_01/qtjambi-win32-msvc2005-4.5.2_01.jar"/>
+ <classpathentry kind="lib" path="lib/scribe-1.3.0.jar"/>
+ <classpathentry kind="lib" path="C:/Users/Yuki/Desktop/pleiades/qtjambi-win64-4.5.2_01.jar"/>
+ <classpathentry kind="lib" path="C:/Users/Yuki/Desktop/pleiades/qtjambi-win64-msvc2005x64-4.5.2_01.jar"/>
+ <classpathentry kind="lib" path="lib/lucene-core-3.6.2.jar"/>
+ <classpathentry kind="lib" path="lib/h2-1.3.163.jar"/>
+ <classpathentry kind="lib" path="lib/h2-lucene-ex-1.3.163.jar"/>
+ <classpathentry kind="lib" path="lib/lucene-analyzers-3.6.2.jar"/>
<classpathentry kind="output" path="bin"/>
</classpath>
import cx.fbn.nevernote.gui.ContainsAttributeFilterTable;
import cx.fbn.nevernote.gui.DateAttributeFilterTable;
import cx.fbn.nevernote.gui.ShortcutKeys;
+import cx.fbn.nevernote.sql.DatabaseConnection;
+import cx.fbn.nevernote.sql.driver.NSqlQuery;
import cx.fbn.nevernote.utilities.ApplicationLogger;
import cx.fbn.nevernote.utilities.Pair;
//public static int minimumWordCount = 2;
// Regular expression to parse text with when indexing
- private static String wordRegex;
+// private static String wordRegex;
// Experimental fixes. Set via Edit/Preferences/Debugging
public static boolean enableCarriageReturnFix = false;
getServer(); // Setup URL to connect to
// Get regular expressions used to parse out words
- settings.beginGroup("General");
- String regex = (String) settings.value("regex", "[,\\s]+");
- setWordRegex(regex);
- settings.endGroup();
+// settings.beginGroup("General");
+// String regex = (String) settings.value("regex", "[,\\s]+");
+// setWordRegex(regex);
+// settings.endGroup();
//Setup debugging information
settings.beginGroup("Debug");
}
// Get/Set word parsing regular expression
- public static String getWordRegex() {
- return wordRegex;
- }
- public static void setWordRegex(String r) {
- wordRegex = r;
- }
+// public static String getWordRegex() {
+// return wordRegex;
+// }
+// public static void setWordRegex(String r) {
+// wordRegex = r;
+// }
// Set the debug message level
public static void setMessageLevel(String msglevel) {
settings.endGroup();
}
// Get/Set characters that shouldn't be removed from a word
- public static String getSpecialIndexCharacters() {
- settings.beginGroup("Index");
- String text = (String)settings.value("specialCharacters", "");
- settings.endGroup();
- return text;
- }
- public static void setSpecialIndexCharacters(String value) {
- settings.beginGroup("Index");
- settings.setValue("specialCharacters", value);
- settings.endGroup();
- databaseCache = value;
- }
+// public static String getSpecialIndexCharacters() {
+// settings.beginGroup("Index");
+// String text = (String)settings.value("specialCharacters", "");
+// settings.endGroup();
+// return text;
+// }
+// public static void setSpecialIndexCharacters(String value) {
+// settings.beginGroup("Index");
+// settings.setValue("specialCharacters", value);
+// settings.endGroup();
+// databaseCache = value;
+// }
//*****************************************************************************
// Control how tag selection behaves (should they be "and" or "or" selections
}
// If we should automatically wildcard searches
- public static boolean automaticWildcardSearches() {
- settings.beginGroup("General");
- try {
- String value = (String)settings.value("automaticWildcard", "false");
- settings.endGroup();
- if (value.equals("true"))
- return true;
- else
- return false;
- } catch (java.lang.ClassCastException e) {
- Boolean value = (Boolean) settings.value("automaticWildcard", false);
- settings.endGroup();
- return value;
- }
-
- }
- public static void setAutomaticWildcardSearches(boolean value) {
- settings.beginGroup("General");
- settings.setValue("automaticWildcard", value);
- settings.endGroup();
- }
+// public static boolean automaticWildcardSearches() {
+// settings.beginGroup("General");
+// try {
+// String value = (String)settings.value("automaticWildcard", "false");
+// settings.endGroup();
+// if (value.equals("true"))
+// return true;
+// else
+// return false;
+// } catch (java.lang.ClassCastException e) {
+// Boolean value = (Boolean) settings.value("automaticWildcard", false);
+// settings.endGroup();
+// return value;
+// }
+//
+// }
+// public static void setAutomaticWildcardSearches(boolean value) {
+// settings.beginGroup("General");
+// settings.setValue("automaticWildcard", value);
+// settings.endGroup();
+// }
// If we should automatically select the children of any tag
public static boolean displayRightToLeft() {
settings.endGroup();
return value;
}
+
+ // タグを排除してプレーンテキストを抽出
+ public static String extractPlainText(String sourceText) {
+ String plainText = sourceText.replaceAll("<.+?>", ""); // タグを除去
+ plainText = plainText.replaceAll("\\s{2,}", " "); // 2個以上の空白文字を1文字の空白に変換
+ String kaigyo = System.getProperty("line.separator");
+ plainText = plainText.replaceAll(kaigyo, ""); // 改行を除去
+// plainText = plainText.replaceAll("<.+?>", ""); // <で始まり>で終わる文字列を除去
+
+ // HTML特殊文字のサニタイジングを解除
+ plainText = plainText.replaceAll("'", "'");
+ plainText = plainText.replaceAll(""", "\"");
+ plainText = plainText.replaceAll(">", ">");
+ plainText = plainText.replaceAll("<", "<");
+ plainText = plainText.replaceAll("&", "&");
+
+ plainText = plainText.replaceAll("&.+?;", ""); // その他HTML特殊文字があれば除去
+
+ return plainText;
+ }
+
+ // 全文検索機能の対象となるテーブルとカラムを再構築
+ public static boolean rebuildFullTextNoteTarget(DatabaseConnection dbConn) {
+ NSqlQuery nQuery = new NSqlQuery(dbConn.getConnection());
+ StringBuilder noteTableTarget = new StringBuilder();
+ boolean success = true;
+
+ if (Global.indexNoteBody()) {
+ noteTableTarget.append("CONTENTTEXT");
+ }
+ if (Global.indexNoteTitle()) {
+ if (noteTableTarget.length() > 0) {
+ noteTableTarget.append(", ");
+ }
+ noteTableTarget.append("TITLE");
+ }
+
+ if (noteTableTarget.length() > 0) {
+ nQuery.prepare("CALL FTL_CREATE_INDEX('PUBLIC', 'NOTE', :column);");
+ nQuery.bindValue(":column", noteTableTarget.toString());
+ if (!nQuery.exec()) {
+ success = false;
+ }
+ }
+
+ return success;
+ }
+
+ public static boolean rebuildFullTextResourceTarget(DatabaseConnection dbConn) {
+ NSqlQuery rQuery = new NSqlQuery(dbConn.getResourceConnection());
+ StringBuilder resourceTableTarget = new StringBuilder();
+ boolean success = true;
+
+ if (Global.indexAttachmentsLocally()) {
+ resourceTableTarget.append("RESOURCETEXT");
+ }
+
+ if (resourceTableTarget.length() > 0) {
+ rQuery.prepare("CALL FTL_CREATE_INDEX('PUBLIC', 'NOTERESOURCES', :column);");
+ rQuery.bindValue(":column", resourceTableTarget.toString());
+ if (!rQuery.exec()) {
+ success = false;
+ }
+ }
+
+ return success;
+ }
}
indexThread = new QThread(indexRunner, "Index Thread");
indexRunner.indexAttachmentsLocally = Global.indexAttachmentsLocally();
indexRunner.indexImageRecognition = Global.indexImageRecognition();
- indexRunner.indexNoteBody = Global.indexNoteBody();
- indexRunner.indexNoteTitle = Global.indexNoteTitle();
- indexRunner.specialIndexCharacters = Global.getSpecialIndexCharacters();
+// indexRunner.indexNoteBody = Global.indexNoteBody();
+// indexRunner.indexNoteTitle = Global.indexNoteTitle();
+// indexRunner.specialIndexCharacters = Global.getSpecialIndexCharacters();
indexThread.start();
synchronizeAnimationTimer = new QTimer();
saveNoteColumnPositions();
saveNoteIndexWidth();
showColumns();
- ConfigDialog settings = new ConfigDialog(this);
+ ConfigDialog settings = new ConfigDialog(this, conn);
String dateFormat = Global.getDateFormat();
String timeFormat = Global.getTimeFormat();
settings.exec();
indexRunner.indexAttachmentsLocally = Global.indexAttachmentsLocally();
- indexRunner.indexNoteBody = Global.indexNoteBody();
- indexRunner.indexNoteTitle = Global.indexNoteTitle();
- indexRunner.specialIndexCharacters = Global.getSpecialIndexCharacters();
+// indexRunner.indexNoteBody = Global.indexNoteBody();
+// indexRunner.indexNoteTitle = Global.indexNoteTitle();
+// indexRunner.specialIndexCharacters = Global.getSpecialIndexCharacters();
indexRunner.indexImageRecognition = Global.indexImageRecognition();
if (Global.showTrayIcon() || Global.minimizeOnClose())
trayIcon.show();
\r
import cx.fbn.nevernote.Global;\r
import cx.fbn.nevernote.NeverNote;\r
+import cx.fbn.nevernote.sql.DatabaseConnection;\r
+import cx.fbn.nevernote.sql.driver.NSqlQuery;\r
public class ConfigDialog extends QDialog {\r
private final QListWidget contentsWidget;\r
private final ConfigFontPage fontPage;\r
private final ConfigIndexPage indexPage;\r
private final ConfigRensoNoteListPage rensoNoteListPage;\r
private final NeverNote parent;\r
+ private final DatabaseConnection conn;\r
\r
- private final String iconPath = new String("classpath:cx/fbn/nevernote/icons/");\r
- \r
- public ConfigDialog(NeverNote parent) {\r
+ private final String iconPath = new String("classpath:cx/fbn/nevernote/icons/");\r
+ \r
+ public ConfigDialog(NeverNote parent, DatabaseConnection conn) {\r
this.parent = parent;\r
+ this.conn = conn;\r
\r
contentsWidget = new QListWidget(this);\r
setWindowIcon(new QIcon(iconPath+"config.png"));\r
Global.setIndexNoteBody(indexPage.getIndexNoteBody());\r
Global.setIndexNoteTitle(indexPage.getIndexNoteTitle());\r
Global.setIndexImageRecognition(indexPage.getIndexImageRecognition());\r
- Global.setAutomaticWildcardSearches(indexPage.getAutomaticWildcardSearches());\r
- Global.setSpecialIndexCharacters(indexPage.getSpecialCharacters());\r
+// Global.setAutomaticWildcardSearches(indexPage.getAutomaticWildcardSearches());\r
+// Global.setSpecialIndexCharacters(indexPage.getSpecialCharacters());\r
Global.setIncludeTagChildren(appearancePage.getIncludeTagChildren());\r
Global.setDisplayRightToLeft(appearancePage.getDisplayRightToLeft());\r
\r
Global.userStoreUrl = "https://"+debugPage.getServer()+"/edam/user";\r
- Global.setWordRegex(indexPage.getRegex());\r
+// Global.setWordRegex(indexPage.getRegex());\r
Global.setRecognitionWeight(indexPage.getRecognitionWeight());\r
Global.setIndexThreadSleepInterval(indexPage.getSleepInterval());\r
Global.setMessageLevel( debugPage.getDebugLevel());\r
Global.setBrowseWeight(rensoNoteListPage.getBrowseWeight());\r
Global.setCopyPasteWeight(rensoNoteListPage.getCopyPasteWeight());\r
Global.setAddNewNoteWeight(rensoNoteListPage.getAddNewNoteWeight());\r
- Global.setRensoItemClickWeight(rensoNoteListPage.getRensoItemClickWeight());
- Global.setSameTagWeight(rensoNoteListPage.getSameTagWeight());
+ Global.setRensoItemClickWeight(rensoNoteListPage.getRensoItemClickWeight());\r
+ Global.setSameTagWeight(rensoNoteListPage.getSameTagWeight());\r
Global.setSameNotebookWeight(rensoNoteListPage.getSameNotebookWeight());\r
Global.setENRelatedNotesWeight(rensoNoteListPage.getENRelatedNotesWeight());\r
\r
// 連想ノートリストをリフレッシュ\r
parent.getRensoNoteList().refreshRensoNoteList(parent.getCurrentNoteGuid());\r
\r
+ // 全文検索の対象項目を再設定\r
+ NSqlQuery nQuery = new NSqlQuery(conn.getConnection());\r
+ NSqlQuery rQuery = new NSqlQuery(conn.getResourceConnection()); \r
+ // カラム単位で削除できないので一度全部消す\r
+ nQuery.exec("CALL FTL_DROP_ALL();");\r
+ rQuery.exec("CALL FTL_DROP_ALL();");\r
+ // 再構築\r
+ Global.rebuildFullTextNoteTarget(conn);\r
+ Global.rebuildFullTextResourceTarget(conn);\r
+ \r
close();\r
}\r
\r
appearancePage.setDisplayRightToLeft(Global.displayRightToLeft());\r
appearancePage.setStartupNotebook(Global.getStartupNotebook());\r
\r
- indexPage.setRegex(Global.getWordRegex());\r
+// indexPage.setRegex(Global.getWordRegex());\r
indexPage.setSleepInterval(Global.getIndexThreadSleepInterval());\r
connectionPage.setSyncInterval(Global.getSyncInterval());\r
\r
private final QCheckBox indexAttachmentsLocally;\r
private final QCheckBox indexImageRecognition;\r
private final QCheckBox indexTitle;\r
- private final QCheckBox automaticWildcard;\r
- private final QLineEdit specialStrip;\r
+// private final QCheckBox automaticWildcard;\r
+// private final QLineEdit specialStrip;\r
private final QCheckBox indexBody;\r
- private final QLineEdit regexEdit;\r
+// private final QLineEdit regexEdit;\r
\r
public ConfigIndexPage(QWidget parent) {\r
// super(parent);\r
indexImageRecognition = new QCheckBox(tr("Index Image Recognition"));\r
indexImageRecognition.setChecked(Global.indexImageRecognition());\r
\r
- automaticWildcard = new QCheckBox(tr("Automatically Wildcard All Searches"));\r
- automaticWildcard.setChecked(Global.automaticWildcardSearches());\r
+// automaticWildcard = new QCheckBox(tr("Automatically Wildcard All Searches"));\r
+// automaticWildcard.setChecked(Global.automaticWildcardSearches());\r
\r
- specialStrip = new QLineEdit();\r
- specialStrip.setText(Global.getSpecialIndexCharacters());\r
+// specialStrip = new QLineEdit();\r
+// specialStrip.setText(Global.getSpecialIndexCharacters());\r
\r
QVBoxLayout attachmentLayout = new QVBoxLayout();\r
attachmentLayout.addWidget(indexBody);\r
attachmentLayout.addWidget(indexTitle);\r
attachmentLayout.addWidget(indexAttachmentsLocally);\r
attachmentLayout.addWidget(indexImageRecognition);\r
- attachmentLayout.addWidget(automaticWildcard);\r
+// attachmentLayout.addWidget(automaticWildcard);\r
\r
- QHBoxLayout specialCharLayout = new QHBoxLayout();\r
- specialCharLayout.addWidget(new QLabel(tr("Special Word Characters")));\r
- specialCharLayout.addWidget(specialStrip);\r
- attachmentLayout.addLayout(specialCharLayout);\r
+// QHBoxLayout specialCharLayout = new QHBoxLayout();\r
+// specialCharLayout.addWidget(new QLabel(tr("Special Word Characters")));\r
+// specialCharLayout.addWidget(specialStrip);\r
+// attachmentLayout.addLayout(specialCharLayout);\r
attachmentGroup.setLayout(attachmentLayout);\r
\r
// Index sleep interval\r
sleepGroup.setLayout(sleepLayout);\r
\r
// Regular Expressions for word parsing\r
- QGroupBox regexGroup = new QGroupBox(tr("Word Parse"));\r
- QLabel regexLabel = new QLabel(tr("Regular Expression"));\r
- regexEdit = new QLineEdit();\r
- regexEdit.setText(Global.getWordRegex());\r
-\r
- QHBoxLayout regexLayout = new QHBoxLayout();\r
- regexLayout.addWidget(regexLabel);\r
- regexLayout.addWidget(regexEdit); \r
- regexGroup.setLayout(regexLayout);\r
+// QGroupBox regexGroup = new QGroupBox(tr("Word Parse"));\r
+// QLabel regexLabel = new QLabel(tr("Regular Expression"));\r
+// regexEdit = new QLineEdit();\r
+// regexEdit.setText(Global.getWordRegex());\r
+//\r
+// QHBoxLayout regexLayout = new QHBoxLayout();\r
+// regexLayout.addWidget(regexLabel);\r
+// regexLayout.addWidget(regexEdit); \r
+// regexGroup.setLayout(regexLayout);\r
\r
\r
QVBoxLayout mainLayout = new QVBoxLayout();\r
mainLayout.addWidget(sleepGroup);\r
mainLayout.addWidget(weightGroup);\r
mainLayout.addWidget(attachmentGroup);\r
- mainLayout.addWidget(regexGroup);\r
+// mainLayout.addWidget(regexGroup);\r
mainLayout.addStretch(1);\r
setLayout(mainLayout);\r
\r
public boolean getIndexNoteTitle() {\r
return indexTitle.isChecked();\r
}\r
- public String getSpecialCharacters() {\r
- return specialStrip.text();\r
- }\r
+// public String getSpecialCharacters() {\r
+// return specialStrip.text();\r
+// }\r
public boolean getIndexImageRecognition() {\r
return indexImageRecognition.isChecked();\r
}\r
- public boolean getAutomaticWildcardSearches() {\r
- return automaticWildcard.isChecked();\r
- }\r
+// public boolean getAutomaticWildcardSearches() {\r
+// return automaticWildcard.isChecked();\r
+// }\r
\r
//*****************************************\r
//* Word length get/set methods \r
//*****************************************\r
//* Regex get/set methods \r
//*****************************************\r
- public void setRegex(String s) {\r
- regexEdit.setText(s);\r
- }\r
- public String getRegex() {\r
- return regexEdit.text();\r
- }\r
+// public void setRegex(String s) {\r
+// regexEdit.setText(s);\r
+// }\r
+// public String getRegex() {\r
+// return regexEdit.text();\r
+// }\r
\r
}\r
private final int relationPoints;
private final String noteCreated;
private final String tagNames;
- private String noteContent;
+ private final String noteContent;
private final RensoNoteList parent;
private final boolean isStared;
private final int allPointSum;
this.tagNames = new String(sb);
// this.noteContent = new String(note.getContent());
- this.noteContent = conn.getNoteTable().getNoteContentNoUTFConversion(note.getGuid());
- this.noteContent = this.noteContent.replaceAll("<.+?>", "");
- this.noteContent = this.noteContent.replaceAll("\\s{2,}", " ");
- String kaigyo = System.getProperty("line.separator");
- this.noteContent = this.noteContent.replaceAll(kaigyo, "");
-
+// this.noteContent = Global.extractPlainText(conn.getNoteTable().getNoteContentNoUTFConversion(note.getGuid()));
+ this.noteContent = conn.getNoteTable().getNoteContentText(note.getGuid());
palette = new QPalette();
palette.setColor(QPalette.ColorRole.Window, new QColor(255, 255, 255));
this.setPalette(palette);
executeSql("alter table note add column attributeContentClass VarChar");
executeSql("update note set attributeContentClass = ''");
}
+
+ // Apache Luceneを使った日本語検索のためのプレーンテキストノートコンテンツカラムを準備
+ if (!dbTableColumnExists("NOTE", "CONTENTTEXT")) {
+ executeSql("alter table note add column contentText VarChar");
+ executeSql("update note set contentText = ''");
+ NSqlQuery query = new NSqlQuery(conn);
+ query.exec("Select guid, content from Note where contentText = ''");
+ while (query.next()) {
+ String guid = query.valueString(0);
+ String content = query.valueString(1);
+ String contentText = Global.extractPlainText(content);
+ NSqlQuery query2 = new NSqlQuery(conn);
+ query2.prepare("update note set contentText=:contentText where guid=:guid");
+ query2.bindValue(":contentText", contentText);
+ query2.bindValue(":guid", guid);
+ query2.exec();
+ }
+
+ // Apache Luceneを使った全文検索のための準備
+ query.exec("CREATE ALIAS IF NOT EXISTS FTL_INIT FOR \"org.h2.fulltext.FullTextLuceneEx.init\"");
+ query.exec("CALL FTL_INIT()");
+
+ Global.rebuildFullTextNoteTarget(this);
+ }
+
+ // Apache Luceneを使った日本語検索のためのプレーンテキストノートリソースカラムを準備
+ NSqlQuery rQuery = new NSqlQuery(resourceConn);
+ rQuery.exec("select TABLE_NAME from INFORMATION_SCHEMA.COLUMNS where TABLE_NAME='NOTERESOURCES' and COLUMN_NAME='RESOURCETEXT'");
+ if (!rQuery.next()) {
+ rQuery.exec("alter table noteResources add column resourceText VarChar");
+ rQuery.exec("update noteResources set resourceText = ''");
+
+ // Apache Luceneを使った全文検索のための準備
+ rQuery.exec("CREATE ALIAS IF NOT EXISTS FTL_INIT FOR \"org.h2.fulltext.FullTextLuceneEx.init\"");
+ rQuery.exec("CALL FTL_INIT()");
+
+ Global.rebuildFullTextResourceTarget(this);
+ }
}
public void executeSql(String sql) {
NSqlQuery query = new NSqlQuery(db.getResourceConnection());\r
query.exec("update noteresources set isdirty=false");\r
}\r
+ \r
+ // リソーステキストを更新\r
+ public void updateResourceText(String guid, String text) {\r
+ logger.log(logger.HIGH, "Entering NoteResourceTable.updateResourceText");\r
+ NSqlQuery query = new NSqlQuery(db.getResourceConnection());\r
+ boolean check = query.prepare("Update noteResources set resourceText=:resourceText where guid=:guid");\r
+ if (!check) {\r
+ logger.log(logger.EXTREME, "Update resourceText sql prepare has failed.");\r
+ logger.log(logger.MEDIUM, query.lastError());\r
+ }\r
+ \r
+ query.bindValue(":resourceText", text);\r
+ query.bindValue(":guid", guid);\r
+ \r
+ check = query.exec();\r
+ if (!check) {\r
+ logger.log(logger.EXTREME, "Update resourceText has failed.");\r
+ logger.log(logger.MEDIUM, query.lastError());\r
+ }\r
+ logger.log(logger.HIGH, "Leaving NoteResourceTable.updateResourceText");\r
+ query.exec();\r
+ }\r
}\r
\r
+"contentHash, contentLength, created, updated, deleted, active, notebookGuid, "
+"attributeSubjectDate, attributeLatitude, attributeLongitude, attributeAltitude, "
+"attributeAuthor, attributeSource, attributeSourceUrl, attributeSourceApplication, "
- +"indexNeeded, isExpunged, isDirty, titlecolor, thumbnailneeded"
+ +"indexNeeded, isExpunged, isDirty, titlecolor, thumbnailneeded, contentText"
+") Values("
+":guid, :updateSequenceNumber, :title, :content, "
+":contentHash, :contentLength, :created, :updated, :deleted, :active, :notebookGuid, "
+":attributeSubjectDate, :attributeLatitude, :attributeLongitude, :attributeAltitude, "
+":attributeAuthor, :attributeSource, :attributeSourceUrl, :attributeSourceApplication, "
- +":indexNeeded, :isExpunged, :isDirty, -1, true) ");
+ +":indexNeeded, :isExpunged, :isDirty, -1, true, :contentText) ");
StringBuilder created = new StringBuilder(simple.format(n.getCreated()));
StringBuilder updated = new StringBuilder(simple.format(n.getUpdated()));
query.bindValue(":title", n.getTitle());
if (isDirty) {
EnmlConverter enml = new EnmlConverter(logger);
+ String contentText = Global.extractPlainText(enml.fixEnXMLCrap(enml.fixEnMediaCrap(n.getContent())));
+
query.bindValue(":content", enml.fixEnXMLCrap(enml.fixEnMediaCrap(n.getContent())));
+ query.bindValue(":contentText", contentText);
}
- else
+ else {
+ String contentText = Global.extractPlainText(n.getContent());
+
query.bindValue(":content", n.getContent());
+ query.bindValue(":contentText", contentText);
+
+ }
query.bindValue(":contentHash", n.getContentHash());
query.bindValue(":contentLength", n.getContentLength());
query.bindValue(":created", created.toString());
public void updateNoteContent(String guid, String content) {
logger.log(logger.HIGH, "Entering NoteTable.updateNoteContent");
NSqlQuery query = new NSqlQuery(db.getConnection());
- boolean check = query.prepare("Update Note set content=:content, updated=CURRENT_TIMESTAMP(), isDirty=true, indexNeeded=true, " +
+ boolean check = query.prepare("Update Note set content=:content, contentText=:contentText, updated=CURRENT_TIMESTAMP(), isDirty=true, indexNeeded=true, " +
" thumbnailneeded=true where guid=:guid");
if (!check) {
logger.log(logger.EXTREME, "Update note content sql prepare has failed.");
// codec = QTextCodec.codecForName("UTF-8");
// query.bindValue(":content", codec.fromUnicode(content).toString());
query.bindValue(":content", content);
+ query.bindValue(":contentText", Global.extractPlainText(content));
query.bindValue(":guid", guid);
check = query.exec();
newSegment +
n.getContent().substring(endPos);
NSqlQuery query = new NSqlQuery(db.getConnection());
- query.prepare("update note set isdirty=true, thumbnailneeded=true, content=:content where guid=:guid");
+ query.prepare("update note set isdirty=true, thumbnailneeded=true, content=:content, contentText=:contentText where guid=:guid");
query.bindValue(":content", content);
+ query.bindValue(":contentText", Global.extractPlainText(content));
query.bindValue(":guid", n.getGuid());
query.exec();
}
*
* boolean retVal = query.valueBoolean(0, false); return retVal; }
*/
+
+ // ノートコンテンツのプレーンテキストを取得
+ public String getNoteContentText(String noteGuid) {
+
+ if (noteGuid == null)
+ return null;
+ if (noteGuid.trim().equals(""))
+ return null;
+
+ NSqlQuery query = new NSqlQuery(db.getConnection());
+ query.prepare("Select contentText from Note where guid=:guid");
+ query.bindValue(":guid", noteGuid);
+ if (!query.exec()) {
+ logger.log(logger.MEDIUM, "Noteテーブルからプレーンコンテンツの取得失敗");
+ logger.log(logger.MEDIUM, query.lastError());
+ return null;
+ }
+ if (!query.next()) {
+ logger.log(logger.EXTREME, "SQL Retrieve failed for note guid "
+ + noteGuid + " in getNotePlainContent()");
+ logger.log(logger.EXTREME, " -> " + query.lastError().toString());
+ logger.log(logger.EXTREME, " -> " + query.lastError());
+ return null;
+ }
+
+ String noteContentText = query.valueString(0);
+ return noteContentText;
+ }
}
import com.evernote.edam.type.Notebook;\r
import com.evernote.edam.type.Tag;\r
\r
-import cx.fbn.nevernote.Global;\r
import cx.fbn.nevernote.sql.driver.NSqlQuery;\r
import cx.fbn.nevernote.utilities.ApplicationLogger;\r
\r
searchPhrases.add(word.toLowerCase());\r
}\r
if (!searchPhrase && pos < 0) {\r
- if (word != null && word.length() > 0 && !Global.automaticWildcardSearches())\r
+ if (word != null && word.length() > 0/* && !Global.automaticWildcardSearches()*/)\r
getWords().add(word); \r
- if (word != null && word.length() > 0 && Global.automaticWildcardSearches()) {\r
- String wildcardWord = word;\r
- if (!wildcardWord.startsWith("*"))\r
- wildcardWord = "*"+wildcardWord;\r
- if (!wildcardWord.endsWith("*"))\r
- wildcardWord = wildcardWord+"*";\r
- getWords().add(wildcardWord); \r
- }\r
+// if (word != null && word.length() > 0 && Global.automaticWildcardSearches()) {\r
+// String wildcardWord = word;\r
+// if (!wildcardWord.startsWith("*"))\r
+// wildcardWord = "*"+wildcardWord;\r
+// if (!wildcardWord.endsWith("*"))\r
+// wildcardWord = wildcardWord+"*";\r
+// getWords().add(wildcardWord); \r
+// }\r
// getWords().add("*"+word+"*"); //// WILDCARD\r
}\r
if (word.startsWith("intitle:")) \r
NSqlQuery indexQuery = new NSqlQuery(conn.getIndexConnection());\r
NSqlQuery mergeQuery = new NSqlQuery(conn.getConnection());\r
NSqlQuery deleteQuery = new NSqlQuery(conn.getConnection());\r
+ NSqlQuery ftlNoteQuery = new NSqlQuery(conn.getConnection());\r
+ NSqlQuery ftlResourceQuery = new NSqlQuery(conn.getResourceConnection());\r
+ ftlNoteQuery.prepare("SELECT N.GUID AS GUID FROM FTL_SEARCH_DATA(:text, 0, 0) FT, NOTE N WHERE FT.TABLE='NOTE' AND N.GUID=FT.KEYS[0]");\r
+ ftlResourceQuery.prepare("SELECT R.GUID AS GUID FROM FTL_SEARCH_DATA(:text, 0, 0) FT, NOTERESOURCES R WHERE FT.TABLE='NOTERESOURCES' AND R.GUID=FT.KEYS[0]");\r
\r
insertQuery.prepare("Insert into SEARCH_RESULTS (guid) values (:guid)");\r
mergeQuery.prepare("Insert into SEARCH_RESULTS_MERGE (guid) values (:guid)");\r
\r
if (subSelect) {\r
for (int i=0; i<getWords().size(); i++) {\r
+ // wordsテーブルから検索\r
if (getWords().get(i).indexOf("*") == -1) {\r
indexQuery.prepare("Select distinct guid from words where weight >= " +minimumRecognitionWeight +\r
" and word=:word");\r
mergeQuery.exec();\r
}\r
}\r
+ \r
+ // luceneによる全文検索 ノートテーブル\r
+ ftlNoteQuery.bindValue(":text", getWords().get(i));\r
+ ftlNoteQuery.exec();\r
+ while(ftlNoteQuery.next()) {\r
+ guid = ftlNoteQuery.valueString(0);\r
+ if (i==0 || any) {\r
+ insertQuery.bindValue(":guid", guid);\r
+ insertQuery.exec();\r
+ } else {\r
+ mergeQuery.bindValue(":guid", guid);\r
+ mergeQuery.exec();\r
+ }\r
+ }\r
+ // luceneによる全文検索 リソーステーブル\r
+ NSqlQuery rQuery = new NSqlQuery(conn.getResourceConnection());\r
+ ftlResourceQuery.bindValue(":text", getWords().get(i));\r
+ ftlResourceQuery.exec();\r
+ while(ftlResourceQuery.next()) {\r
+ guid = ftlResourceQuery.valueString(0);\r
+ \r
+ // リソースguidからノートguidを算出\r
+ rQuery.prepare("Select noteGuid from noteResources where guid=:guid");\r
+ rQuery.bindValue(":guid", guid);\r
+ rQuery.exec();\r
+ while(rQuery.next()) {\r
+ guid = rQuery.valueString(0);\r
+ if (i==0 || any) {\r
+ insertQuery.bindValue(":guid", guid);\r
+ insertQuery.exec();\r
+ } else {\r
+ mergeQuery.bindValue(":guid", guid);\r
+ mergeQuery.exec();\r
+ }\r
+ }\r
+ }\r
+ \r
if (i>0 && !any) {\r
deleteQuery.exec("Delete from SEARCH_RESULTS where guid not in (select guid from SEARCH_RESULTS_MERGE)");\r
deleteQuery.exec("Delete from SEARCH_RESULTS_MERGE");\r
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.locks.LockSupport;
-import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
import org.xml.sax.SAXException;
import com.evernote.edam.type.Data;
-import com.evernote.edam.type.Note;
import com.evernote.edam.type.Resource;
import com.trolltech.qt.core.QByteArray;
import com.trolltech.qt.core.QIODevice.OpenModeFlag;
public final int REINDEXNOTE=3;
public boolean keepRunning;
private final QDomDocument doc;
- private static String regex = Global.getWordRegex();
- public String specialIndexCharacters = "";
- public boolean indexNoteBody = true;
- public boolean indexNoteTitle = true;
+// private static String regex = Global.getWordRegex();
+// public String specialIndexCharacters = "";
+// public boolean indexNoteBody = true;
+// public boolean indexNoteTitle = true;
public boolean indexImageRecognition = true;
private final DatabaseConnection conn;
private volatile LinkedBlockingQueue<String> workQueue;
}
// Reindex a note
- public void indexNoteContent() {
- foundWords.clear();
-
- logger.log(logger.EXTREME, "Entering indexRunner.indexNoteContent()");
-
- logger.log(logger.EXTREME, "Getting note content");
- Note n = conn.getNoteTable().getNote(guid,true,false,true,true, true);
- String data;
- if (indexNoteBody) {
- data = n.getContent();
- data = conn.getNoteTable().getNoteContentNoUTFConversion(n.getGuid());
-
- logger.log(logger.EXTREME, "Removing any encrypted data");
- data = removeEnCrypt(data.toString());
- logger.log(logger.EXTREME, "Removing xml markups");
- } else
- data = "";
- String text;
- if (indexNoteTitle)
- text = removeTags(StringEscapeUtils.unescapeHtml4(data) +" "+ n.getTitle());
- else
- text = removeTags(StringEscapeUtils.unescapeHtml4(data));
-
- logger.log(logger.EXTREME, "Splitting words");
- String[] result = text.toString().split(regex);
- conn.commitTransaction();
- conn.beginTransaction();
- logger.log(logger.EXTREME, "Deleting existing words for note from index");
- conn.getWordsTable().expungeFromWordIndex(guid, "CONTENT");
-
- logger.log(logger.EXTREME, "Number of words found: " +result.length);
- for (int j=0; j<result.length && keepRunning; j++) {
- if (interrupt) {
- processInterrupt();
- }
- if (!result[j].trim().equals("")) {
- logger.log(logger.EXTREME, "Result word: " +result[j].trim());
- addToIndex(guid, result[j], "CONTENT");
- }
- }
-
- // Add tags
- for (int j=0; j<n.getTagNamesSize(); j++) {
- if (n.getTagNames() != null && n.getTagNames().get(j) != null && !n.getTagNames().get(j).trim().equals(""))
- addToIndex(guid, n.getTagNames().get(j), "CONTENT");
- }
-
- // If we were interrupted, we will reindex this note next time
- if (Global.keepRunning) {
- logger.log(logger.EXTREME, "Resetting note guid needed");
- conn.getNoteTable().setIndexNeeded(guid, false);
- }
- conn.commitTransaction();
- uncommittedCount = 0;
- logger.log(logger.EXTREME, "Leaving indexRunner.indexNoteContent()");
- }
+// public void indexNoteContent() {
+// foundWords.clear();
+//
+// logger.log(logger.EXTREME, "Entering indexRunner.indexNoteContent()");
+//
+// logger.log(logger.EXTREME, "Getting note content");
+// Note n = conn.getNoteTable().getNote(guid,true,false,true,true, true);
+// String data;
+// if (indexNoteBody) {
+// data = n.getContent();
+// data = conn.getNoteTable().getNoteContentNoUTFConversion(n.getGuid());
+//
+// logger.log(logger.EXTREME, "Removing any encrypted data");
+// data = removeEnCrypt(data.toString());
+// logger.log(logger.EXTREME, "Removing xml markups");
+// } else
+// data = "";
+// String text;
+// if (indexNoteTitle)
+// text = removeTags(StringEscapeUtils.unescapeHtml4(data) +" "+ n.getTitle());
+// else
+// text = removeTags(StringEscapeUtils.unescapeHtml4(data));
+//
+// logger.log(logger.EXTREME, "Splitting words");
+// String[] result = text.toString().split(regex);
+// conn.commitTransaction();
+// conn.beginTransaction();
+// logger.log(logger.EXTREME, "Deleting existing words for note from index");
+// conn.getWordsTable().expungeFromWordIndex(guid, "CONTENT");
+//
+// logger.log(logger.EXTREME, "Number of words found: " +result.length);
+// for (int j=0; j<result.length && keepRunning; j++) {
+// if (interrupt) {
+// processInterrupt();
+// }
+// if (!result[j].trim().equals("")) {
+// logger.log(logger.EXTREME, "Result word: " +result[j].trim());
+// addToIndex(guid, result[j], "CONTENT");
+// }
+// }
+//
+// // Add tags
+// for (int j=0; j<n.getTagNamesSize(); j++) {
+// if (n.getTagNames() != null && n.getTagNames().get(j) != null && !n.getTagNames().get(j).trim().equals(""))
+// addToIndex(guid, n.getTagNames().get(j), "CONTENT");
+// }
+//
+// // If we were interrupted, we will reindex this note next time
+// if (Global.keepRunning) {
+// logger.log(logger.EXTREME, "Resetting note guid needed");
+// conn.getNoteTable().setIndexNeeded(guid, false);
+// }
+// conn.commitTransaction();
+// uncommittedCount = 0;
+// logger.log(logger.EXTREME, "Leaving indexRunner.indexNoteContent()");
+// }
private String removeTags(String text) {
RTFParser parser = new RTFParser();
ParseContext context = new ParseContext();
parser.parse(input, textHandler, metadata, context);
- String[] result = textHandler.toString().split(regex);
- for (int i=0; i<result.length && keepRunning; i++) {
- addToIndex(r.getNoteGuid(), result[i], "RESOURCE");
- }
+// String[] result = textHandler.toString().split(regex);
+// for (int i=0; i<result.length && keepRunning; i++) {
+// addToIndex(r.getNoteGuid(), result[i], "RESOURCE");
+// }
+ updateResourceText(r.getGuid(), textHandler.toString());
input.close();
f.close();
OpenDocumentParser parser = new OpenDocumentParser();
ParseContext context = new ParseContext();
parser.parse(input, textHandler, metadata, context);
- String[] result = textHandler.toString().split(regex);
- for (int i=0; i<result.length && keepRunning; i++) {
- if (interrupt) {
- processInterrupt();
- }
- addToIndex(r.getNoteGuid(), result[i], "RESOURCE");
- }
+// String[] result = textHandler.toString().split(regex);
+// for (int i=0; i<result.length && keepRunning; i++) {
+// if (interrupt) {
+// processInterrupt();
+// }
+// addToIndex(r.getNoteGuid(), result[i], "RESOURCE");
+// }
+ updateResourceText(r.getGuid(), textHandler.toString());
input.close();
f.close();
OfficeParser parser = new OfficeParser();
ParseContext context = new ParseContext();
parser.parse(input, textHandler, metadata, context);
- String[] result = textHandler.toString().split(regex);
- for (int i=0; i<result.length && keepRunning; i++) {
- if (interrupt) {
- processInterrupt();
- }
- addToIndex(r.getNoteGuid(), result[i], "RESOURCE");
- }
+// String[] result = textHandler.toString().split(regex);
+// for (int i=0; i<result.length && keepRunning; i++) {
+// if (interrupt) {
+// processInterrupt();
+// }
+// addToIndex(r.getNoteGuid(), result[i], "RESOURCE");
+// }
+ updateResourceText(r.getGuid(), textHandler.toString());
input.close();
f.close();
PDFParser parser = new PDFParser();
ParseContext context = new ParseContext();
parser.parse(input, textHandler, metadata, context);
- String[] result = textHandler.toString().split(regex);
- for (int i=0; i<result.length && keepRunning; i++) {
- if (interrupt) {
- processInterrupt();
- }
- addToIndex(r.getNoteGuid(), result[i], "RESOURCE");
- }
+// String[] result = textHandler.toString().split(regex);
+// for (int i=0; i<result.length && keepRunning; i++) {
+// if (interrupt) {
+// processInterrupt();
+// }
+// addToIndex(r.getNoteGuid(), result[i], "RESOURCE");
+// }
+ updateResourceText(r.getGuid(), textHandler.toString());
input.close();
f.close();
OOXMLParser parser = new OOXMLParser();
ParseContext context = new ParseContext();
parser.parse(input, textHandler, metadata, context);
- String[] result = textHandler.toString().split(regex);
- for (int i=0; i<result.length && keepRunning; i++) {
- if (interrupt) {
- processInterrupt();
- }
- addToIndex(r.getNoteGuid(), result[i], "RESOURCE");
- }
+// String[] result = textHandler.toString().split(regex);
+// for (int i=0; i<result.length && keepRunning; i++) {
+// if (interrupt) {
+// processInterrupt();
+// }
+// addToIndex(r.getNoteGuid(), result[i], "RESOURCE");
+// }
+ updateResourceText(r.getGuid(), textHandler.toString());
input.close();
f.close();
}
- private void addToIndex(String guid, String word, String type) {
- if (foundWords.contains(word))
- return;
- StringBuffer buffer = new StringBuffer(word.toLowerCase());
- for (int i=buffer.length()-1; i>=0; i--) {
- if (!Character.isLetterOrDigit(buffer.charAt(i)) && specialIndexCharacters.indexOf(buffer.charAt(i)) == -1)
- buffer.deleteCharAt(i);
- else
- break;
- }
- buffer = buffer.reverse();
- for (int i=buffer.length()-1; i>=0; i--) {
- if (!Character.isLetterOrDigit(buffer.charAt(i)))
- buffer.deleteCharAt(i);
- else
- break;
- }
- buffer = buffer.reverse();
- if (buffer.length() > 0) {
- // We have a good word, now let's trim off junk at the beginning or end
- if (!foundWords.contains(buffer.toString())) {
- foundWords.add(buffer.toString());
- foundWords.add(word);
- conn.getWordsTable().addWordToNoteIndex(guid, buffer.toString(), type, 100);
- uncommittedCount++;
- if (uncommittedCount > 100) {
- conn.commitTransaction();
- uncommittedCount=0;
- }
- }
- }
- return;
+// private void addToIndex(String guid, String word, String type) {
+// if (foundWords.contains(word))
+// return;
+// StringBuffer buffer = new StringBuffer(word.toLowerCase());
+// for (int i=buffer.length()-1; i>=0; i--) {
+// if (!Character.isLetterOrDigit(buffer.charAt(i)) && specialIndexCharacters.indexOf(buffer.charAt(i)) == -1)
+// buffer.deleteCharAt(i);
+// else
+// break;
+// }
+// buffer = buffer.reverse();
+// for (int i=buffer.length()-1; i>=0; i--) {
+// if (!Character.isLetterOrDigit(buffer.charAt(i)))
+// buffer.deleteCharAt(i);
+// else
+// break;
+// }
+// buffer = buffer.reverse();
+// if (buffer.length() > 0) {
+// // We have a good word, now let's trim off junk at the beginning or end
+// if (!foundWords.contains(buffer.toString())) {
+// foundWords.add(buffer.toString());
+// foundWords.add(word);
+// conn.getWordsTable().addWordToNoteIndex(guid, buffer.toString(), type, 100);
+// uncommittedCount++;
+// if (uncommittedCount > 100) {
+// conn.commitTransaction();
+// uncommittedCount=0;
+// }
+// }
+// }
+// return;
+// }
+
+ // ノートリソーステーブルのリソーステキストに追加
+ private void updateResourceText(String guid, String text) {
+ conn.getNoteTable().noteResourceTable.updateResourceText(guid, text);
}
private void scanUnindexed() {
- List<String> notes = conn.getNoteTable().getUnindexed();
+// List<String> notes = conn.getNoteTable().getUnindexed();
guid = null;
boolean started = false;
- if (notes.size() > 0) {
- signal.indexStarted.emit();
- started = true;
- }
- for (int i=0; i<notes.size() && keepRunning; i++) {
- if (interrupt) {
- processInterrupt();
- }
- guid = notes.get(i);
- if (guid != null && keepRunning) {
- indexNoteContent();
- }
- }
+// if (notes.size() > 0) {
+// signal.indexStarted.emit();
+// started = true;
+// }
+// for (int i=0; i<notes.size() && keepRunning; i++) {
+// if (interrupt) {
+// processInterrupt();
+// }
+// guid = notes.get(i);
+// if (guid != null && keepRunning) {
+// indexNoteContent();
+// }
+// }
List<String> unindexedResources = conn.getNoteTable().noteResourceTable.getUnindexed();
if (unindexedResources.size() > 0 && !started) {
logger.log(logger.EXTREME, "Note guid: " +currentNoteGuid);\r
logger.log(logger.EXTREME, "Note Text:" +currentNote);\r
QDomDocument doc = new QDomDocument();\r
- QDomDocument.Result result = doc.setContent(currentNote.getContent());\r
-\r
+// QDomDocument.Result result = doc.setContent(currentNote.getContent());\r
+ QDomDocument.Result result = doc.setContent(conn.getNoteTable().getNoteContentNoUTFConversion(currentNote.getGuid()));\r
+ \r
// Handle any errors\r
if (!result.success) {\r
logger.log(logger.LOW, "Error parsing document. Attempting to restructure");\r
for (int j=z-1; j>i+1; j--) \r
html.deleteCharAt(j);\r
} \r
+ \r
+ QTextCodec codec;\r
+ codec = QTextCodec.codecForName("UTF-8");\r
+ String value = codec.fromUnicode(html.toString()).toString();\r
+ return value;\r
\r
- return html.toString(); //.replace("<Body", "<Body dir=\"rtl\"");\r
+// return html.toString(); //.replace("<Body", "<Body dir=\"rtl\"");\r
} \r
\r
private void addImageHilight(String resGuid, QFile f) {\r
}\r
\r
// We found a text node, so we need to search for things to hilight\r
- private void scanWords(QDomNode node) {\r
+ private void scanWords(QDomNode node) { \r
String value = node.nodeValue();\r
+ \r
QDomDocumentFragment fragment = doc.createDocumentFragment();\r
boolean matchFound = false;\r
int previousPosition = 0;\r
String term = terms.get(i);\r
if (term.indexOf("*") > -1) {\r
term = term.replace("*", "");\r
- } else {\r
+ }/* else {\r
term = "\\b"+term+"\\b";\r
- }\r
+ }*/\r
regex.append(term);\r
if (i<terms.size()-1)\r
regex.append("|"); \r
OSDN Git Service
