From fbf3c5b68df83d12fb6aa96becfc1f48d62391a6 Mon Sep 17 00:00:00 2001 From: vysitor Date: Sun, 9 Mar 2025 23:05:35 +0100 Subject: [PATCH] Correction to enable local database use, slight rework --- .DS_Store | Bin 6148 -> 0 bytes src/.DS_Store | Bin 6148 -> 0 bytes src/main/.DS_Store | Bin 6148 -> 0 bytes src/main/java/.DS_Store | Bin 6148 -> 0 bytes .../project/gruppe_05_1/Main.java | 23 +--- .../gruppe_05_1/database/MongoDBHandler.java | 120 +++++++++++------- .../gruppe_05_1/database/MongoPprUtils.java | 51 ++------ .../project/gruppe_05_1/util/PPRUtils.java | 17 --- src/main/resources/.DS_Store | Bin 6148 -> 0 bytes 9 files changed, 91 insertions(+), 120 deletions(-) delete mode 100644 .DS_Store delete mode 100644 src/.DS_Store delete mode 100644 src/main/.DS_Store delete mode 100644 src/main/java/.DS_Store delete mode 100644 src/main/resources/.DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 6799dc56e42b2b388b6a47a9c6899b991b6bd97e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKJ5Iwu5S=9{END_v?iF%_B{C<-0YHf*C}M3IdUu?Gnp1EA?!irX^WhV%a7hup zk!GHEKD+iSc03}IHc!J^WF;aUR8gMQw4!mfB}AOmE844gXy=-DdWl_M<~ zAOmFJn*qHaGE}hyPL6hUz}N@?)Zcg$JliY*%<2G^z{wFFNb*vkm!=pol9$7uBd-Kb zj$SUwPR2fQa*7Eh+3E0SOP5p}X~_T?IAvhr%a!*3Tj(G2|7j6dWPl8u6$5G6Z~F~C zDSB)3aoTG$^d9;p$hCABYsElo#aL)7zW$Y$^%?OJI5}E5{8kR+kAU(*Ap`%xz$d5U BCcgjx diff --git a/src/.DS_Store b/src/.DS_Store deleted file mode 100644 index 7cbb9350a57ec1469c8047c4081470492225b483..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKK~BR!475v-BK1;n;4huGjSuh_#0U5iSH|{&5=G_6 z0mhPdvbHCi%ps0NL|2d7rN}}=YPg|1+cGuFn@?<3A{9{W9=qG+`lf65lgh&Z<5tqj zwcN|Yh(G(d9Ol}tX*RoVgIN9YG=DvLdp{rNw?F3hcY|j;J_D6T1*iZOpaN9jUn+o} zZC1|%xl#ctKn1=PuB-*`7%`z!%W769ggeIOz*4Jt6GT1E^F zI^wC+<$`@+&_(@Z+^3wZWkOLu9r5JlqB)Q&6`%rR1&(67u>OAmKQRA~N!(EZD)6Ti z(5hKCE4)(at&^9tUYp=o@GnEHmt*l(4D?ovjkV&_L0z(IXhxcS>-l)sZ!M39h}>{9>l1Z}D1$O49SkkPe%6L0YGJ42dn~T{qiMNZ zH6qRMUm4)DOKDDHx}|Y+e#yf}-M1`@VzMYFux0N@cTdMJuY=e>`_;d?uDu&+cttw2 zrc1h^k~Wx4c^>V624`5IC91xc``+`{iR`5{@@H4)>$lI{k5eu6b1IBX7@ z#ehqTQ8C0LX>Dyh9M@V8y@0ZCTqU?o!6cMo#BwR_L4&~Vc>>H8D?wNw_9GBzu)!Gk HQwF{PY~fF| diff --git a/src/main/java/.DS_Store b/src/main/java/.DS_Store deleted file mode 100644 index f1ac8ac2ee0489b828c3923f4b30172b71962d9d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKF-`+P47AA=C()d!DEABb!4e&Pq-Ka+;mGs>bK_U`Qq?$HO#Jl$nNe2yH<11XjFg-Pys4H1^%@H z=-FoVc_2qBKn1A4hXVG0NN~ekun+WK2L`|91MM2`hPl5bfXM>DT(A#B1g1d+235-w zLxYZZDs{PF9~g8|KNZNQt7RukF#Ez;5YE2q1MYWcq;~aE5^cF@#UZ{*&O*?un%-P;!X$hB4E1E JsK9S1@D65dC|v*m diff --git a/src/main/java/org/texttechnologylab/project/gruppe_05_1/Main.java b/src/main/java/org/texttechnologylab/project/gruppe_05_1/Main.java index 21d0b64..2ab030f 100644 --- a/src/main/java/org/texttechnologylab/project/gruppe_05_1/Main.java +++ b/src/main/java/org/texttechnologylab/project/gruppe_05_1/Main.java @@ -1,26 +1,16 @@ package org.texttechnologylab.project.gruppe_05_1; -import com.mongodb.client.MongoDatabase; import org.texttechnologylab.project.gruppe_05_1.database.*; -import org.texttechnologylab.project.gruppe_05_1.domain.mdb.Mdb; -import org.texttechnologylab.project.gruppe_05_1.domain.mdb.MdbDocument; -import org.texttechnologylab.project.gruppe_05_1.nlp.NlpUtils; -import org.texttechnologylab.project.gruppe_05_1.nlp.XmiExtractor; import org.texttechnologylab.project.gruppe_05_1.rest.RESTHandler; import org.texttechnologylab.project.gruppe_05_1.util.Logger; import org.texttechnologylab.project.gruppe_05_1.util.PPRUtils; -import org.texttechnologylab.project.gruppe_05_1.util.PropertiesUtils; -import org.texttechnologylab.project.gruppe_05_1.util.XmlUtils; import org.texttechnologylab.project.gruppe_05_1.xml.FileObjectFactory; import org.texttechnologylab.project.gruppe_05_1.xml.speeches.SpeechParser; -import org.w3c.dom.Element; import org.xml.sax.SAXException; import javax.xml.parsers.ParserConfigurationException; import java.io.IOException; -import java.util.List; -import java.util.Properties; import static java.lang.Boolean.FALSE; import static java.lang.Boolean.TRUE; @@ -57,10 +47,10 @@ public class Main { public static void main(String[] args) throws ParserConfigurationException, IOException, SAXException { //TEST - + MongoDBHandler mongoDBHandler = new MongoDBHandler(); SpeechIndexFactoryImpl speechIndexFactory = new SpeechIndexFactoryImpl(); - if (MongoPprUtils.getSpeechCollection().countDocuments() != 0) { + if (mongoDBHandler.getDatabase().getCollection(MongoPprUtils.SPEECH_COLLECTION_NAME).countDocuments() != 0) { System.out.println("Speeches werden nicht gelesen, da sie bereits in der Datenbank stehen"); } else { @@ -79,8 +69,7 @@ public class Main { System.out.println("SESSIONCOUNT: " + speechIndex.getSessions().size()); System.out.println("AGENDAITEMCOUNT: " + speechIndex.getAgendaItems().size()); - MongoDBHandler mongoDBHandler = new MongoDBHandler(); - mongoDBHandler.deleteAllDocuments(); // Clear the DB + mongoDBHandler.deleteSpeechRelatedDocuments(); // Clear speeches, sessions, agendas (history) Logger.pink("Adding Sessions to DB..."); mongoDBHandler.insertSessions(speechIndex.getSessions()); @@ -90,13 +79,8 @@ public class Main { Logger.pink("Adding Speeches to DB..."); mongoDBHandler.insertSpeeches(speechIndex.getSpeeches()); - - mongoDBHandler.close(); // Close the connection to the DB } - // Stellt fest, dass alle nötigen Datenbank-Collections existieren - PPRUtils.ensureCollectionExist(); - // Alle Informationen (Parlamentarier, Reden, Kommentare etc.) lesen und in die Mongo-DB einfügen, falls diese noch nicht vorhanden sind. PPRUtils.parlamentExplorerInit(xmlFactory, mongoFactory); @@ -105,6 +89,7 @@ public class Main { // NLP-Verarbeitung - TODO // NlpUtils.importXmiData(); + mongoDBHandler.close(); // Close the connection to the DB RESTHandler restHandler = new RESTHandler(); restHandler.startJavalin(); diff --git a/src/main/java/org/texttechnologylab/project/gruppe_05_1/database/MongoDBHandler.java b/src/main/java/org/texttechnologylab/project/gruppe_05_1/database/MongoDBHandler.java index f23152c..44f84c0 100644 --- a/src/main/java/org/texttechnologylab/project/gruppe_05_1/database/MongoDBHandler.java +++ b/src/main/java/org/texttechnologylab/project/gruppe_05_1/database/MongoDBHandler.java @@ -8,16 +8,12 @@ import com.mongodb.client.MongoClients; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import com.mongodb.client.model.*; -import exceptions.AgendaItemNotFoundException; -import exceptions.MemberNotFoundException; import exceptions.ServerErrorException; import exceptions.SessionNotFoundException; import org.bson.Document; import org.bson.conversions.Bson; import org.bson.types.ObjectId; -import org.texttechnologylab.DockerUnifiedUIMAInterface.connection.mongodb.MongoDBConfig; import org.texttechnologylab.project.gruppe_05_1.database.domainimp.speeches.AgendaItem_MongoDB_Impl; -import org.texttechnologylab.project.gruppe_05_1.database.domainimp.speeches.MemberOfParliament_MongoDB_Impl; import org.texttechnologylab.project.gruppe_05_1.database.domainimp.speeches.Session_MongoDB_Impl; import org.texttechnologylab.project.gruppe_05_1.database.domainimp.speeches.Speech_MongoDB_Impl; import org.texttechnologylab.project.gruppe_05_1.util.Logger; @@ -25,7 +21,6 @@ import org.texttechnologylab.project.gruppe_05_1.util.PropertiesUtils; import org.texttechnologylab.project.gruppe_05_1.xml.speeches.Impls.*; import org.texttechnologylab.project.gruppe_05_1.xml.speeches.Interfaces.*; -import java.io.IOException; import java.util.*; import java.util.concurrent.TimeUnit; @@ -49,8 +44,8 @@ public class MongoDBHandler { private static String collection; private static String databaseName; + private MongoCollection speakerCollection; private MongoCollection speechesCollection; - private MongoCollection sessionsCollection; private MongoCollection agendaItemsCollection; private MongoCollection historyCollection; @@ -71,27 +66,41 @@ public class MongoDBHandler { collection = mongoProperties.getProperty("remote_collection"); databaseName = mongoProperties.getProperty("remote_database"); - MongoCredential credential = MongoCredential - .createCredential( - user, - databaseName, - password.toCharArray()); + // URI für lokale Datenbank oder für eine Datenbank auf dem Server + String uri; + if ( (localServer != null) && (! localServer.isBlank())) { + uri = localServer; + mongoClient = MongoClients.create(uri); - MongoClientSettings settings = MongoClientSettings.builder() - .credential(credential) - .timeout(180, TimeUnit.HOURS) // needs increased timeout for the bulk speech inserts - .applyToClusterSettings(builder -> - builder.hosts(List.of(new ServerAddress(remoteServer, Integer.parseInt(port))))) - .build(); + // Connect + database = mongoClient.getDatabase(databaseName); + } else { + MongoCredential credential = MongoCredential + .createCredential( + user, + databaseName, + password.toCharArray()); - mongoClient = MongoClients.create(settings); - database = mongoClient.getDatabase(databaseName); - speechesCollection = database.getCollection("speech"); - sessionsCollection = database.getCollection("sessions"); - agendaItemsCollection = database.getCollection("agendaItems"); - historyCollection = database.getCollection("history"); + MongoClientSettings settings = MongoClientSettings.builder() + .credential(credential) + .timeout(180, TimeUnit.HOURS) // needs increased timeout for the bulk speech inserts + .applyToClusterSettings(builder -> + builder.hosts(List.of(new ServerAddress(remoteServer, Integer.parseInt(port))))) + .build(); + + mongoClient = MongoClients.create(settings); + database = mongoClient.getDatabase(databaseName); + } + + speakerCollection = database.getCollection(MongoPprUtils.SPEAKER_COLLECTION_NAME); + speechesCollection = database.getCollection(MongoPprUtils.SPEECH_COLLECTION_NAME); + sessionsCollection = database.getCollection(MongoPprUtils.SESSION_COLLECTION_NAME); + agendaItemsCollection = database.getCollection(MongoPprUtils.AGENDA_ITEMS_COLLECTION_NAME); + historyCollection = database.getCollection(MongoPprUtils.HISTORY_COLLECTION_NAME); + + createIndicesForSpeakerCollection(); + createIndicesForSpeechCollection(); Logger.info("Connected to MongoDB database: " + databaseName); - } public MongoDatabase getDatabase() { @@ -105,18 +114,22 @@ public class MongoDBHandler { */ static public MongoDatabase getMongoDatabase() { - if (mongoDatabase == null) { - Properties mongoProperties = PropertiesUtils.readPropertiesFromResource(propertiesFileName); - // Zugangsdaten - localServer = mongoProperties.getProperty("localserver"); - remoteServer = mongoProperties.getProperty("remote_host"); - user = mongoProperties.getProperty("remote_user"); - password = mongoProperties.getProperty("remote_password"); - port = mongoProperties.getProperty("remote_port"); - collection = mongoProperties.getProperty("remote_collection"); - databaseName = mongoProperties.getProperty("remote_database"); + if (mongoDatabase != null) { + return mongoDatabase; } + + Properties mongoProperties = PropertiesUtils.readPropertiesFromResource(propertiesFileName); + // Zugangsdaten + localServer = mongoProperties.getProperty("localserver"); + remoteServer = mongoProperties.getProperty("remote_host"); + user = mongoProperties.getProperty("remote_user"); + password = mongoProperties.getProperty("remote_password"); + port = mongoProperties.getProperty("remote_port"); + collection = mongoProperties.getProperty("remote_collection"); + databaseName = mongoProperties.getProperty("remote_database"); + + // MongoDBClient erzeugen // String uri = mongoServer + "://" + mongoUser + ":" + mongoPassword + "@" + mongoNeetwork; // cluster, network, user... @@ -158,9 +171,9 @@ public class MongoDBHandler { * * @return List with the names of all collections */ - static public Set getCollectionNames() { - // return getMongoDatabase().listCollectionNames().into(new ArrayList<>()); - return getMongoDatabase().listCollectionNames().into(new HashSet<>()); + public Set getCollectionNames() { + // return getDatabase().listCollectionNames().into(new ArrayList<>()); + return getDatabase().listCollectionNames().into(new HashSet<>()); } /** @@ -168,8 +181,8 @@ public class MongoDBHandler { * @param name Name of collection to check for existance * @return does the collection exist */ - static public boolean collectionExists(String name) { - return getMongoDatabase().listCollectionNames().into(new ArrayList<>()).contains(name); + public boolean collectionExists(String name) { + return getDatabase().listCollectionNames().into(new ArrayList<>()).contains(name); } @@ -193,8 +206,8 @@ public class MongoDBHandler { } } - static public void createCollectionIfNotExist(String collectionName) { - createCollectionIfNotExist(getMongoDatabase(), collectionName); + public void createCollectionIfNotExist(String collectionName) { + createCollectionIfNotExist(getDatabase(), collectionName); } @@ -211,8 +224,8 @@ public class MongoDBHandler { } } - static public void createCollection(String collectionName) { - createCollection(getMongoDatabase(), collectionName); + public void createCollection(String collectionName) { + createCollection(getDatabase(), collectionName); } @@ -261,8 +274,23 @@ public class MongoDBHandler { } } - static public void createOrTrancateCollection(String collectionName) { - createOrTrancateCollection(getMongoDatabase(), collectionName); + public void createIndicesForSpeakerCollection() { + if (speakerCollection.listIndexes().into(new ArrayList<>()).size() == 1) { + MongoDBHandler.createIndexForCollection(speakerCollection,"name", true); + MongoDBHandler.createIndexForCollection(speakerCollection,"firstName", true); + MongoDBHandler.createIndexForCollection(speakerCollection,"party", true); + } + + } + public void createIndicesForSpeechCollection() { + if (speechesCollection.listIndexes().into(new ArrayList<>()).size() == 1) { + MongoDBHandler.createIndexForCollection(speechesCollection, "speakerId", true); + MongoDBHandler.createIndexForCollection(speechesCollection, "speechKey", true); + } + } + + public void createOrTrancateCollection(String collectionName) { + createOrTrancateCollection(database, collectionName); } /** @@ -648,7 +676,7 @@ public class MongoDBHandler { ); } - public void deleteAllDocuments() { + public void deleteSpeechRelatedDocuments() { speechesCollection.deleteMany(new Document()); sessionsCollection.deleteMany(new Document()); agendaItemsCollection.deleteMany(new Document()); diff --git a/src/main/java/org/texttechnologylab/project/gruppe_05_1/database/MongoPprUtils.java b/src/main/java/org/texttechnologylab/project/gruppe_05_1/database/MongoPprUtils.java index aec92aa..b020bbf 100644 --- a/src/main/java/org/texttechnologylab/project/gruppe_05_1/database/MongoPprUtils.java +++ b/src/main/java/org/texttechnologylab/project/gruppe_05_1/database/MongoPprUtils.java @@ -2,7 +2,6 @@ package org.texttechnologylab.project.gruppe_05_1.database; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; -import lombok.extern.slf4j.Slf4j; import org.bson.Document; import org.texttechnologylab.project.gruppe_05_1.domain.html.Parlamentarier; import org.texttechnologylab.project.gruppe_05_1.domain.html.ParlamentarierDetails; @@ -16,7 +15,6 @@ import java.util.*; * Diese Klasse beinhaltet Mongo-Utilities, welche spezifisch für die PPR-Datenstrukturen sind. * Mongo-Utilities genereller Natur stehen in der Klasse MongoDBHandler. */ -@Slf4j public class MongoPprUtils { /* @@ -25,6 +23,10 @@ public class MongoPprUtils { */ public static final String SPEAKER_COLLECTION_NAME = "speaker"; public static final String SPEECH_COLLECTION_NAME = "speech"; + + public static final String SESSION_COLLECTION_NAME = "sessions"; + public static final String AGENDA_ITEMS_COLLECTION_NAME = "agendaItems"; + public static final String HISTORY_COLLECTION_NAME = "history"; public static final String PICTURES_COLLECTION_NAME = "pictures"; public static final String COMMENT_COLLECTION_NAME = "comment"; @@ -43,53 +45,26 @@ public class MongoPprUtils { return speechCollection; } - public static MongoCollection getPicturesCollection() { - if (picturesCollection == null) picturesCollection = MongoDBHandler.getMongoDatabase().getCollection(PICTURES_COLLECTION_NAME); - return picturesCollection; - } - - public static MongoCollection getCommentCollection() { - if (commentCollection == null) commentCollection = MongoDBHandler.getMongoDatabase().getCollection(COMMENT_COLLECTION_NAME); - return commentCollection; - } /** * Create the Speaker Collection and useful indices for it */ - public static void createSpeakerCollection() { - MongoDBHandler.createCollection(MongoPprUtils.SPEAKER_COLLECTION_NAME); - - MongoDBHandler.createIndexForCollection(getSpeakerCollection(), Arrays.asList("name", "firstName", "party"), true); + public static void createIndexForSpeakerCollection() { + // MongoDBHandler.createIndexForCollection(getSpeakerCollection(), Arrays.asList("name", "firstName", "party"), true); + MongoDBHandler.createIndexForCollection(getSpeakerCollection(),"name", true); + MongoDBHandler.createIndexForCollection(getSpeakerCollection(),"firstName", true); + MongoDBHandler.createIndexForCollection(getSpeakerCollection(),"party", true); } /** * Create the Speech Collection and useful indices for it */ - public static void createSpeechCollection() { - MongoDBHandler.createCollection(MongoPprUtils.SPEECH_COLLECTION_NAME); - MongoDBHandler.createIndexForCollection(getSpeechCollection(), "speaker", true); + public static void createIndexForSpeechCollection() { + MongoDBHandler.createIndexForCollection(getSpeechCollection(), "speakerId", true); + MongoDBHandler.createIndexForCollection(getSpeechCollection(), "speechKey", true); } - /** - * Create the Comment Collection and useful indices for it - */ - public static void createCommentCollection() { - MongoDBHandler.createCollection(MongoPprUtils.COMMENT_COLLECTION_NAME); - MongoDBHandler.createIndexForCollection(getCommentCollection(), Arrays.asList("speaker", "speech"), true); - } - - - /** - * Create the Picture Collection and useful indices for it - */ - public static void createPictureCollection() { - MongoDBHandler.createCollection(MongoPprUtils.PICTURES_COLLECTION_NAME); - - // TODO: für welche Felder sollen Indizes gebaut werden? - // MongoDBHandler.createIndexForCollection(getPicturesCollection(), Arrays.asList("field_1", "field_2"), true); - } - /** * Truncate the Speaker Collection. * Note that it is quicker (and saves space) to drop and re-create rather than removing all documents using "remove({})" @@ -97,7 +72,7 @@ public class MongoPprUtils { public static void truncateSpeakerCollection() { getSpeakerCollection().drop(); - createSpeechCollection(); + createIndexForSpeechCollection(); } /* diff --git a/src/main/java/org/texttechnologylab/project/gruppe_05_1/util/PPRUtils.java b/src/main/java/org/texttechnologylab/project/gruppe_05_1/util/PPRUtils.java index b506a78..3da9b02 100644 --- a/src/main/java/org/texttechnologylab/project/gruppe_05_1/util/PPRUtils.java +++ b/src/main/java/org/texttechnologylab/project/gruppe_05_1/util/PPRUtils.java @@ -36,23 +36,6 @@ public abstract class PPRUtils { - /** - * Prüfe, ob die Collections existieren. Falls nicht: erzeuge sie und lege Indizes an - */ - public static void ensureCollectionExist() { - - Set existingCollectionNames = MongoDBHandler.getCollectionNames(); - - if (!existingCollectionNames.contains(MongoPprUtils.SPEAKER_COLLECTION_NAME)) { - MongoPprUtils.createSpeakerCollection(); - } - - if (!existingCollectionNames.contains(MongoPprUtils.SPEECH_COLLECTION_NAME)) { - MongoPprUtils.createSpeechCollection(); - } - - } - /** * Alle Informationen lesen... * - Parlamentarier diff --git a/src/main/resources/.DS_Store b/src/main/resources/.DS_Store deleted file mode 100644 index 558c0a3d69821fe11caa8220681a138569846b17..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKu};G<5PfbdR0Km{V$8x${~%Q13%W5uT2v4Ti7L8hWk7rhJNy72z>N3-eu8(l z)y9NUSP(*Y()l^(JNx|7=4=3FIG+rF2tc1I*x6+BiO9HUOFG`NO*AscU^Fg^s^zDK zzsrEUyDkb$kmb|X`|~43nWkA@X7rUG!p_~p)8R6%C?{WxR_Co*BUH$7hAW(6#{LW? zuBn&QTt>o7_TRYH+L&gJx6e2CyN@=6GAAuSZ+LWwTa_$!8S;k3uzFY#DAbm1^|WzOSQHvWcU{0Yd! z)*U8ssI4-f41^4H-D4#6|7i97KMc|*Wk4DDR}7e5I!cG!lCQ0;%}K3|sV7ts@v9v! kLfEmTn7&eq_o!BAkEKIQJk}0rq3A)t(x8np@T&~G0}@VjjQ{`u