Correction to enable local database use, slight rework

This commit is contained in:
vysitor 2025-03-09 23:05:35 +01:00
parent b72f1b9d1e
commit fbf3c5b68d
9 changed files with 91 additions and 120 deletions

BIN
.DS_Store vendored

Binary file not shown.

BIN
src/.DS_Store vendored

Binary file not shown.

BIN
src/main/.DS_Store vendored

Binary file not shown.

Binary file not shown.

View file

@ -1,26 +1,16 @@
package org.texttechnologylab.project.gruppe_05_1;
import com.mongodb.client.MongoDatabase;
import org.texttechnologylab.project.gruppe_05_1.database.*;
import org.texttechnologylab.project.gruppe_05_1.domain.mdb.Mdb;
import org.texttechnologylab.project.gruppe_05_1.domain.mdb.MdbDocument;
import org.texttechnologylab.project.gruppe_05_1.nlp.NlpUtils;
import org.texttechnologylab.project.gruppe_05_1.nlp.XmiExtractor;
import org.texttechnologylab.project.gruppe_05_1.rest.RESTHandler;
import org.texttechnologylab.project.gruppe_05_1.util.Logger;
import org.texttechnologylab.project.gruppe_05_1.util.PPRUtils;
import org.texttechnologylab.project.gruppe_05_1.util.PropertiesUtils;
import org.texttechnologylab.project.gruppe_05_1.util.XmlUtils;
import org.texttechnologylab.project.gruppe_05_1.xml.FileObjectFactory;
import org.texttechnologylab.project.gruppe_05_1.xml.speeches.SpeechParser;
import org.w3c.dom.Element;
import org.xml.sax.SAXException;
import javax.xml.parsers.ParserConfigurationException;
import java.io.IOException;
import java.util.List;
import java.util.Properties;
import static java.lang.Boolean.FALSE;
import static java.lang.Boolean.TRUE;
@ -57,10 +47,10 @@ public class Main {
public static void main(String[] args) throws ParserConfigurationException, IOException, SAXException {
//TEST
MongoDBHandler mongoDBHandler = new MongoDBHandler();
SpeechIndexFactoryImpl speechIndexFactory = new SpeechIndexFactoryImpl();
if (MongoPprUtils.getSpeechCollection().countDocuments() != 0) {
if (mongoDBHandler.getDatabase().getCollection(MongoPprUtils.SPEECH_COLLECTION_NAME).countDocuments() != 0) {
System.out.println("Speeches werden nicht gelesen, da sie bereits in der Datenbank stehen");
}
else {
@ -79,8 +69,7 @@ public class Main {
System.out.println("SESSIONCOUNT: " + speechIndex.getSessions().size());
System.out.println("AGENDAITEMCOUNT: " + speechIndex.getAgendaItems().size());
MongoDBHandler mongoDBHandler = new MongoDBHandler();
mongoDBHandler.deleteAllDocuments(); // Clear the DB
mongoDBHandler.deleteSpeechRelatedDocuments(); // Clear speeches, sessions, agendas (history)
Logger.pink("Adding Sessions to DB...");
mongoDBHandler.insertSessions(speechIndex.getSessions());
@ -90,13 +79,8 @@ public class Main {
Logger.pink("Adding Speeches to DB...");
mongoDBHandler.insertSpeeches(speechIndex.getSpeeches());
mongoDBHandler.close(); // Close the connection to the DB
}
// Stellt fest, dass alle nötigen Datenbank-Collections existieren
PPRUtils.ensureCollectionExist();
// Alle Informationen (Parlamentarier, Reden, Kommentare etc.) lesen und in die Mongo-DB einfügen, falls diese noch nicht vorhanden sind.
PPRUtils.parlamentExplorerInit(xmlFactory, mongoFactory);
@ -105,6 +89,7 @@ public class Main {
// NLP-Verarbeitung - TODO
// NlpUtils.importXmiData();
mongoDBHandler.close(); // Close the connection to the DB
RESTHandler restHandler = new RESTHandler();
restHandler.startJavalin();

View file

@ -8,16 +8,12 @@ import com.mongodb.client.MongoClients;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.*;
import exceptions.AgendaItemNotFoundException;
import exceptions.MemberNotFoundException;
import exceptions.ServerErrorException;
import exceptions.SessionNotFoundException;
import org.bson.Document;
import org.bson.conversions.Bson;
import org.bson.types.ObjectId;
import org.texttechnologylab.DockerUnifiedUIMAInterface.connection.mongodb.MongoDBConfig;
import org.texttechnologylab.project.gruppe_05_1.database.domainimp.speeches.AgendaItem_MongoDB_Impl;
import org.texttechnologylab.project.gruppe_05_1.database.domainimp.speeches.MemberOfParliament_MongoDB_Impl;
import org.texttechnologylab.project.gruppe_05_1.database.domainimp.speeches.Session_MongoDB_Impl;
import org.texttechnologylab.project.gruppe_05_1.database.domainimp.speeches.Speech_MongoDB_Impl;
import org.texttechnologylab.project.gruppe_05_1.util.Logger;
@ -25,7 +21,6 @@ import org.texttechnologylab.project.gruppe_05_1.util.PropertiesUtils;
import org.texttechnologylab.project.gruppe_05_1.xml.speeches.Impls.*;
import org.texttechnologylab.project.gruppe_05_1.xml.speeches.Interfaces.*;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.TimeUnit;
@ -49,8 +44,8 @@ public class MongoDBHandler {
private static String collection;
private static String databaseName;
private MongoCollection<Document> speakerCollection;
private MongoCollection<Document> speechesCollection;
private MongoCollection<Document> sessionsCollection;
private MongoCollection<Document> agendaItemsCollection;
private MongoCollection<Document> historyCollection;
@ -71,27 +66,41 @@ public class MongoDBHandler {
collection = mongoProperties.getProperty("remote_collection");
databaseName = mongoProperties.getProperty("remote_database");
MongoCredential credential = MongoCredential
.createCredential(
user,
databaseName,
password.toCharArray());
// URI für lokale Datenbank oder für eine Datenbank auf dem Server
String uri;
if ( (localServer != null) && (! localServer.isBlank())) {
uri = localServer;
mongoClient = MongoClients.create(uri);
MongoClientSettings settings = MongoClientSettings.builder()
.credential(credential)
.timeout(180, TimeUnit.HOURS) // needs increased timeout for the bulk speech inserts
.applyToClusterSettings(builder ->
builder.hosts(List.of(new ServerAddress(remoteServer, Integer.parseInt(port)))))
.build();
// Connect
database = mongoClient.getDatabase(databaseName);
} else {
MongoCredential credential = MongoCredential
.createCredential(
user,
databaseName,
password.toCharArray());
mongoClient = MongoClients.create(settings);
database = mongoClient.getDatabase(databaseName);
speechesCollection = database.getCollection("speech");
sessionsCollection = database.getCollection("sessions");
agendaItemsCollection = database.getCollection("agendaItems");
historyCollection = database.getCollection("history");
MongoClientSettings settings = MongoClientSettings.builder()
.credential(credential)
.timeout(180, TimeUnit.HOURS) // needs increased timeout for the bulk speech inserts
.applyToClusterSettings(builder ->
builder.hosts(List.of(new ServerAddress(remoteServer, Integer.parseInt(port)))))
.build();
mongoClient = MongoClients.create(settings);
database = mongoClient.getDatabase(databaseName);
}
speakerCollection = database.getCollection(MongoPprUtils.SPEAKER_COLLECTION_NAME);
speechesCollection = database.getCollection(MongoPprUtils.SPEECH_COLLECTION_NAME);
sessionsCollection = database.getCollection(MongoPprUtils.SESSION_COLLECTION_NAME);
agendaItemsCollection = database.getCollection(MongoPprUtils.AGENDA_ITEMS_COLLECTION_NAME);
historyCollection = database.getCollection(MongoPprUtils.HISTORY_COLLECTION_NAME);
createIndicesForSpeakerCollection();
createIndicesForSpeechCollection();
Logger.info("Connected to MongoDB database: " + databaseName);
}
public MongoDatabase getDatabase() {
@ -105,18 +114,22 @@ public class MongoDBHandler {
*/
static public MongoDatabase getMongoDatabase() {
if (mongoDatabase == null) {
Properties mongoProperties = PropertiesUtils.readPropertiesFromResource(propertiesFileName);
// Zugangsdaten
localServer = mongoProperties.getProperty("localserver");
remoteServer = mongoProperties.getProperty("remote_host");
user = mongoProperties.getProperty("remote_user");
password = mongoProperties.getProperty("remote_password");
port = mongoProperties.getProperty("remote_port");
collection = mongoProperties.getProperty("remote_collection");
databaseName = mongoProperties.getProperty("remote_database");
if (mongoDatabase != null) {
return mongoDatabase;
}
Properties mongoProperties = PropertiesUtils.readPropertiesFromResource(propertiesFileName);
// Zugangsdaten
localServer = mongoProperties.getProperty("localserver");
remoteServer = mongoProperties.getProperty("remote_host");
user = mongoProperties.getProperty("remote_user");
password = mongoProperties.getProperty("remote_password");
port = mongoProperties.getProperty("remote_port");
collection = mongoProperties.getProperty("remote_collection");
databaseName = mongoProperties.getProperty("remote_database");
// MongoDBClient erzeugen
// String uri = mongoServer + "://" + mongoUser + ":" + mongoPassword + "@" + mongoNeetwork; // cluster, network, user...
@ -158,9 +171,9 @@ public class MongoDBHandler {
*
* @return List<String> with the names of all collections
*/
static public Set<String> getCollectionNames() {
// return getMongoDatabase().listCollectionNames().into(new ArrayList<>());
return getMongoDatabase().listCollectionNames().into(new HashSet<>());
public Set<String> getCollectionNames() {
// return getDatabase().listCollectionNames().into(new ArrayList<>());
return getDatabase().listCollectionNames().into(new HashSet<>());
}
/**
@ -168,8 +181,8 @@ public class MongoDBHandler {
* @param name Name of collection to check for existance
* @return does the collection exist
*/
static public boolean collectionExists(String name) {
return getMongoDatabase().listCollectionNames().into(new ArrayList<>()).contains(name);
public boolean collectionExists(String name) {
return getDatabase().listCollectionNames().into(new ArrayList<>()).contains(name);
}
@ -193,8 +206,8 @@ public class MongoDBHandler {
}
}
static public void createCollectionIfNotExist(String collectionName) {
createCollectionIfNotExist(getMongoDatabase(), collectionName);
public void createCollectionIfNotExist(String collectionName) {
createCollectionIfNotExist(getDatabase(), collectionName);
}
@ -211,8 +224,8 @@ public class MongoDBHandler {
}
}
static public void createCollection(String collectionName) {
createCollection(getMongoDatabase(), collectionName);
public void createCollection(String collectionName) {
createCollection(getDatabase(), collectionName);
}
@ -261,8 +274,23 @@ public class MongoDBHandler {
}
}
static public void createOrTrancateCollection(String collectionName) {
createOrTrancateCollection(getMongoDatabase(), collectionName);
public void createIndicesForSpeakerCollection() {
if (speakerCollection.listIndexes().into(new ArrayList<>()).size() == 1) {
MongoDBHandler.createIndexForCollection(speakerCollection,"name", true);
MongoDBHandler.createIndexForCollection(speakerCollection,"firstName", true);
MongoDBHandler.createIndexForCollection(speakerCollection,"party", true);
}
}
public void createIndicesForSpeechCollection() {
if (speechesCollection.listIndexes().into(new ArrayList<>()).size() == 1) {
MongoDBHandler.createIndexForCollection(speechesCollection, "speakerId", true);
MongoDBHandler.createIndexForCollection(speechesCollection, "speechKey", true);
}
}
public void createOrTrancateCollection(String collectionName) {
createOrTrancateCollection(database, collectionName);
}
/**
@ -648,7 +676,7 @@ public class MongoDBHandler {
);
}
public void deleteAllDocuments() {
public void deleteSpeechRelatedDocuments() {
speechesCollection.deleteMany(new Document());
sessionsCollection.deleteMany(new Document());
agendaItemsCollection.deleteMany(new Document());

View file

@ -2,7 +2,6 @@ package org.texttechnologylab.project.gruppe_05_1.database;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoCursor;
import lombok.extern.slf4j.Slf4j;
import org.bson.Document;
import org.texttechnologylab.project.gruppe_05_1.domain.html.Parlamentarier;
import org.texttechnologylab.project.gruppe_05_1.domain.html.ParlamentarierDetails;
@ -16,7 +15,6 @@ import java.util.*;
* Diese Klasse beinhaltet Mongo-Utilities, welche spezifisch für die PPR-Datenstrukturen sind.
* Mongo-Utilities genereller Natur stehen in der Klasse MongoDBHandler.
*/
@Slf4j
public class MongoPprUtils {
/*
@ -25,6 +23,10 @@ public class MongoPprUtils {
*/
public static final String SPEAKER_COLLECTION_NAME = "speaker";
public static final String SPEECH_COLLECTION_NAME = "speech";
public static final String SESSION_COLLECTION_NAME = "sessions";
public static final String AGENDA_ITEMS_COLLECTION_NAME = "agendaItems";
public static final String HISTORY_COLLECTION_NAME = "history";
public static final String PICTURES_COLLECTION_NAME = "pictures";
public static final String COMMENT_COLLECTION_NAME = "comment";
@ -43,53 +45,26 @@ public class MongoPprUtils {
return speechCollection;
}
public static MongoCollection<Document> getPicturesCollection() {
if (picturesCollection == null) picturesCollection = MongoDBHandler.getMongoDatabase().getCollection(PICTURES_COLLECTION_NAME);
return picturesCollection;
}
public static MongoCollection<Document> getCommentCollection() {
if (commentCollection == null) commentCollection = MongoDBHandler.getMongoDatabase().getCollection(COMMENT_COLLECTION_NAME);
return commentCollection;
}
/**
* Create the Speaker Collection and useful indices for it
*/
public static void createSpeakerCollection() {
MongoDBHandler.createCollection(MongoPprUtils.SPEAKER_COLLECTION_NAME);
MongoDBHandler.createIndexForCollection(getSpeakerCollection(), Arrays.asList("name", "firstName", "party"), true);
public static void createIndexForSpeakerCollection() {
// MongoDBHandler.createIndexForCollection(getSpeakerCollection(), Arrays.asList("name", "firstName", "party"), true);
MongoDBHandler.createIndexForCollection(getSpeakerCollection(),"name", true);
MongoDBHandler.createIndexForCollection(getSpeakerCollection(),"firstName", true);
MongoDBHandler.createIndexForCollection(getSpeakerCollection(),"party", true);
}
/**
* Create the Speech Collection and useful indices for it
*/
public static void createSpeechCollection() {
MongoDBHandler.createCollection(MongoPprUtils.SPEECH_COLLECTION_NAME);
MongoDBHandler.createIndexForCollection(getSpeechCollection(), "speaker", true);
public static void createIndexForSpeechCollection() {
MongoDBHandler.createIndexForCollection(getSpeechCollection(), "speakerId", true);
MongoDBHandler.createIndexForCollection(getSpeechCollection(), "speechKey", true);
}
/**
* Create the Comment Collection and useful indices for it
*/
public static void createCommentCollection() {
MongoDBHandler.createCollection(MongoPprUtils.COMMENT_COLLECTION_NAME);
MongoDBHandler.createIndexForCollection(getCommentCollection(), Arrays.asList("speaker", "speech"), true);
}
/**
* Create the Picture Collection and useful indices for it
*/
public static void createPictureCollection() {
MongoDBHandler.createCollection(MongoPprUtils.PICTURES_COLLECTION_NAME);
// TODO: für welche Felder sollen Indizes gebaut werden?
// MongoDBHandler.createIndexForCollection(getPicturesCollection(), Arrays.asList("field_1", "field_2"), true);
}
/**
* Truncate the Speaker Collection.
* Note that it is quicker (and saves space) to drop and re-create rather than removing all documents using "remove({})"
@ -97,7 +72,7 @@ public class MongoPprUtils {
public static void truncateSpeakerCollection() {
getSpeakerCollection().drop();
createSpeechCollection();
createIndexForSpeechCollection();
}
/*

View file

@ -36,23 +36,6 @@ public abstract class PPRUtils {
/**
* Prüfe, ob die Collections existieren. Falls nicht: erzeuge sie und lege Indizes an
*/
public static void ensureCollectionExist() {
Set<String> existingCollectionNames = MongoDBHandler.getCollectionNames();
if (!existingCollectionNames.contains(MongoPprUtils.SPEAKER_COLLECTION_NAME)) {
MongoPprUtils.createSpeakerCollection();
}
if (!existingCollectionNames.contains(MongoPprUtils.SPEECH_COLLECTION_NAME)) {
MongoPprUtils.createSpeechCollection();
}
}
/**
* Alle Informationen lesen...
* - Parlamentarier

Binary file not shown.