Correction to enable local database use, slight rework
This commit is contained in:
parent
b72f1b9d1e
commit
fbf3c5b68d
9 changed files with 91 additions and 120 deletions
BIN
.DS_Store
vendored
BIN
.DS_Store
vendored
Binary file not shown.
BIN
src/.DS_Store
vendored
BIN
src/.DS_Store
vendored
Binary file not shown.
BIN
src/main/.DS_Store
vendored
BIN
src/main/.DS_Store
vendored
Binary file not shown.
BIN
src/main/java/.DS_Store
vendored
BIN
src/main/java/.DS_Store
vendored
Binary file not shown.
|
@ -1,26 +1,16 @@
|
|||
package org.texttechnologylab.project.gruppe_05_1;
|
||||
|
||||
|
||||
import com.mongodb.client.MongoDatabase;
|
||||
import org.texttechnologylab.project.gruppe_05_1.database.*;
|
||||
import org.texttechnologylab.project.gruppe_05_1.domain.mdb.Mdb;
|
||||
import org.texttechnologylab.project.gruppe_05_1.domain.mdb.MdbDocument;
|
||||
import org.texttechnologylab.project.gruppe_05_1.nlp.NlpUtils;
|
||||
import org.texttechnologylab.project.gruppe_05_1.nlp.XmiExtractor;
|
||||
import org.texttechnologylab.project.gruppe_05_1.rest.RESTHandler;
|
||||
import org.texttechnologylab.project.gruppe_05_1.util.Logger;
|
||||
import org.texttechnologylab.project.gruppe_05_1.util.PPRUtils;
|
||||
import org.texttechnologylab.project.gruppe_05_1.util.PropertiesUtils;
|
||||
import org.texttechnologylab.project.gruppe_05_1.util.XmlUtils;
|
||||
import org.texttechnologylab.project.gruppe_05_1.xml.FileObjectFactory;
|
||||
import org.texttechnologylab.project.gruppe_05_1.xml.speeches.SpeechParser;
|
||||
import org.w3c.dom.Element;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
import javax.xml.parsers.ParserConfigurationException;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import static java.lang.Boolean.FALSE;
|
||||
import static java.lang.Boolean.TRUE;
|
||||
|
@ -57,10 +47,10 @@ public class Main {
|
|||
public static void main(String[] args) throws ParserConfigurationException, IOException, SAXException {
|
||||
|
||||
//TEST
|
||||
|
||||
MongoDBHandler mongoDBHandler = new MongoDBHandler();
|
||||
|
||||
SpeechIndexFactoryImpl speechIndexFactory = new SpeechIndexFactoryImpl();
|
||||
if (MongoPprUtils.getSpeechCollection().countDocuments() != 0) {
|
||||
if (mongoDBHandler.getDatabase().getCollection(MongoPprUtils.SPEECH_COLLECTION_NAME).countDocuments() != 0) {
|
||||
System.out.println("Speeches werden nicht gelesen, da sie bereits in der Datenbank stehen");
|
||||
}
|
||||
else {
|
||||
|
@ -79,8 +69,7 @@ public class Main {
|
|||
System.out.println("SESSIONCOUNT: " + speechIndex.getSessions().size());
|
||||
System.out.println("AGENDAITEMCOUNT: " + speechIndex.getAgendaItems().size());
|
||||
|
||||
MongoDBHandler mongoDBHandler = new MongoDBHandler();
|
||||
mongoDBHandler.deleteAllDocuments(); // Clear the DB
|
||||
mongoDBHandler.deleteSpeechRelatedDocuments(); // Clear speeches, sessions, agendas (history)
|
||||
|
||||
Logger.pink("Adding Sessions to DB...");
|
||||
mongoDBHandler.insertSessions(speechIndex.getSessions());
|
||||
|
@ -90,13 +79,8 @@ public class Main {
|
|||
|
||||
Logger.pink("Adding Speeches to DB...");
|
||||
mongoDBHandler.insertSpeeches(speechIndex.getSpeeches());
|
||||
|
||||
mongoDBHandler.close(); // Close the connection to the DB
|
||||
}
|
||||
|
||||
// Stellt fest, dass alle nötigen Datenbank-Collections existieren
|
||||
PPRUtils.ensureCollectionExist();
|
||||
|
||||
// Alle Informationen (Parlamentarier, Reden, Kommentare etc.) lesen und in die Mongo-DB einfügen, falls diese noch nicht vorhanden sind.
|
||||
PPRUtils.parlamentExplorerInit(xmlFactory, mongoFactory);
|
||||
|
||||
|
@ -105,6 +89,7 @@ public class Main {
|
|||
// NLP-Verarbeitung - TODO
|
||||
// NlpUtils.importXmiData();
|
||||
|
||||
mongoDBHandler.close(); // Close the connection to the DB
|
||||
|
||||
RESTHandler restHandler = new RESTHandler();
|
||||
restHandler.startJavalin();
|
||||
|
|
|
@ -8,16 +8,12 @@ import com.mongodb.client.MongoClients;
|
|||
import com.mongodb.client.MongoCollection;
|
||||
import com.mongodb.client.MongoDatabase;
|
||||
import com.mongodb.client.model.*;
|
||||
import exceptions.AgendaItemNotFoundException;
|
||||
import exceptions.MemberNotFoundException;
|
||||
import exceptions.ServerErrorException;
|
||||
import exceptions.SessionNotFoundException;
|
||||
import org.bson.Document;
|
||||
import org.bson.conversions.Bson;
|
||||
import org.bson.types.ObjectId;
|
||||
import org.texttechnologylab.DockerUnifiedUIMAInterface.connection.mongodb.MongoDBConfig;
|
||||
import org.texttechnologylab.project.gruppe_05_1.database.domainimp.speeches.AgendaItem_MongoDB_Impl;
|
||||
import org.texttechnologylab.project.gruppe_05_1.database.domainimp.speeches.MemberOfParliament_MongoDB_Impl;
|
||||
import org.texttechnologylab.project.gruppe_05_1.database.domainimp.speeches.Session_MongoDB_Impl;
|
||||
import org.texttechnologylab.project.gruppe_05_1.database.domainimp.speeches.Speech_MongoDB_Impl;
|
||||
import org.texttechnologylab.project.gruppe_05_1.util.Logger;
|
||||
|
@ -25,7 +21,6 @@ import org.texttechnologylab.project.gruppe_05_1.util.PropertiesUtils;
|
|||
import org.texttechnologylab.project.gruppe_05_1.xml.speeches.Impls.*;
|
||||
import org.texttechnologylab.project.gruppe_05_1.xml.speeches.Interfaces.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
|
@ -49,8 +44,8 @@ public class MongoDBHandler {
|
|||
private static String collection;
|
||||
private static String databaseName;
|
||||
|
||||
private MongoCollection<Document> speakerCollection;
|
||||
private MongoCollection<Document> speechesCollection;
|
||||
|
||||
private MongoCollection<Document> sessionsCollection;
|
||||
private MongoCollection<Document> agendaItemsCollection;
|
||||
private MongoCollection<Document> historyCollection;
|
||||
|
@ -71,27 +66,41 @@ public class MongoDBHandler {
|
|||
collection = mongoProperties.getProperty("remote_collection");
|
||||
databaseName = mongoProperties.getProperty("remote_database");
|
||||
|
||||
MongoCredential credential = MongoCredential
|
||||
.createCredential(
|
||||
user,
|
||||
databaseName,
|
||||
password.toCharArray());
|
||||
// URI für lokale Datenbank oder für eine Datenbank auf dem Server
|
||||
String uri;
|
||||
if ( (localServer != null) && (! localServer.isBlank())) {
|
||||
uri = localServer;
|
||||
mongoClient = MongoClients.create(uri);
|
||||
|
||||
MongoClientSettings settings = MongoClientSettings.builder()
|
||||
.credential(credential)
|
||||
.timeout(180, TimeUnit.HOURS) // needs increased timeout for the bulk speech inserts
|
||||
.applyToClusterSettings(builder ->
|
||||
builder.hosts(List.of(new ServerAddress(remoteServer, Integer.parseInt(port)))))
|
||||
.build();
|
||||
// Connect
|
||||
database = mongoClient.getDatabase(databaseName);
|
||||
} else {
|
||||
MongoCredential credential = MongoCredential
|
||||
.createCredential(
|
||||
user,
|
||||
databaseName,
|
||||
password.toCharArray());
|
||||
|
||||
mongoClient = MongoClients.create(settings);
|
||||
database = mongoClient.getDatabase(databaseName);
|
||||
speechesCollection = database.getCollection("speech");
|
||||
sessionsCollection = database.getCollection("sessions");
|
||||
agendaItemsCollection = database.getCollection("agendaItems");
|
||||
historyCollection = database.getCollection("history");
|
||||
MongoClientSettings settings = MongoClientSettings.builder()
|
||||
.credential(credential)
|
||||
.timeout(180, TimeUnit.HOURS) // needs increased timeout for the bulk speech inserts
|
||||
.applyToClusterSettings(builder ->
|
||||
builder.hosts(List.of(new ServerAddress(remoteServer, Integer.parseInt(port)))))
|
||||
.build();
|
||||
|
||||
mongoClient = MongoClients.create(settings);
|
||||
database = mongoClient.getDatabase(databaseName);
|
||||
}
|
||||
|
||||
speakerCollection = database.getCollection(MongoPprUtils.SPEAKER_COLLECTION_NAME);
|
||||
speechesCollection = database.getCollection(MongoPprUtils.SPEECH_COLLECTION_NAME);
|
||||
sessionsCollection = database.getCollection(MongoPprUtils.SESSION_COLLECTION_NAME);
|
||||
agendaItemsCollection = database.getCollection(MongoPprUtils.AGENDA_ITEMS_COLLECTION_NAME);
|
||||
historyCollection = database.getCollection(MongoPprUtils.HISTORY_COLLECTION_NAME);
|
||||
|
||||
createIndicesForSpeakerCollection();
|
||||
createIndicesForSpeechCollection();
|
||||
Logger.info("Connected to MongoDB database: " + databaseName);
|
||||
|
||||
}
|
||||
|
||||
public MongoDatabase getDatabase() {
|
||||
|
@ -105,18 +114,22 @@ public class MongoDBHandler {
|
|||
*/
|
||||
static public MongoDatabase getMongoDatabase() {
|
||||
|
||||
if (mongoDatabase == null) {
|
||||
Properties mongoProperties = PropertiesUtils.readPropertiesFromResource(propertiesFileName);
|
||||
// Zugangsdaten
|
||||
localServer = mongoProperties.getProperty("localserver");
|
||||
remoteServer = mongoProperties.getProperty("remote_host");
|
||||
user = mongoProperties.getProperty("remote_user");
|
||||
password = mongoProperties.getProperty("remote_password");
|
||||
port = mongoProperties.getProperty("remote_port");
|
||||
collection = mongoProperties.getProperty("remote_collection");
|
||||
databaseName = mongoProperties.getProperty("remote_database");
|
||||
if (mongoDatabase != null) {
|
||||
return mongoDatabase;
|
||||
}
|
||||
|
||||
|
||||
Properties mongoProperties = PropertiesUtils.readPropertiesFromResource(propertiesFileName);
|
||||
// Zugangsdaten
|
||||
localServer = mongoProperties.getProperty("localserver");
|
||||
remoteServer = mongoProperties.getProperty("remote_host");
|
||||
user = mongoProperties.getProperty("remote_user");
|
||||
password = mongoProperties.getProperty("remote_password");
|
||||
port = mongoProperties.getProperty("remote_port");
|
||||
collection = mongoProperties.getProperty("remote_collection");
|
||||
databaseName = mongoProperties.getProperty("remote_database");
|
||||
|
||||
|
||||
// MongoDBClient erzeugen
|
||||
// String uri = mongoServer + "://" + mongoUser + ":" + mongoPassword + "@" + mongoNeetwork; // cluster, network, user...
|
||||
|
||||
|
@ -158,9 +171,9 @@ public class MongoDBHandler {
|
|||
*
|
||||
* @return List<String> with the names of all collections
|
||||
*/
|
||||
static public Set<String> getCollectionNames() {
|
||||
// return getMongoDatabase().listCollectionNames().into(new ArrayList<>());
|
||||
return getMongoDatabase().listCollectionNames().into(new HashSet<>());
|
||||
public Set<String> getCollectionNames() {
|
||||
// return getDatabase().listCollectionNames().into(new ArrayList<>());
|
||||
return getDatabase().listCollectionNames().into(new HashSet<>());
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -168,8 +181,8 @@ public class MongoDBHandler {
|
|||
* @param name Name of collection to check for existance
|
||||
* @return does the collection exist
|
||||
*/
|
||||
static public boolean collectionExists(String name) {
|
||||
return getMongoDatabase().listCollectionNames().into(new ArrayList<>()).contains(name);
|
||||
public boolean collectionExists(String name) {
|
||||
return getDatabase().listCollectionNames().into(new ArrayList<>()).contains(name);
|
||||
}
|
||||
|
||||
|
||||
|
@ -193,8 +206,8 @@ public class MongoDBHandler {
|
|||
}
|
||||
}
|
||||
|
||||
static public void createCollectionIfNotExist(String collectionName) {
|
||||
createCollectionIfNotExist(getMongoDatabase(), collectionName);
|
||||
public void createCollectionIfNotExist(String collectionName) {
|
||||
createCollectionIfNotExist(getDatabase(), collectionName);
|
||||
}
|
||||
|
||||
|
||||
|
@ -211,8 +224,8 @@ public class MongoDBHandler {
|
|||
}
|
||||
}
|
||||
|
||||
static public void createCollection(String collectionName) {
|
||||
createCollection(getMongoDatabase(), collectionName);
|
||||
public void createCollection(String collectionName) {
|
||||
createCollection(getDatabase(), collectionName);
|
||||
}
|
||||
|
||||
|
||||
|
@ -261,8 +274,23 @@ public class MongoDBHandler {
|
|||
}
|
||||
}
|
||||
|
||||
static public void createOrTrancateCollection(String collectionName) {
|
||||
createOrTrancateCollection(getMongoDatabase(), collectionName);
|
||||
public void createIndicesForSpeakerCollection() {
|
||||
if (speakerCollection.listIndexes().into(new ArrayList<>()).size() == 1) {
|
||||
MongoDBHandler.createIndexForCollection(speakerCollection,"name", true);
|
||||
MongoDBHandler.createIndexForCollection(speakerCollection,"firstName", true);
|
||||
MongoDBHandler.createIndexForCollection(speakerCollection,"party", true);
|
||||
}
|
||||
|
||||
}
|
||||
public void createIndicesForSpeechCollection() {
|
||||
if (speechesCollection.listIndexes().into(new ArrayList<>()).size() == 1) {
|
||||
MongoDBHandler.createIndexForCollection(speechesCollection, "speakerId", true);
|
||||
MongoDBHandler.createIndexForCollection(speechesCollection, "speechKey", true);
|
||||
}
|
||||
}
|
||||
|
||||
public void createOrTrancateCollection(String collectionName) {
|
||||
createOrTrancateCollection(database, collectionName);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -648,7 +676,7 @@ public class MongoDBHandler {
|
|||
);
|
||||
}
|
||||
|
||||
public void deleteAllDocuments() {
|
||||
public void deleteSpeechRelatedDocuments() {
|
||||
speechesCollection.deleteMany(new Document());
|
||||
sessionsCollection.deleteMany(new Document());
|
||||
agendaItemsCollection.deleteMany(new Document());
|
||||
|
|
|
@ -2,7 +2,6 @@ package org.texttechnologylab.project.gruppe_05_1.database;
|
|||
|
||||
import com.mongodb.client.MongoCollection;
|
||||
import com.mongodb.client.MongoCursor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.bson.Document;
|
||||
import org.texttechnologylab.project.gruppe_05_1.domain.html.Parlamentarier;
|
||||
import org.texttechnologylab.project.gruppe_05_1.domain.html.ParlamentarierDetails;
|
||||
|
@ -16,7 +15,6 @@ import java.util.*;
|
|||
* Diese Klasse beinhaltet Mongo-Utilities, welche spezifisch für die PPR-Datenstrukturen sind.
|
||||
* Mongo-Utilities genereller Natur stehen in der Klasse MongoDBHandler.
|
||||
*/
|
||||
@Slf4j
|
||||
public class MongoPprUtils {
|
||||
|
||||
/*
|
||||
|
@ -25,6 +23,10 @@ public class MongoPprUtils {
|
|||
*/
|
||||
public static final String SPEAKER_COLLECTION_NAME = "speaker";
|
||||
public static final String SPEECH_COLLECTION_NAME = "speech";
|
||||
|
||||
public static final String SESSION_COLLECTION_NAME = "sessions";
|
||||
public static final String AGENDA_ITEMS_COLLECTION_NAME = "agendaItems";
|
||||
public static final String HISTORY_COLLECTION_NAME = "history";
|
||||
public static final String PICTURES_COLLECTION_NAME = "pictures";
|
||||
public static final String COMMENT_COLLECTION_NAME = "comment";
|
||||
|
||||
|
@ -43,53 +45,26 @@ public class MongoPprUtils {
|
|||
return speechCollection;
|
||||
}
|
||||
|
||||
public static MongoCollection<Document> getPicturesCollection() {
|
||||
if (picturesCollection == null) picturesCollection = MongoDBHandler.getMongoDatabase().getCollection(PICTURES_COLLECTION_NAME);
|
||||
return picturesCollection;
|
||||
}
|
||||
|
||||
public static MongoCollection<Document> getCommentCollection() {
|
||||
if (commentCollection == null) commentCollection = MongoDBHandler.getMongoDatabase().getCollection(COMMENT_COLLECTION_NAME);
|
||||
return commentCollection;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create the Speaker Collection and useful indices for it
|
||||
*/
|
||||
public static void createSpeakerCollection() {
|
||||
MongoDBHandler.createCollection(MongoPprUtils.SPEAKER_COLLECTION_NAME);
|
||||
|
||||
MongoDBHandler.createIndexForCollection(getSpeakerCollection(), Arrays.asList("name", "firstName", "party"), true);
|
||||
public static void createIndexForSpeakerCollection() {
|
||||
// MongoDBHandler.createIndexForCollection(getSpeakerCollection(), Arrays.asList("name", "firstName", "party"), true);
|
||||
MongoDBHandler.createIndexForCollection(getSpeakerCollection(),"name", true);
|
||||
MongoDBHandler.createIndexForCollection(getSpeakerCollection(),"firstName", true);
|
||||
MongoDBHandler.createIndexForCollection(getSpeakerCollection(),"party", true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create the Speech Collection and useful indices for it
|
||||
*/
|
||||
public static void createSpeechCollection() {
|
||||
MongoDBHandler.createCollection(MongoPprUtils.SPEECH_COLLECTION_NAME);
|
||||
MongoDBHandler.createIndexForCollection(getSpeechCollection(), "speaker", true);
|
||||
public static void createIndexForSpeechCollection() {
|
||||
MongoDBHandler.createIndexForCollection(getSpeechCollection(), "speakerId", true);
|
||||
MongoDBHandler.createIndexForCollection(getSpeechCollection(), "speechKey", true);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Create the Comment Collection and useful indices for it
|
||||
*/
|
||||
public static void createCommentCollection() {
|
||||
MongoDBHandler.createCollection(MongoPprUtils.COMMENT_COLLECTION_NAME);
|
||||
MongoDBHandler.createIndexForCollection(getCommentCollection(), Arrays.asList("speaker", "speech"), true);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Create the Picture Collection and useful indices for it
|
||||
*/
|
||||
public static void createPictureCollection() {
|
||||
MongoDBHandler.createCollection(MongoPprUtils.PICTURES_COLLECTION_NAME);
|
||||
|
||||
// TODO: für welche Felder sollen Indizes gebaut werden?
|
||||
// MongoDBHandler.createIndexForCollection(getPicturesCollection(), Arrays.asList("field_1", "field_2"), true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncate the Speaker Collection.
|
||||
* Note that it is quicker (and saves space) to drop and re-create rather than removing all documents using "remove({})"
|
||||
|
@ -97,7 +72,7 @@ public class MongoPprUtils {
|
|||
public static void truncateSpeakerCollection() {
|
||||
|
||||
getSpeakerCollection().drop();
|
||||
createSpeechCollection();
|
||||
createIndexForSpeechCollection();
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
|
@ -36,23 +36,6 @@ public abstract class PPRUtils {
|
|||
|
||||
|
||||
|
||||
/**
|
||||
* Prüfe, ob die Collections existieren. Falls nicht: erzeuge sie und lege Indizes an
|
||||
*/
|
||||
public static void ensureCollectionExist() {
|
||||
|
||||
Set<String> existingCollectionNames = MongoDBHandler.getCollectionNames();
|
||||
|
||||
if (!existingCollectionNames.contains(MongoPprUtils.SPEAKER_COLLECTION_NAME)) {
|
||||
MongoPprUtils.createSpeakerCollection();
|
||||
}
|
||||
|
||||
if (!existingCollectionNames.contains(MongoPprUtils.SPEECH_COLLECTION_NAME)) {
|
||||
MongoPprUtils.createSpeechCollection();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Alle Informationen lesen...
|
||||
* - Parlamentarier
|
||||
|
|
BIN
src/main/resources/.DS_Store
vendored
BIN
src/main/resources/.DS_Store
vendored
Binary file not shown.
Loading…
Add table
Add a link
Reference in a new issue