Jonas_Jones/multimodal_parliament_explorer_05_1
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'main' of https://ppr.gitlab.texttechnologylab.org/s1188354/multimodal_parliament_explorer_05_1

Browse source
This commit is contained in:
Jonas Werner 2025-03-22 13:22:19 +01:00
commit 038cc460bc
7 changed files with 138 additions and 73 deletions
Download patch file Download diff file Expand all files Collapse all files

10
src/main/java/org/texttechnologylab/project/gruppe_05_1/Main.java
View file

@ -28,6 +28,8 @@ public class Main {
public static boolean FORCE_UPLOAD_MEMBERS;
public static boolean FORCE_UPLOAD_SPEECHES;
public static boolean ONLY_RUN_WEB;
public static boolean REBUILD_METADATA;
public static boolean DEBUG_LOGGING;
private static final FileObjectFactory xmlFactory = FileObjectFactory.getFactory();
private static final MongoObjectFactory mongoFactory = MongoObjectFactory.getFactory();
@ -43,6 +45,7 @@ public class Main {
FORCE_UPLOAD_MEMBERS = Arrays.asList(args).contains("forceUploadMembers");
FORCE_UPLOAD_SPEECHES = Arrays.asList(args).contains("forceUploadSpeeches");
ONLY_RUN_WEB = Arrays.asList(args).contains("onlyRunWeb");
REBUILD_METADATA = Arrays.asList(args).contains("rebuildMetadata");
DEBUG_LOGGING = Arrays.asList(args).contains("debugLogging");
System.out.println("Starting Multimodal Parliament Explorer...");
@ -52,6 +55,7 @@ public class Main {
System.out.println(" - Force Upload Members: " + FORCE_UPLOAD_MEMBERS);
System.out.println(" - Force Upload Speeches: " + FORCE_UPLOAD_SPEECHES);
System.out.println(" - Only Run javalin Web Server: " + ONLY_RUN_WEB);
System.out.println(" - Rebuild Metadata: " + REBUILD_METADATA);
System.out.println(" - Debug Logging: " + DEBUG_LOGGING);
System.out.println("--------------------------------------------o");
@ -64,6 +68,12 @@ public class Main {
MongoDBHandler mongoDBHandler = new MongoDBHandler();
if (REBUILD_METADATA) {
Logger.info("Rebuilding Metadata...");
MongoPprUtils.rebuildMetadata();
System.exit(0);
}
SpeechIndexFactoryImpl speechIndexFactory = new SpeechIndexFactoryImpl();
if ((mongoDBHandler.getDatabase().getCollection(MongoPprUtils.SPEECH_COLLECTION_NAME).countDocuments() != 0) && !FORCE_UPLOAD_SPEECHES) {
Logger.info("Skipping Speech parsing and DB insertion as they are already present...");

76
src/main/java/org/texttechnologylab/project/gruppe_05_1/database/MongoPprUtils.java
View file

@ -3,6 +3,9 @@ package org.texttechnologylab.project.gruppe_05_1.database;
import com.mongodb.client.FindIterable;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoCursor;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.Accumulators;
import com.mongodb.client.model.Aggregates;
import com.mongodb.client.model.Filters;
import com.mongodb.client.model.Projections;
import io.javalin.http.Context;
@ -45,6 +48,7 @@ public class MongoPprUtils {
public static final String HISTORY_COLLECTION_NAME = "history";
public static final String PICTURES_COLLECTION_NAME = "pictures";
public static final String COMMENT_COLLECTION_NAME = "comment";
public static final String METADATA_COLLECTION_NAME = "metadata";
private static MongoCollection<Document> speakerCollection = null;
private static MongoCollection<Document> speechCollection = null;
@ -52,6 +56,7 @@ public class MongoPprUtils {
private static MongoCollection<Document> agendaItemsCollection = null;
private static MongoCollection<Document> picturesCollection = null;
private static MongoCollection<Document> commentCollection = null;
private static MongoCollection<Document> metadataCollection = null;
public static MongoCollection<Document> getSpeakerCollection() {
if (speakerCollection == null) speakerCollection = MongoDBHandler.getMongoDatabase().getCollection(SPEAKER_COLLECTION_NAME);
@ -78,6 +83,11 @@ public class MongoPprUtils {
return picturesCollection;
}
public static MongoCollection<Document> getMetadataCollection() {
if (metadataCollection == null) metadataCollection = MongoDBHandler.getMongoDatabase().getCollection(METADATA_COLLECTION_NAME);
return metadataCollection;
}
/**
* Create the Speaker Collection and useful indices for it
*/
@ -626,10 +636,76 @@ public class MongoPprUtils {
// getMemberPhoto
/**
* Liefert das Bild eines Abgeordneten zurück
* @param id
* @return Base64-encoded Photo
*/
public static String getMemberPhoto(String id) {
Document doc = MongoDBHandler.findFirstDocumentInCollection(getPicturesCollection(), "memberId", id);
if (doc == null) {
return null;
} else return doc.getString("base64");
}
/**
* Aktualisiert (or erzeugt, falls nicht bereits vorhanden) diverse Metadaten:
* - Die Liste der Parteien/Fraktionen, wie sie im Speaker-Collection stehen
* - Die Liste der Parteien/Fraktionen, wie sie im Speech-Collection stehen (diese Listen sind recht unterschiedlich)
* - Topics nach NLP-Analyse der Reden
*/
public static void rebuildMetadata() {
MongoDatabase db = MongoDBHandler.getMongoDatabase();
Logger.info("Collecting Partei/Fraktion Information");
List<String> distinctPartiesOfSpeakers = getSpeakerCollection().distinct("party", String.class).into(new java.util.ArrayList<>());
List<String> distinctPartiesFromSpeeches = getSpeechCollection().distinct("fraction", String.class).into(new java.util.ArrayList<>());
Logger.info("Collecting Topics Information");
Set<String> topics = new HashSet<>();
// Aggregation pipeline
List<Bson> pipeline = List.of(
Aggregates.unwind("$analysisResults.topics"), // Unwind the "topics" array
Aggregates.project(Projections.fields(Projections.include("analysisResults.topics.topic"))), // Project only the "topic" field
Aggregates.group(null, Accumulators.addToSet("distinctTopics", "$analysisResults.topics.topic")) // Group to get distinct values
);
List<String> topicsList = null;
List<Document> results = getSpeechCollection().aggregate(pipeline).into(new java.util.ArrayList<>());
// Extract and print all distinct "topic" values
if (!results.isEmpty()) {
Document result = results.get(0); // Get the first (and only) document
List<String> distinctTopics = result.getList("distinctTopics", String.class);
topicsList = distinctTopics;
for (String topic : distinctTopics) {
System.out.println(topic);
}
} else {
System.out.println("No topics found.");
}
Logger.info("Updating Metadata Collection: begin");
MongoDBHandler.createCollection(db, METADATA_COLLECTION_NAME);
MongoCollection<Document> metadataCollection = getMetadataCollection();
Document filterPartiesFromSpeeches = new Document("type", "parties_from_speeches");
Document partiesDocFromSpeeches = MongoDBHandler.createDocument(false, Map.of("type", "parties_from_speeches",
"value", distinctPartiesFromSpeeches));
metadataCollection.replaceOne(filterPartiesFromSpeeches, partiesDocFromSpeeches, new com.mongodb.client.model.ReplaceOptions().upsert(true));
Document filterPartiesOfSpeakers = new Document("type", "parties_of_speakers");
Document partiesDocOfSpeakers = MongoDBHandler.createDocument(false, Map.of("type", "parties_of_speakers",
"value", distinctPartiesOfSpeakers));
metadataCollection.replaceOne(filterPartiesOfSpeakers, partiesDocOfSpeakers, new com.mongodb.client.model.ReplaceOptions().upsert(true));
Document filterTopics = new Document("type", "topics");
Document topicsDoc = MongoDBHandler.createDocument(false, Map.of("type", "topics",
"value", topicsList));
metadataCollection.replaceOne(filterTopics, topicsDoc, new com.mongodb.client.model.ReplaceOptions().upsert(true));
Logger.info("Updating Metadata Collection: end");
}
}

4
src/main/java/org/texttechnologylab/project/gruppe_05_1/domain/nlp/Sentiment.java
View file

@ -113,9 +113,9 @@ public class Sentiment {
doc.getInteger("begin"),
doc.getInteger("end"),
MongoDBHandler.getFieldAsDouble(doc, "score"),
MongoDBHandler.getFieldAsDouble(doc, "pos"),
MongoDBHandler.getFieldAsDouble(doc, "neg"),
MongoDBHandler.getFieldAsDouble(doc, "neu"),
MongoDBHandler.getFieldAsDouble(doc, "neg")
MongoDBHandler.getFieldAsDouble(doc, "pos")
));
}
return sentiments;

5
src/main/java/org/texttechnologylab/project/gruppe_05_1/rest/SpeechController.java
View file

@ -158,8 +158,9 @@ public class SpeechController {
// Der erste Sentiment gilt der gesamten Rede. Die weitere Sentiments entsprechen die Sätze.
List<Sentiment> sentiments = speech.getNlp().getSentiments();
if ((sentiments != null) && ! sentiments.isEmpty()) {
Sentiment overallSentiment = sentiments.get(0);
attributes.put("overallSentiment", overallSentiment);
List<Sentiment> overallSentiments = new ArrayList<>(sentiments);
attributes.put("overallSentiments", overallSentiments);
sentiments.remove(0);
// Sentiment-Icon

3
src/main/resources/templates/nlp.ftl
View file

@ -20,8 +20,9 @@
</div>
<div class="chart">
<#if overallSentiment??>
<#if overallSentiments??>
<h3>Sentiments Information (als Radar Chart)</h3>
<#assign sentiments = overallSentiments>
<#include "sentimentsRadarChart.ftl">
<#else>
<h3>Keine Sentiments Information für diese Rede verfügbar</h3>

4
src/main/resources/templates/posBarChart.ftl
View file

@ -1,12 +1,10 @@
<svg class="chart-svg" id="posBarchart"></svg>
<script>
// Define variables only in JavaScript
const barChartWidth = 1000;
const barChartHeight = 750;
const margin = { top: 20, right: 30, bottom: 50, left: 50 };
// Ensure posList exists before processing
var posData = [];
<#if posList?? && posList?size gt 0>
@ -46,8 +44,6 @@
.attr("height", d => Math.max(0, barChartHeight - margin.bottom - yScale(d.count))) // Prevents negative heights
.attr("fill", d => colorScale(d.pos));
console.log("Number of bars created:", bars.size());
// X Axis
svg.append("g")
.attr("transform", "translate(0," + (barChartHeight - margin.bottom) + ")")

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 2.3 KiB

After

Width:  |  Height:  |  Size: 2.1 KiB

Before After
Before After

109
src/main/resources/templates/sentimentsRadarChart.ftl
View file

@ -1,35 +1,39 @@
<svg class="chart-svg" id="sentimentsRadarChart"></svg>
<script src="https://d3js.org/d3.v6.min.js"></script>
<script>
// Retrieve sentiment values from the passed attribute "overallSentiment"
var sentimentData = {
pos: ${overallSentiment.positive?string?replace(',', '.')},
neu: ${overallSentiment.neutral?string?replace(',', '.')},
neg: ${overallSentiment.negative?string?replace(',', '.')}
};
var sentimentData = [];
<#if sentiments?? && sentiments?size gt 0>
<#list sentiments as sentiment>
sentimentData.push({
pos: ${sentiment.positive?number},
neu: ${sentiment.neutral?number},
neg: ${sentiment.negative?number}
});
</#list>
<#else>
sentimentData.push({ pos: 0, neu: 0, neg: 0 });
</#if>
console.log("Sentiment Data:", sentimentData);
// Set up SVG dimensions and center the chart
var width = 1000, height = 1000;
var svg = d3.select("#sentimentsRadarChart")
.attr("width", width)
.attr("height", height);
var centerX = width / 2, centerY = height / 2;
var maxRadius = 200; // Maximum radius for the chart
var maxRadius = 400; // Maximum radius for the chart
// Create a radial scale assuming sentiment values are normalized between 0 and 1
// Create a radial scale (assumes sentiment values are normalized between 0 and 1)
var rScale = d3.scaleLinear()
.domain([0, 1])
.range([0, maxRadius]);
// Define the three axes for the radar chart with fixed angles (in degrees)
// All relevant radar chart axes
var axes = [
{ axis: "Positive", angle: 0, value: sentimentData.pos },
{ axis: "Neutral", angle: 120, value: sentimentData.neu },
{ axis: "Negative", angle: 240, value: sentimentData.neg }
{ axis: "Positive", angle: 0 },
{ axis: "Neutral", angle: 120 },
{ axis: "Negative", angle: 240 }
];
// Function to convert polar coordinates to cartesian
function polarToCartesian(cx, cy, r, angleDegrees) {
var angleRadians = (angleDegrees - 90) * Math.PI / 180;
return {
@ -38,8 +42,8 @@
};
}
// Draw the concentric grid (web) lines
var gridSteps = 5; // Number of concentric polygons
// Draw grid (web) lines
var gridSteps = 5;
for (var i = 1; i <= gridSteps; i++) {
var step = i / gridSteps;
var gridPoints = axes.map(function(d) {
@ -66,7 +70,7 @@
.attr("y1", centerY)
.attr("x2", endPoint.x)
.attr("y2", endPoint.y)
.attr("stroke", "#ccc")
.attr("stroke", "#aaaaaa")
.attr("stroke-width", 1);
var labelOffset = 20;
@ -79,58 +83,35 @@
.text(d.axis);
});
// Compute radar chart data points from sentiment values
var radarPoints = axes.map(function(d) {
var r = rScale(d.value);
return polarToCartesian(centerX, centerY, r, d.angle);
});
console.log("Radar Points:", radarPoints);
var colors = d3.schemeCategory10;
// Create a closed polygon from the radar points
var radarLine = d3.line()
.x(function(d) { return d.x; })
.y(function(d) { return d.y; })
.curve(d3.curveLinearClosed);
svg.append("path")
.datum(radarPoints)
.attr("d", radarLine)
.attr("stroke", "blue")
.attr("stroke-width", 2)
.attr("fill", "blue")
.attr("fill-opacity", 0.3);
sentimentData.forEach(function(sent, i) {
// Map each axis to a point using the sentiment value for that axis.
var polygonPoints = axes.map(function(d) {
var value;
if (d.axis === "Positive") {
value = sent.pos;
} else if (d.axis === "Neutral") {
value = sent.neu;
} else if (d.axis === "Negative") {
value = sent.neg;
}
var r = rScale(value);
return polarToCartesian(centerX, centerY, r, d.angle);
});
// Create a tooltip div appended to the body and style it
var tooltip = d3.select("body").append("div")
.attr("id", "tooltip")
.style("position", "absolute")
.style("background", "#fff")
.style("border", "1px solid #ccc")
.style("padding", "4px 8px")
.style("border-radius", "4px")
.style("pointer-events", "none")
.style("font-size", "12px")
.style("box-shadow", "0px 0px 5px rgba(0,0,0,0.3)")
.style("display", "none");
// Draw circles at each radar point and attach tooltip events
axes.forEach(function(d, i) {
var point = radarPoints[i];
svg.append("circle")
.attr("cx", point.x)
.attr("cy", point.y)
.attr("r", 4)
.attr("fill", "red")
.on("mouseover", function(event) {
tooltip.style("display", "block")
.html(d.axis + ": " + d.value);
})
.on("mousemove", function(event) {
tooltip.style("left", (event.pageX + 10) + "px")
.style("top", (event.pageY - 10) + "px");
})
.on("mouseout", function() {
tooltip.style("display", "none");
});
svg.append("path")
.datum(polygonPoints)
.attr("d", radarLine)
.attr("stroke", colors[i % colors.length])
.attr("stroke-width", 2)
.attr("fill", colors[i % colors.length])
.attr("fill-opacity", 0.3);
});
</script>

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 4.9 KiB

After

Width:  |  Height:  |  Size: 3.7 KiB

Before After
Before After