Jonas_Jones/multimodal_parliament_explorer_05_1
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

added direct photos fetching backend

Browse source
This commit is contained in:
s5260822 2025-03-23 21:47:09 +01:00
parent 90a5bfead2
commit 1c36e16006
3 changed files with 120 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/main/java/org/texttechnologylab/project/gruppe_05_1/Main.java
View file

@ -147,7 +147,7 @@ public class Main {
if (UPLOAD_MEMBER_PHOTOS) {
Logger.pink("Uploading Member Photos to DB...");
mongoDBHandler.uploadMemberPhotos();
mongoDBHandler.uploadMemberPhotosFromResourceFolder();
}
SpeechVideoUpdater.init();
ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);

28
src/main/java/org/texttechnologylab/project/gruppe_05_1/database/MongoDBHandler.java
View file

@ -30,6 +30,8 @@ import java.util.concurrent.TimeUnit;
import static com.mongodb.client.model.Filters.eq;
import static org.texttechnologylab.project.gruppe_05_1.Main.MEMBER_IMAGES_DIR;
import static org.texttechnologylab.project.gruppe_05_1.util.PPRUtils.fetchMemberImageBase64FromNameString;
import static org.texttechnologylab.project.gruppe_05_1.util.PPRUtils.getSessionCookies;
public class MongoDBHandler {
@ -885,6 +887,32 @@ public class MongoDBHandler {
}
public void uploadMemberPhotos() {
// get a list of the string of first and last name of all members from the DB
// only fetch the first and lastname
List<Document> speakers = speakerCollection.find().projection(Projections.include("_id", "name", "firstName")).into(new ArrayList<>());
try {
Logger.pink(getSessionCookies());
} catch (IOException e) {
Logger.error("Failed to get session cookies: " + e.getMessage());
}
for (Document speaker : speakers) {
String memberId = speaker.getString("_id");
String name = speaker.getString("name");
String firstName = speaker.getString("firstName");
try {
String base64String = fetchMemberImageBase64FromNameString(firstName + " " + name);
uploadMemberPhoto(memberId, base64String);
Logger.debug("Uploaded member photo for " + firstName + " " + name);
} catch (IOException e) {
Logger.error("Failed to fetch member image for " + firstName + " " + name + ": " + e.getMessage());
}
}
}
public void uploadMemberPhotosFromResourceFolder() {
Logger.info("Found " + PPRUtils.listFilesInDirectory(MEMBER_IMAGES_DIR).size() + " member photos to upload.");
// loop over file names in the directory
// for each file name, extract the name of the member

91
src/main/java/org/texttechnologylab/project/gruppe_05_1/util/PPRUtils.java
View file

@ -1,6 +1,8 @@
package org.texttechnologylab.project.gruppe_05_1.util;
import com.mongodb.client.MongoCollection;
import org.json.JSONArray;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
@ -22,6 +24,7 @@ import javax.xml.parsers.DocumentBuilderFactory;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -519,4 +522,92 @@ public abstract class PPRUtils {
}
return fractions;
}
public static String fetchMemberImageBase64FromNameString(String inputString) throws IOException {
// Step 1: Send POST request
String urlString = "https://bilddatenbank.bundestag.de/ajax/picture-result";
URL url = new URL(urlString);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("POST");
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
conn.setRequestProperty("Cookie", getSessionCookies());
conn.setDoOutput(true);
// Form data
String postData = "query=" + URLEncoder.encode(inputString, "UTF-8") + "&sortVal=2";
try (OutputStream os = conn.getOutputStream()) {
byte[] input = postData.getBytes("UTF-8");
os.write(input, 0, input.length);
}
// Read response
int responseCode = conn.getResponseCode();
if (responseCode == HttpURLConnection.HTTP_OK) {
try (BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(), "UTF-8"))) {
StringBuilder response = new StringBuilder();
String line;
while ((line = br.readLine()) != null) {
response.append(line);
}
// Parse JSON response
JSONObject jsonResponse = new JSONObject(response.toString());
JSONArray fotosArray = jsonResponse.optJSONArray("fotos");
if (fotosArray != null && !fotosArray.isEmpty()) {
String hqBild = fotosArray.getJSONObject(0).optString("hqBild", "");
if (!hqBild.isEmpty()) {
// Step 2: Fetch image
String imageUrl = "https://bilddatenbank.bundestag.de/fotos/" + hqBild;
URL imageDownloadUrl = new URL(imageUrl);
HttpURLConnection imageConn = (HttpURLConnection) imageDownloadUrl.openConnection();
imageConn.setRequestMethod("GET");
if (imageConn.getResponseCode() == HttpURLConnection.HTTP_OK) {
try (InputStream is = imageConn.getInputStream(); ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
byte[] buffer = new byte[8192];
int bytesRead;
while ((bytesRead = is.read(buffer)) != -1) {
baos.write(buffer, 0, bytesRead);
}
return Base64.getEncoder().encodeToString(baos.toByteArray());
}
}
}
}
}
}
return "Error: Unable to retrieve image";
}
public static String getSessionCookies() throws IOException {
String urlString = "https://bilddatenbank.bundestag.de/search/picture-result?query=Angela+Merkel&sortVal=2";
URL url = new URL(urlString);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("GET");
// get both header fields with the name "Set-Cookie"
Map<String, List<String>> headerFields = conn.getHeaderFields();
String phpSessId = "";
String csrfToken = "";
// iterate over the header fields
for (Map.Entry<String, List<String>> entry : headerFields.entrySet()) {
// if header field is "Set-Cookie"
if (entry.getKey() != null && entry.getKey().equals("set-cookie")) {
// iterate over the values of the header field
for (String value : entry.getValue()) {
// if value contains "PHPSESSID"
if (value.contains("PHPSESSID")) {
phpSessId = value.split(";")[0];
}
// if value contains "_csrf"
if (value.contains("_csrf")) {
csrfToken = value.split(";")[0];
}
}
}
}
return "PHPSESSID=" + phpSessId + "; _csrf=" + csrfToken;
}
}