From 7542a0ca66cb1bc447d62f0f6d8abe56b1023000 Mon Sep 17 00:00:00 2001 From: pidoubleyou <22942659+pidoubleyou@users.noreply.github.com> Date: Sun, 10 May 2026 18:15:31 +0200 Subject: [PATCH 1/3] #1140 search compilations --- .../crawler/sender/ard/ArdCrawler.java | 21 +++++- .../crawler/sender/ard/ArdFilmInfoDto.java | 13 +++- .../ard/json/ArdDayPageDeserializer.java | 2 +- .../sender/ard/json/ArdFilmDeserializer.java | 2 +- .../ard/json/ArdTeasersDeserializer.java | 26 ++++++-- .../json/ArdTopicCompilationDeserializer.java | 64 +++++++++++++++++++ .../ard/tasks/ArdTopicCompilationTask.java | 42 ++++++++++++ 7 files changed, 157 insertions(+), 13 deletions(-) create mode 100644 src/main/java/mServer/crawler/sender/ard/json/ArdTopicCompilationDeserializer.java create mode 100644 src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicCompilationTask.java diff --git a/src/main/java/mServer/crawler/sender/ard/ArdCrawler.java b/src/main/java/mServer/crawler/sender/ard/ArdCrawler.java index fe48b6438..9b3386e8f 100644 --- a/src/main/java/mServer/crawler/sender/ard/ArdCrawler.java +++ b/src/main/java/mServer/crawler/sender/ard/ArdCrawler.java @@ -12,6 +12,7 @@ import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; @@ -27,6 +28,7 @@ public class ArdCrawler extends MediathekCrawler { = DateTimeFormatter.ofPattern("yyyy-MM-dd"); public static final String[] MISSING_TOPIC_IDS = new String[]{ +// "Y3JpZDovL2JyLmRlL2Jyb2FkY2FzdFNlcmllcy9icm9hZGNhc3RTZXJpZXM6L2JyZGUvZmVybnNlaGVuL2JheWVyaXNjaGVzLWZlcm5zZWhlbi9zZW5kdW5nZW4vZGFob2FtLWlzLWRhaG9hbQ" }; public ArdCrawler(FilmeSuchen ssearch, int startPrio) { @@ -131,6 +133,7 @@ private Set getDaysEntries() throws InterruptedException, Execut } private Set getTopicsEntries() throws ExecutionException, InterruptedException { + final Set shows = new HashSet<>(); Set topics = new HashSet<>(); topics.addAll(getTopicEntriesBySender(ArdConstants.DEFAULT_CLIENT)); for (String client : ArdConstants.CLIENTS) { @@ -149,9 +152,21 @@ private Set getTopicsEntries() throws ExecutionException, Interr ConcurrentLinkedQueue topicUrls = new ConcurrentLinkedQueue<>(assitUrls); final ArdTopicPageTask topicTask = new ArdTopicPageTask(this, topicUrls); - final Set filmInfos = forkJoinPool.submit(topicTask).get(); - Log.sysLog("ard shows by topics: " + filmInfos.size()); - return filmInfos; + final Set ardFilmInfosWithCompilations = forkJoinPool.submit(topicTask).get(); + + // add filmInfos without compilation + shows.addAll(ardFilmInfosWithCompilations.stream().filter(filmInfo -> !filmInfo.isCompilation()).toList()); + + // search compilations + final List compilations = ardFilmInfosWithCompilations.stream().filter(ArdFilmInfoDto::isCompilation).toList(); + final ArdTopicCompilationTask compilationTask = new ArdTopicCompilationTask(this, new ConcurrentLinkedQueue<>(compilations)); + final Set ardCompilationEntries = forkJoinPool.submit(compilationTask).get(); + + final int sizeBefore = shows.size(); + shows.addAll(ardCompilationEntries.stream().filter(filmInfo -> !filmInfo.isCompilation()).toList()); + Log.sysLog("ard shows by topics compilation: " + (shows.size() - sizeBefore)); + Log.sysLog("ard shows by topics: " + shows.size()); + return shows; } // temporary workaround for missing topics diff --git a/src/main/java/mServer/crawler/sender/ard/ArdFilmInfoDto.java b/src/main/java/mServer/crawler/sender/ard/ArdFilmInfoDto.java index 5bca8623b..5f6bf0dcc 100644 --- a/src/main/java/mServer/crawler/sender/ard/ArdFilmInfoDto.java +++ b/src/main/java/mServer/crawler/sender/ard/ArdFilmInfoDto.java @@ -7,18 +7,24 @@ public class ArdFilmInfoDto extends CrawlerUrlDTO { private final String id; private final int numberOfClips; + private final boolean isCompilation; - public ArdFilmInfoDto(String id, String aUrl, int numberOfClips) { + public ArdFilmInfoDto(String id, String aUrl, int numberOfClips, boolean isCompilation) { super(aUrl); this.id = id; this.numberOfClips = numberOfClips; + this.isCompilation = isCompilation; } public String getId() { return id; } + public boolean isCompilation() { + return isCompilation; + } + public int getNumberOfClips() { return numberOfClips; } @@ -36,11 +42,12 @@ public boolean equals(Object o) { } ArdFilmInfoDto that = (ArdFilmInfoDto) o; return numberOfClips == that.numberOfClips - && Objects.equals(id, that.id); + && Objects.equals(id, that.id) + && isCompilation == that.isCompilation; } @Override public int hashCode() { - return Objects.hash(super.hashCode(), id, numberOfClips); + return Objects.hash(super.hashCode(), id, numberOfClips, isCompilation); } } diff --git a/src/main/java/mServer/crawler/sender/ard/json/ArdDayPageDeserializer.java b/src/main/java/mServer/crawler/sender/ard/json/ArdDayPageDeserializer.java index c489ec270..dbdeb8703 100644 --- a/src/main/java/mServer/crawler/sender/ard/json/ArdDayPageDeserializer.java +++ b/src/main/java/mServer/crawler/sender/ard/json/ArdDayPageDeserializer.java @@ -50,7 +50,7 @@ private Set parseChannels(JsonArray channels) { private ArdFilmInfoDto createFilmInfo(final String id, final int numberOfClips) { final String url = String.format(ArdConstants.ITEM_URL, id); - return new ArdFilmInfoDto(id, url, numberOfClips); + return new ArdFilmInfoDto(id, url, numberOfClips, false); } private Optional toId(final JsonObject teaserObject) { diff --git a/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java b/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java index 58cda37a8..18beb797b 100644 --- a/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java +++ b/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java @@ -383,7 +383,7 @@ private void parseRelatedFilms(final ArdFilmDto filmDto, final JsonObject player = JsonUtils.getAttributeAsString(teasersItemObject, ATTRIBUTE_ID); if (id.isPresent()) { final String url = String.format(ArdConstants.ITEM_URL, id.get()); - filmDto.addRelatedFilm(new ArdFilmInfoDto(id.get(), url, 0)); + filmDto.addRelatedFilm(new ArdFilmInfoDto(id.get(), url, 0, false)); } } } diff --git a/src/main/java/mServer/crawler/sender/ard/json/ArdTeasersDeserializer.java b/src/main/java/mServer/crawler/sender/ard/json/ArdTeasersDeserializer.java index 35cc2ee69..217c9a3df 100644 --- a/src/main/java/mServer/crawler/sender/ard/json/ArdTeasersDeserializer.java +++ b/src/main/java/mServer/crawler/sender/ard/json/ArdTeasersDeserializer.java @@ -18,8 +18,10 @@ abstract class ArdTeasersDeserializer { private static final String ELEMENT_LINKS = "links"; private static final String ELEMENT_TARGET = "target"; + private static final String ATTRIBUTE_HREF = "href"; private static final String ATTRIBUTE_ID = "id"; private static final String ATTRIBUTE_NUMBER_OF_CLIPS = "numberOfClips"; + private static final String ATTRIBUTE_TYPE = "type"; Set parseTeasers(final JsonArray teasers) { return StreamSupport.stream(teasers.spliterator(), true) @@ -30,9 +32,23 @@ Set parseTeasers(final JsonArray teasers) { } private ArdFilmInfoDto toFilmInfo(final JsonObject teaserObject) { - return toId(teaserObject) - .map(id -> createFilmInfo(id, getNumberOfClips(teaserObject))) - .orElse(null); + final boolean compilation = isCompilation(teaserObject); + if (compilation) { + final Optional url = JsonUtils.getElementValueAsString(teaserObject, ELEMENT_LINKS, ELEMENT_TARGET, ATTRIBUTE_HREF); + final Optional id = toId(teaserObject); + return url.map(s -> new ArdFilmInfoDto(id.orElse(""), s, getNumberOfClips(teaserObject), compilation)).orElse(null); + } else { + return toId(teaserObject) + .map(id -> createFilmInfo(id, getNumberOfClips(teaserObject), compilation)) + .orElse(null); + } + } + + private boolean isCompilation(final JsonObject teaserObject) { + if (teaserObject.has(ATTRIBUTE_TYPE)) { + return "compilation".equals(teaserObject.get(ATTRIBUTE_TYPE).getAsString()); + } + return false; } private int getNumberOfClips(final JsonObject teaserObject) { @@ -51,12 +67,12 @@ private Optional toId(final JsonObject teaserObject) { return JsonUtils.getAttributeAsString(teaserObject, ATTRIBUTE_ID); } - private ArdFilmInfoDto createFilmInfo(final String id, final int numberOfClips) { + private ArdFilmInfoDto createFilmInfo(final String id, final int numberOfClips, final boolean isCompilation) { String refId = id; if(id.contains(":")) { refId = id.replace(":", "%3A"); } final String url = String.format(ArdConstants.ITEM_URL, refId); - return new ArdFilmInfoDto(id, url, numberOfClips); + return new ArdFilmInfoDto(id, url, numberOfClips, isCompilation); } } diff --git a/src/main/java/mServer/crawler/sender/ard/json/ArdTopicCompilationDeserializer.java b/src/main/java/mServer/crawler/sender/ard/json/ArdTopicCompilationDeserializer.java new file mode 100644 index 000000000..33375a647 --- /dev/null +++ b/src/main/java/mServer/crawler/sender/ard/json/ArdTopicCompilationDeserializer.java @@ -0,0 +1,64 @@ +package mServer.crawler.sender.ard.json; + +import com.google.gson.*; +import mServer.crawler.sender.ard.ArdFilmInfoDto; +import mServer.crawler.sender.ard.ArdTopicInfoDto; + +import java.lang.reflect.Type; +import java.util.HashSet; +import java.util.Set; + +public class ArdTopicCompilationDeserializer extends ArdTeasersDeserializer + implements JsonDeserializer { + + private static final String ELEMENT_WIDGETS = "widgets"; + private static final String ELEMENT_TEASERS = "teasers"; + private static final String ELEMENT_PAGE_NUMBER = "pageNumber"; + private static final String ELEMENT_TOTAL_ELEMENTS = "totalElements"; + private static final String ELEMENT_PAGE_SIZE = "pageSize"; + private static final String ELEMENT_PAGINATION = "pagination"; + + @Override + public ArdTopicInfoDto deserialize( + final JsonElement showPageElement, final Type type, final JsonDeserializationContext context) { + final Set results = new HashSet<>(); + final ArdTopicInfoDto ardTopicInfoDto = new ArdTopicInfoDto(results); + + final JsonObject showPageObject = showPageElement.getAsJsonObject(); + if (showPageObject.has(ELEMENT_WIDGETS)) { + final JsonArray widgets = showPageObject.get(ELEMENT_WIDGETS).getAsJsonArray(); + widgets.forEach(widget -> { + if (widget.getAsJsonObject().has(ELEMENT_TEASERS)) { + final JsonArray teasers = widget.getAsJsonObject().get(ELEMENT_TEASERS).getAsJsonArray(); + results.addAll(parseTeasers(teasers)); + } + }); + } + + final JsonElement paginationElement = showPageObject.get(ELEMENT_PAGINATION); + final int pageNumber = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_PAGE_NUMBER); + final int totalElements = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_TOTAL_ELEMENTS); + final int pageSize = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_PAGE_SIZE); + ardTopicInfoDto.setMaxSubPageNumber(pageSize == 0 ? 0 : + (totalElements + pageSize - 1) / pageSize); + ardTopicInfoDto.setSubPageNumber(pageNumber); + + return ardTopicInfoDto; + } + + private int getChildElementAsIntOrNullIfNotExist( + final JsonElement parentElement, final String childElementName) { + if (parentElement == null || parentElement.isJsonNull()) { + return 0; + } + return getJsonElementAsIntOrNullIfNotExist( + parentElement.getAsJsonObject().get(childElementName)); + } + + private int getJsonElementAsIntOrNullIfNotExist(final JsonElement element) { + if (element.isJsonNull()) { + return 0; + } + return element.getAsInt(); + } +} diff --git a/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicCompilationTask.java b/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicCompilationTask.java new file mode 100644 index 000000000..0d4894115 --- /dev/null +++ b/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicCompilationTask.java @@ -0,0 +1,42 @@ +package mServer.crawler.sender.ard.tasks; + +import com.google.gson.reflect.TypeToken; +import jakarta.ws.rs.client.WebTarget; +import mServer.crawler.sender.MediathekReader; +import mServer.crawler.sender.ard.ArdFilmInfoDto; +import mServer.crawler.sender.ard.ArdTopicInfoDto; +import mServer.crawler.sender.ard.json.ArdTopicCompilationDeserializer; +import mServer.crawler.sender.base.AbstractRecursivConverterTask; +import mServer.crawler.sender.base.CrawlerUrlDTO; + +import java.lang.reflect.Type; +import java.util.concurrent.ConcurrentLinkedQueue; + +public class ArdTopicCompilationTask extends ArdTaskBase { + + private static final Type ARDTOPICINFODTO_TYPE_TOKEN = + new TypeToken() {}.getType(); + + public ArdTopicCompilationTask( + final MediathekReader aCrawler, final ConcurrentLinkedQueue aUrlToCrawlDtos) { + super(aCrawler, aUrlToCrawlDtos); + + registerJsonDeserializer(ARDTOPICINFODTO_TYPE_TOKEN, new ArdTopicCompilationDeserializer()); + } + + @Override + protected void processRestTarget(final CrawlerUrlDTO aDTO, final WebTarget aTarget) { + final ArdTopicInfoDto topicInfo = deserialize(aTarget, ARDTOPICINFODTO_TYPE_TOKEN); + if (topicInfo != null + && topicInfo.getFilmInfos() != null + && !topicInfo.getFilmInfos().isEmpty()) { + taskResults.addAll(topicInfo.getFilmInfos()); + } + } + + @Override + protected AbstractRecursivConverterTask createNewOwnInstance( + final ConcurrentLinkedQueue aElementsToProcess) { + return new ArdTopicCompilationTask(crawler, aElementsToProcess); + } +} \ No newline at end of file From 1f25e6448f7339ebeb6d6360f80d7b782f60e743 Mon Sep 17 00:00:00 2001 From: pidoubleyou <22942659+pidoubleyou@users.noreply.github.com> Date: Sun, 10 May 2026 21:08:52 +0200 Subject: [PATCH 2/3] #1144 adaptive urls for ov, ad, dgs --- .../sender/ard/json/ArdFilmDeserializer.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java b/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java index 18beb797b..7eee3837a 100644 --- a/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java +++ b/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java @@ -234,8 +234,11 @@ public List deserialize( Optional> videoInfoStandard = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE); Optional> videoInfoAdaptive = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_CATEGORY_MPEG, MARKER_VIDEO_DE); Optional> videoInfoAD = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_AD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE); + Optional> videoInfoADAdaptive = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_AD, MARKER_VIDEO_CATEGORY_MPEG, MARKER_VIDEO_DE); Optional> videoInfoDGS = parseVideoUrls(itemObject, MARKER_VIDEO_DGS, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE); + Optional> videoInfoDGSAdaptive = parseVideoUrls(itemObject, MARKER_VIDEO_DGS, MARKER_VIDEO_STANDARD, MARKER_VIDEO_CATEGORY_MPEG, MARKER_VIDEO_DE); Optional> videoInfoOV = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_OV); + Optional> videoInfoOVAdaptive = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_CATEGORY_MPEG, MARKER_VIDEO_OV); Optional subtitles = prepareSubtitleUrl(itemObject); if (topic.isEmpty() || title.isEmpty() || partner.isEmpty() || ADDITIONAL_SENDER.get(partner.get()) == null) { @@ -249,7 +252,16 @@ public List deserialize( || titleoriginal.get().contains("- Hörfassung") || titleoriginal.get().contains("(mit Audiodeskription)"); // mainly funk if (videoInfoStandard.isEmpty() && videoInfoAD.isEmpty() && videoInfoDGS.isEmpty() && videoInfoOV.isEmpty() && videoInfoAdaptive.isPresent()) { - videoInfoStandard = resolveFallbackFromPlaylist(videoInfoAdaptive); + videoInfoStandard = getResolutionsFromAdaptiveUrl(videoInfoAdaptive); + } + if (videoInfoAD.isEmpty() && videoInfoADAdaptive.isPresent()) { + videoInfoAD = getResolutionsFromAdaptiveUrl(videoInfoADAdaptive); + } + if (videoInfoDGS.isEmpty() && videoInfoDGSAdaptive.isPresent()) { + videoInfoDGS = getResolutionsFromAdaptiveUrl(videoInfoDGSAdaptive); + } + if (videoInfoOV.isEmpty() && videoInfoOVAdaptive.isPresent()) { + videoInfoOV = getResolutionsFromAdaptiveUrl(videoInfoOVAdaptive); } // incorrect langueage code for OV if ((titleoriginal.get().contains(" - (Originalversion)") || titleoriginal.get().contains(" (OV)")) && videoInfoOV.isEmpty()) { @@ -488,7 +500,7 @@ private Optional> parseVideoUrlMap(final JsonObject playerP return Optional.of(videoInfo); } - private Optional> resolveFallbackFromPlaylist(Optional> videoInfoAdaptive) { + private Optional> getResolutionsFromAdaptiveUrl(Optional> videoInfoAdaptive) { Map qualitiesUrls = videoInfoAdaptive.get().entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> { try { From 0c30d5f013a15bc33a383ac486b1501e88c1a650 Mon Sep 17 00:00:00 2001 From: pidoubleyou <22942659+pidoubleyou@users.noreply.github.com> Date: Sun, 10 May 2026 22:39:41 +0200 Subject: [PATCH 3/3] #1144 convert m3u8 to mp4-urls --- .../sender/ard/WdrM3U8ToMp4Converter.java | 100 ++++++++++++++++++ .../sender/ard/json/ArdFilmDeserializer.java | 70 +++++++----- 2 files changed, 145 insertions(+), 25 deletions(-) create mode 100644 src/main/java/mServer/crawler/sender/ard/WdrM3U8ToMp4Converter.java diff --git a/src/main/java/mServer/crawler/sender/ard/WdrM3U8ToMp4Converter.java b/src/main/java/mServer/crawler/sender/ard/WdrM3U8ToMp4Converter.java new file mode 100644 index 000000000..2eae4eb0c --- /dev/null +++ b/src/main/java/mServer/crawler/sender/ard/WdrM3U8ToMp4Converter.java @@ -0,0 +1,100 @@ +package mServer.crawler.sender.ard; + +import de.mediathekview.mlib.tool.Log; +import mServer.crawler.sender.base.Qualities; + +import java.util.EnumMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * * Converts an m3u8 playlist URL (in the style shown in the example) * into a list of progressive + * mp4 URLs. * * Default progressive base host: https://wdr-progressive.ard-mcdn.de * * Example: * + * input: * + * https://wdrvod-rwrtr.akamaized.net/i/,/media/.../ID_AVC-,270,360,540,720,1080,.mp4.csmil/index-...m3u8 + * * * produced outputs: * https://wdr-progressive.ard-mcdn.de/media/.../ID_AVC-270.mp4 * ... + */ +public class WdrM3U8ToMp4Converter { + + // Looks for /media/...-.mp4.csmil + // group(1) = /media/... (path before codec) + // group(2) = _- (codec token + trailing '-') + // group(3) = the comma separated tail that contains bitrate numbers (e.g. ",270,360,540,") + private static final Pattern MEDIA_PATTERN = + Pattern.compile( + "(/media/.+?)(_[A-Za-z0-9]+-)([^/]+?)\\.mp4\\.csmil", Pattern.CASE_INSENSITIVE); + + private final String progressiveBase; // no trailing slash + + public WdrM3U8ToMp4Converter() { + this("https://wdr-progressive.ard-mcdn.de"); + } + + public WdrM3U8ToMp4Converter(String progressiveBase) { + if (progressiveBase == null) + throw new IllegalArgumentException("progressiveBase must not be null"); + // ensure no trailing slash to make concatenation predictable + this.progressiveBase = progressiveBase.replaceAll("/+$", ""); + } + + /** + * Convert an m3u8 url into a list of mp4 URLs. + * + * @param m3u8Url the source m3u8 url + * @return map of mp4 URLs (in the same order as bitrates found) + * @throws IllegalArgumentException if the url cannot be parsed or no bitrates found + */ + public Map convert(String m3u8Url) { + if (m3u8Url == null) throw new IllegalArgumentException("m3u8Url must not be null"); + + Map result = new EnumMap<>(Qualities.class); + + Matcher m = MEDIA_PATTERN.matcher(m3u8Url); + if (!m.find()) { + return result; + } + + String pathBeforeCodec = m.group(1); // includes leading /media/... + String codecWithDash = m.group(2); // e.g. _AVC- + String bitrateListPart = m.group(3); // e.g. ",270,360,540,720,1080," or "270,360" + + // build the base path (starts with /media/...) + String basePrefix = pathBeforeCodec + codecWithDash; // ends with '-' + + // extract numeric tokens (bitrate values) + Pattern digits = Pattern.compile("\\d+"); + Matcher mDigits = digits.matcher(bitrateListPart); + + while (mDigits.find()) { + String bitrate = mDigits.group(); + // join progressive base + basePrefix + bitrate + .mp4 + String mp4 = progressiveBase + basePrefix + bitrate + ".mp4"; + final Qualities resolution = getResolutionFromWidth(bitrate); + result.put(resolution, mp4); + } + + if (result.isEmpty()) { + throw new IllegalArgumentException("No numeric bitrate tokens found in m3u8 URL: " + m3u8Url); + } + + return result; + } + + private Qualities getResolutionFromWidth(String bitrate) { + try { + return switch (Integer.parseInt(bitrate)) { + case 720 -> Qualities.NORMAL; + case 1080 -> Qualities.HD; + case 540, 360, 270 -> Qualities.SMALL; + default -> { + Log.sysLog("Unknown bitrate found in m3u8 URL: " + bitrate + ", defaulting to VERY_SMALL"); + yield Qualities.SMALL; + } + }; + } catch (NumberFormatException e) { + Log.errorLog(165346373, e); + return Qualities.SMALL; + } + } +} diff --git a/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java b/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java index 7eee3837a..7a33dbdcd 100644 --- a/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java +++ b/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java @@ -32,15 +32,17 @@ import mServer.crawler.sender.ard.ArdConstants; import mServer.crawler.sender.ard.ArdFilmDto; import mServer.crawler.sender.ard.ArdFilmInfoDto; +import mServer.crawler.sender.ard.WdrM3U8ToMp4Converter; import mServer.crawler.sender.base.JsonUtils; import mServer.crawler.sender.base.Qualities; import mServer.crawler.sender.base.UrlUtils; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; public class ArdFilmDeserializer implements JsonDeserializer> { - private static final org.apache.logging.log4j.Logger LOG + private static final Logger LOG = LogManager.getLogger(ArdFilmDeserializer.class); private static final String GERMAN_TIME_ZONE = "Europe/Berlin"; @@ -110,6 +112,12 @@ public class ArdFilmDeserializer implements JsonDeserializer> { //IGNORED_SENDER "zdf", "kika", "3sat", "arte" } + private final WdrM3U8ToMp4Converter converter; + + public ArdFilmDeserializer() { + converter = new WdrM3U8ToMp4Converter(); + } + private static Optional getMediaCollectionObject(final JsonObject itemObject) { if (itemObject.has(ELEMENT_MEDIA_COLLECTION) && !itemObject.get(ELEMENT_MEDIA_COLLECTION).isJsonNull() @@ -501,28 +509,40 @@ private Optional> parseVideoUrlMap(final JsonObject playerP } private Optional> getResolutionsFromAdaptiveUrl(Optional> videoInfoAdaptive) { - Map qualitiesUrls = videoInfoAdaptive.get().entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> { - try { - return new URL(entry.getValue()); - } catch (MalformedURLException e) { - LOG.error("failed converting string {} to url", entry.getValue(), e); - return null; - } - })); - if (!qualitiesUrls.containsKey(Qualities.NORMAL)) { - qualitiesUrls.put(Qualities.NORMAL, qualitiesUrls.entrySet().stream().findFirst().get().getValue()); - } - // - ArdVideoInfoJsonDeserializer.loadM3U8(qualitiesUrls); - // - Map fallback = qualitiesUrls.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().toString())); - // - if (!fallback.containsKey(Qualities.NORMAL) && !fallback.isEmpty()) { - fallback.put(Qualities.NORMAL, fallback.entrySet().stream().findFirst().get().getValue()); - } - - return Optional.of(fallback); - } + if (videoInfoAdaptive.isPresent()) { + if (videoInfoAdaptive.get().containsKey(Qualities.NORMAL)) { + final String url = videoInfoAdaptive.get().get(Qualities.NORMAL); + final Map mp4Urls = converter.convert(url); + if (!mp4Urls.isEmpty()) { + return Optional.of(mp4Urls); + } + } + + Map qualitiesUrls = videoInfoAdaptive.get().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> { + try { + return new URL(entry.getValue()); + } catch (MalformedURLException e) { + LOG.error("failed converting string {} to url", entry.getValue(), e); + return null; + } + })); + if (!qualitiesUrls.containsKey(Qualities.NORMAL)) { + qualitiesUrls.put(Qualities.NORMAL, qualitiesUrls.entrySet().stream().findFirst().get().getValue()); + } + // + ArdVideoInfoJsonDeserializer.loadM3U8(qualitiesUrls); + // + Map fallback = qualitiesUrls.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().toString())); + // + if (!fallback.containsKey(Qualities.NORMAL) && !fallback.isEmpty()) { + fallback.put(Qualities.NORMAL, fallback.entrySet().stream().findFirst().get().getValue()); + } + + return Optional.of(fallback); + } + + return Optional.empty(); + } }