Skip to content

Commit 1e8e341

Browse files
alex1702alex1702
committed
Merge branch 'release/3.0.2'
Fixed: ZDF-Crawler "Abmeldung" nicht zuverlässig closed #85
2 parents 0834ab3 + 8319e6b commit 1e8e341

File tree

3 files changed

+70
-109
lines changed

3 files changed

+70
-109
lines changed

‎build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ apply from: "${project.rootDir}/gradle/eclipse.gradle"
1515
sourceCompatibility = 1.8
1616
targetCompatibility = 1.8
1717
group = 'de.mediathekview'
18-
version = '3.0.1'
18+
version = '3.0.2'
1919

2020
def jarName = 'MServer.jar'
2121
def mainClass = 'mServer.Main'

‎src/main/java/mServer/crawler/FilmeSuchen.java‎

Lines changed: 10 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -19,42 +19,20 @@
1919
*/
2020
package mServer.crawler;
2121

22-
import java.text.SimpleDateFormat;
23-
import java.util.ArrayList;
24-
import java.util.Date;
25-
import java.util.LinkedList;
26-
27-
import javax.swing.event.EventListenerList;
28-
29-
import org.apache.commons.lang3.time.FastDateFormat;
30-
3122
import de.mediathekview.mlib.Config;
3223
import de.mediathekview.mlib.Const;
3324
import de.mediathekview.mlib.daten.ListeFilme;
3425
import de.mediathekview.mlib.filmesuchen.ListenerFilmeLaden;
3526
import de.mediathekview.mlib.filmesuchen.ListenerFilmeLadenEvent;
3627
import de.mediathekview.mlib.tool.Log;
37-
import mServer.crawler.sender.Mediathek3Sat;
38-
import mServer.crawler.sender.MediathekArd;
39-
import mServer.crawler.sender.MediathekArte_de;
40-
import mServer.crawler.sender.MediathekArte_fr;
41-
import mServer.crawler.sender.MediathekBr;
42-
import mServer.crawler.sender.MediathekDw;
43-
import mServer.crawler.sender.MediathekHr;
44-
import mServer.crawler.sender.MediathekKika;
45-
import mServer.crawler.sender.MediathekMdr;
46-
import mServer.crawler.sender.MediathekNdr;
47-
import mServer.crawler.sender.MediathekOrf;
48-
import mServer.crawler.sender.MediathekPhoenix;
49-
import mServer.crawler.sender.MediathekRbb;
50-
import mServer.crawler.sender.MediathekReader;
51-
import mServer.crawler.sender.MediathekSr;
52-
import mServer.crawler.sender.MediathekSrf;
53-
import mServer.crawler.sender.MediathekSrfPod;
54-
import mServer.crawler.sender.MediathekSwr;
55-
import mServer.crawler.sender.MediathekWdr;
56-
import mServer.crawler.sender.MediathekZdf;
57-
import mServer.crawler.sender.MediathekZdfTivi;
28+
import java.text.SimpleDateFormat;
29+
import java.util.ArrayList;
30+
import java.util.Date;
31+
import java.util.LinkedList;
32+
import javax.swing.event.EventListenerList;
33+
import mServer.crawler.CrawlerConfig;
34+
import mServer.crawler.sender.*;
35+
import org.apache.commons.lang3.time.FastDateFormat;
5836

5937
/**
6038
* ###########################################################################################################
@@ -145,7 +123,8 @@ public void updateSender(String[] nameSender, ListeFilme listeFilme) {
145123
for (String s : nameSender) {
146124
if (reader.checkNameSenderFilmliste(s)) {
147125
starten = true;
148-
reader.start();
126+
new Thread(reader).start();
127+
//reader.start();
149128
}
150129
}
151130
}

‎src/main/java/mServer/crawler/sender/MediathekZdf.java‎

Lines changed: 59 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -19,41 +19,30 @@
1919
*/
2020
package mServer.crawler.sender;
2121

22-
import java.util.Collection;
23-
import java.util.concurrent.ExecutorService;
24-
import java.util.concurrent.ForkJoinPool;
25-
import java.util.concurrent.Phaser;
26-
import java.util.concurrent.RecursiveAction;
27-
import java.util.concurrent.TimeUnit;
28-
2922
import de.mediathekview.mlib.Config;
3023
import de.mediathekview.mlib.Const;
3124
import de.mediathekview.mlib.daten.DatenFilm;
3225
import de.mediathekview.mlib.tool.Log;
3326
import etm.core.configuration.EtmManager;
3427
import etm.core.monitor.EtmPoint;
28+
import java.util.Collection;
29+
import java.util.concurrent.*;
3530
import mServer.crawler.CrawlerTool;
3631
import mServer.crawler.FilmeSuchen;
3732
import mServer.crawler.RunSender;
38-
import mServer.crawler.sender.newsearch.DownloadDTO;
39-
import mServer.crawler.sender.newsearch.GeoLocations;
40-
import mServer.crawler.sender.newsearch.Qualities;
41-
import mServer.crawler.sender.newsearch.VideoDTO;
42-
import mServer.crawler.sender.newsearch.ZDFSearchTask;
43-
import mServer.crawler.sender.newsearch.ZdfDatenFilm;
33+
import mServer.crawler.sender.newsearch.*;
4434

45-
public class MediathekZdf extends MediathekReader
46-
{
35+
public class MediathekZdf extends MediathekReader {
4736

4837
public final static String SENDERNAME = Const.ZDF;
4938
// public static final String URL_PATTERN_SENDUNG_VERPASST = "https://www.zdf.de/sendung-verpasst?airtimeDate=%s";
5039
// public static final String[] KATEGORIE_ENDS = {"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "0+-+9"};
5140
// public static final String KATEGORIEN_URL_PATTERN = "https://www.zdf.de/sendungen-a-z/?group=%s";
52-
private final ForkJoinPool forkJoinPool = new ForkJoinPool(Runtime.getRuntime().availableProcessors() * 4);
41+
//private final ForkJoinPool forkJoinPool = new ForkJoinPool(Runtime.getRuntime().availableProcessors() * 4);
42+
private ForkJoinPool forkJoinPool;
5343
// private final MSStringBuilder seite = new MSStringBuilder(Const.STRING_BUFFER_START_BUFFER);
5444

55-
public MediathekZdf(FilmeSuchen ssearch, int startPrio)
56-
{
45+
public MediathekZdf(FilmeSuchen ssearch, int startPrio) {
5746
super(ssearch, SENDERNAME, 0 /* threads */, 150 /* urlWarten */, startPrio);
5847
setName("MediathekZdf");
5948
}
@@ -64,10 +53,11 @@ public MediathekZdf(FilmeSuchen ssearch, int startPrio)
6453
public void addToList() {
6554
meldungStart();
6655
meldungAddThread();
67-
56+
6857
int days = CrawlerTool.loadLongMax() ? 300 : 20;
69-
58+
7059
final ZDFSearchTask newTask = new ZDFSearchTask(days);
60+
forkJoinPool = new ForkJoinPool(Runtime.getRuntime().availableProcessors() * 4);
7161
forkJoinPool.execute(newTask);
7262
Collection<VideoDTO> filmList = newTask.join();
7363

@@ -91,8 +81,9 @@ public void addToList() {
9181
wasInterrupted = true;
9282
phaser.forceTermination();
9383
shutdownAndAwaitTermination(forkJoinPool, 5, TimeUnit.SECONDS);
94-
} else
84+
} else {
9585
TimeUnit.SECONDS.sleep(1);
86+
}
9687
} catch (InterruptedException ignored) {
9788
}
9889
}
@@ -101,10 +92,11 @@ public void addToList() {
10192
shutdownAndAwaitTermination(forkJoinPool, 60, TimeUnit.SECONDS);
10293

10394
perfPoint.collect();
104-
if (wasInterrupted)
95+
if (wasInterrupted) {
10596
Log.sysLog("VideoDTO conversion interrupted.");
106-
else
97+
} else {
10798
Log.sysLog("convert VideoDTO to DatenFilm finished.");
99+
}
108100

109101
meldungThreadUndFertig();
110102
}
@@ -114,8 +106,9 @@ void shutdownAndAwaitTermination(ExecutorService pool, long delay, TimeUnit dela
114106
try {
115107
if (!pool.awaitTermination(delay, delayUnit)) {
116108
pool.shutdownNow();
117-
if (!pool.awaitTermination(delay, delayUnit))
109+
if (!pool.awaitTermination(delay, delayUnit)) {
118110
Log.sysLog("Pool did not terminate");
111+
}
119112
}
120113
} catch (InterruptedException ie) {
121114
pool.shutdownNow();
@@ -125,6 +118,7 @@ void shutdownAndAwaitTermination(ExecutorService pool, long delay, TimeUnit dela
125118

126119
@SuppressWarnings("serial")
127120
private class VideoDtoDatenFilmConverterAction extends RecursiveAction {
121+
128122
private final VideoDTO video;
129123

130124
public VideoDtoDatenFilmConverterAction(VideoDTO aVideoDTO) {
@@ -134,42 +128,39 @@ public VideoDtoDatenFilmConverterAction(VideoDTO aVideoDTO) {
134128

135129
@Override
136130
protected void compute() {
137-
if(video != null) {
138-
try {
139-
DownloadDTO download = video.getDownloadDto();
140-
141-
DatenFilm film = new ZdfDatenFilm(SENDERNAME, video.getTopic(), video.getWebsiteUrl() /*urlThema*/,
142-
video.getTitle(), download.getUrl(Qualities.NORMAL), "" /*urlRtmp*/,
143-
video.getDate(), video.getTime(), video.getDuration(), video.getDescription());
144-
urlTauschen(film, video.getWebsiteUrl(), mlibFilmeSuchen);
145-
146-
//don´t use addFilm here
147-
if (mlibFilmeSuchen.listeFilmeNeu.addFilmVomSender(film)) {
148-
// dann ist er neu
149-
FilmeSuchen.listeSenderLaufen.inc(film.arr[DatenFilm.FILM_SENDER], RunSender.Count.FILME);
150-
}
151-
152-
if (!download.getUrl(Qualities.HD).isEmpty())
153-
{
154-
CrawlerTool.addUrlHd(film, download.getUrl(Qualities.HD), "");
155-
}
156-
if (!download.getUrl(Qualities.SMALL).isEmpty())
157-
{
158-
CrawlerTool.addUrlKlein(film, download.getUrl(Qualities.SMALL), "");
159-
}
160-
if (!download.getSubTitleUrl().isEmpty())
161-
{
162-
CrawlerTool.addUrlSubtitle(film, download.getSubTitleUrl());
163-
}
164-
if(download.getGeoLocation() != GeoLocations.GEO_NONE) {
165-
film.arr[DatenFilm.FILM_GEO] = download.getGeoLocation().getDescription();
166-
}
167-
} catch (Exception ex) {
168-
Log.errorLog(496583211, ex, "add film failed: " + video.getWebsiteUrl());
169-
}
131+
if (video != null) {
132+
try {
133+
DownloadDTO download = video.getDownloadDto();
134+
135+
DatenFilm film = new ZdfDatenFilm(SENDERNAME, video.getTopic(), video.getWebsiteUrl() /*urlThema*/,
136+
video.getTitle(), download.getUrl(Qualities.NORMAL), "" /*urlRtmp*/,
137+
video.getDate(), video.getTime(), video.getDuration(), video.getDescription());
138+
urlTauschen(film, video.getWebsiteUrl(), mlibFilmeSuchen);
139+
140+
//don´t use addFilm here
141+
if (mlibFilmeSuchen.listeFilmeNeu.addFilmVomSender(film)) {
142+
// dann ist er neu
143+
FilmeSuchen.listeSenderLaufen.inc(film.arr[DatenFilm.FILM_SENDER], RunSender.Count.FILME);
144+
}
145+
146+
if (!download.getUrl(Qualities.HD).isEmpty()) {
147+
CrawlerTool.addUrlHd(film, download.getUrl(Qualities.HD), "");
148+
}
149+
if (!download.getUrl(Qualities.SMALL).isEmpty()) {
150+
CrawlerTool.addUrlKlein(film, download.getUrl(Qualities.SMALL), "");
151+
}
152+
if (!download.getSubTitleUrl().isEmpty()) {
153+
CrawlerTool.addUrlSubtitle(film, download.getSubTitleUrl());
154+
}
155+
if (download.getGeoLocation() != GeoLocations.GEO_NONE) {
156+
film.arr[DatenFilm.FILM_GEO] = download.getGeoLocation().getDescription();
157+
}
158+
} catch (Exception ex) {
159+
Log.errorLog(496583211, ex, "add film failed: " + video.getWebsiteUrl());
160+
}
170161
}
171-
phaser.arrive();
172-
}
162+
phaser.arriveAndDeregister();
163+
}
173164
}
174165

175166
private static void updateHdStatus(DatenFilm film, String urlSeite) {
@@ -213,39 +204,30 @@ public static void urlTauschen(DatenFilm film, String urlSeite, FilmeSuchen mSFi
213204
updateHdStatus(film, urlSeite);
214205
}
215206

216-
private static void changeUrl(String from, String to, DatenFilm film, String urlSeite, FilmeSuchen mSFilmeSuchen)
217-
{
218-
if (film.arr[DatenFilm.FILM_URL].endsWith(from))
219-
{
207+
private static void changeUrl(String from, String to, DatenFilm film, String urlSeite, FilmeSuchen mSFilmeSuchen) {
208+
if (film.arr[DatenFilm.FILM_URL].endsWith(from)) {
220209
String url = film.arr[DatenFilm.FILM_URL].substring(0, film.arr[DatenFilm.FILM_URL].lastIndexOf(from)) + to;
221210
String l = mSFilmeSuchen.listeFilmeAlt.getFileSizeUrl(url);
222211
// zum Testen immer machen!!
223-
if (!l.isEmpty())
224-
{
212+
if (!l.isEmpty()) {
225213
film.arr[DatenFilm.FILM_GROESSE] = l;
226214
film.arr[DatenFilm.FILM_URL] = url;
227-
} else if (urlExists(url))
228-
{
215+
} else if (urlExists(url)) {
229216
// dann wars wohl nur ein "403er"
230217
film.arr[DatenFilm.FILM_URL] = url;
231-
} else
232-
{
218+
} else {
233219
Log.errorLog(945120369, "urlTauschen: " + urlSeite);
234220
}
235221
}
236222
}
237223

238-
private static void updateHd(String from, String to, DatenFilm film, String urlSeite)
239-
{
240-
if (film.arr[DatenFilm.FILM_URL_HD].isEmpty() && film.arr[DatenFilm.FILM_URL].endsWith(from))
241-
{
224+
private static void updateHd(String from, String to, DatenFilm film, String urlSeite) {
225+
if (film.arr[DatenFilm.FILM_URL_HD].isEmpty() && film.arr[DatenFilm.FILM_URL].endsWith(from)) {
242226
String url = film.arr[DatenFilm.FILM_URL].substring(0, film.arr[DatenFilm.FILM_URL].lastIndexOf(from)) + to;
243227
// zum Testen immer machen!!
244-
if (urlExists(url))
245-
{
228+
if (urlExists(url)) {
246229
CrawlerTool.addUrlHd(film, url, "");
247-
} else
248-
{
230+
} else {
249231
Log.errorLog(945120147, "urlTauschen: " + urlSeite);
250232
}
251233
}

0 commit comments

Comments
 (0)