Merge pull request #927 from mediathekview/dev/livestreamImport

Dev/livestream import

Author: codingPF

MServer-Config.yaml

@@ -32,6 +32,12 @@ senderIncluded:
 # list of films to be ignored by crawler search
 ignoreFilmlistPath: "ignoreFilmlist.txt"
 
+# list of livestreams to be added
+importLivestreamConfiguration: 
+  active: false
+  path: "live-streams.json"
+  format: OLD_JSON
+
 # The formats in which the filmlist should be saved to.
 # Possible are: JSON, OLD_JSON, JSON_COMPRESSED_XZ, OLD_JSON_COMPRESSED_XZ, JSON_COMPRESSED_GZIP, OLD_JSON_COMPRESSED_BZIP, JSON_COMPRESSED_GZIP, OLD_JSON_COMPRESSED_BZIP
 filmlistSaveFormats:

src/main/java/de/mediathekview/mserver/base/config/ImportLivestreamConfiguration.java

@@ -0,0 +1,58 @@
+package de.mediathekview.mserver.base.config;
+
+import java.util.Objects;
+
+import de.mediathekview.mlib.filmlisten.FilmlistFormats;
+
+public class ImportLivestreamConfiguration {
+  private final Boolean active;
+  private final String path;
+  private final FilmlistFormats format;
+
+  public ImportLivestreamConfiguration(Boolean active, String path, FilmlistFormats format) {
+    this.active = active;
+    this.path = path;
+    this.format = format;
+  }
+  
+  public ImportLivestreamConfiguration() {
+    this.active = null;
+    this.path = null;
+    this.format = null;
+  }
+  
+  public Boolean isActive() {
+    return active;
+  }
+  public String getPath() {
+    return path;
+  }
+  public FilmlistFormats getFormat() {
+    return format;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (!(obj instanceof final ImportLivestreamConfiguration that)) {
+      return false;
+    }
+    if (!super.equals(obj)) {
+      return false;
+    }
+    return Objects.equals(isActive(), that.isActive())
+        && Objects.equals(getPath(), that.getPath())
+        && Objects.equals(getFormat(), that.getFormat());
+  }
+  
+  @Override
+  public int hashCode() {
+    return Objects.hash(
+        super.hashCode(),
+        isActive(),
+        getPath(),
+        getFormat());
+  }
+}

src/main/java/de/mediathekview/mserver/base/config/MServerConfigDTO.java

@@ -16,6 +16,8 @@ public class MServerConfigDTO extends MServerBasicConfigDTO implements ConfigDTO
   private final String filmlistIdFilePath;
   /** ignore certain film by title **/
   private final String ignoreFilmlistPath;
+  /** add livestreams from external list **/
+  private final ImportLivestreamConfiguration importLivestreamConfiguration;
   /** The maximum amount of cpu threads to be used. */
   private Integer maximumCpuThreads;
   /**
@@ -86,7 +88,8 @@ public MServerConfigDTO() {
     writeFilmlistIdFileEnabled = true;
     filmlistIdFilePath = "filmlist.id";
     ignoreFilmlistPath = "ignoreFilmlist.txt";
-
+    importLivestreamConfiguration = new ImportLivestreamConfiguration(false, "live-streams.json", FilmlistFormats.OLD_JSON);
+    
     Arrays.stream(Sender.values())
         .forEach(sender -> senderConfigurations.put(sender, new MServerBasicConfigDTO(this)));
   }
@@ -235,6 +238,11 @@ public String getIgnoreFilmslistPath() {
     return ignoreFilmlistPath;
   }
 
+  public ImportLivestreamConfiguration getImportLivestreamConfiguration() {
+    return importLivestreamConfiguration;
+  }
+  
+ 
   /**
    * Loads the {@link Sender} specific configuration and if it not exist creates one.
    *
@@ -285,7 +293,8 @@ && getFilmlistImportFormat() == that.getFilmlistImportFormat()
         && Objects.equals(getFilmlistHashFilePath(), that.getFilmlistHashFilePath())
         && Objects.equals(getWriteFilmlistIdFileEnabled(), that.getWriteFilmlistIdFileEnabled())
         && Objects.equals(getFilmlistIdFilePath(), that.getFilmlistIdFilePath())
-        && Objects.equals(getIgnoreFilmslistPath(), that.getIgnoreFilmslistPath());
+        && Objects.equals(getIgnoreFilmslistPath(), that.getIgnoreFilmslistPath())
+        && Objects.equals(getImportLivestreamConfiguration(), that.getImportLivestreamConfiguration());
   }
 
   @Override
@@ -310,7 +319,8 @@ public int hashCode() {
         getFilmlistHashFilePath(),
         getWriteFilmlistIdFileEnabled(),
         getFilmlistIdFilePath(),
-        getIgnoreFilmslistPath());
+        getIgnoreFilmslistPath(),
+        getImportLivestreamConfiguration());
   }
 
   public void initializeSenderConfigurations() {

src/main/java/de/mediathekview/mserver/crawler/CrawlerManager.java

@@ -155,6 +155,30 @@ public void importFilmlist() {
       importFilmlist(config.getFilmlistImportFormat(), config.getFilmlistImportLocation());
     }
   }
+  
+  public void importLivestreamFilmlist() {
+    if (Boolean.TRUE.equals(config.getImportLivestreamConfiguration().isActive())) {
+      importLivestreamFilmlist(config.getImportLivestreamConfiguration().getFormat(), config.getImportLivestreamConfiguration().getPath());
+    }
+  }
+  
+  public void importLivestreamFilmlist(final FilmlistFormats aFormat, final String aFilmlistLocation) {
+    try {
+      final Optional<Filmlist> importedFilmlist;
+      if (aFilmlistLocation.startsWith(HTTP)) {
+        importedFilmlist = importFilmListFromURl(aFormat, aFilmlistLocation);
+      } else {
+        importedFilmlist = importFilmlistFromFile(aFormat, aFilmlistLocation);
+      }
+      // remove livestreams
+      filmlist.getFilms().entrySet().removeIf(entry -> entry.getValue().getThema().equalsIgnoreCase("Livestream"));
+      // add new
+      importedFilmlist.ifPresent( imp -> imp.getFilms().entrySet().forEach( entry -> filmlist.add(entry.getValue())));
+      //
+    } catch (final IOException ioException) {
+      LOG.fatal(String.format(FILMLIST_IMPORT_ERROR_TEMPLATE, aFilmlistLocation), ioException);
+    }
+  }
 
   /**
    * Imports the film list with the given {@link FilmlistFormats} and the given location.
@@ -503,4 +527,9 @@ private void runCrawlers(final AbstractCrawler... aCrawlers) {
       Thread.currentThread().interrupt();
     }
   }
+  
+  // Added to allow JUNIT tests
+  public Filmlist getFilmlist() {
+    return filmlist;
+  }
 }

src/main/java/de/mediathekview/mserver/ui/config/MServerConfigUI.java

@@ -101,6 +101,9 @@ void start() {
       if (config.getFilmlistImporEnabled() != null && config.getFilmlistImporEnabled()) {
         manager.importFilmlist();
       }
+      if (Boolean.TRUE.equals(config.getImportLivestreamConfiguration().isActive())) {
+        manager.importLivestreamFilmlist();
+      }
     } finally {
       manager.filterFilmlist();
       manager.saveFilmlist();

src/main/resources/MServer-Config.yaml

@@ -26,6 +26,12 @@ senderIncluded:
 # list of films to be ignored by crawler search
 ignoreFilmlistPath: "ignoreFilmlist.txt"
 
+# list of livestreams to be added
+importLivestreamConfiguration: 
+  active: false
+  path: "src/main/resources/live-streams.json"
+  format: OLD_JSON
+
 # The formats in which the filmlist should be saved to.
 # Possible are: JSON, OLD_JSON, JSON_COMPRESSED_XZ, OLD_JSON_COMPRESSED_XZ, JSON_COMPRESSED_GZIP, OLD_JSON_COMPRESSED_BZIP, JSON_COMPRESSED_GZIP, OLD_JSON_COMPRESSED_BZIP
 filmlistSaveFormats:

src/main/resources/live-streams.json

@@ -0,0 +1 @@
+{"Filmliste":["20.08.2023, 14:22","20.08.2023, 14:22","3.1.213"," [Vers.: 3.1.213 ]","b37878aa-d85f-41d1-89ab-053ccf278b2f"],"Filmliste":["Sender","Thema","Titel","Datum","Zeit","Dauer","Größe [MB]","Beschreibung","Url","Website","Url Untertitel","Url RTMP","Url Klein","Url RTMP Klein","Url HD","Url RTMP HD","DatumL","Url History","Geo","neu"],"X":["ARD","Livestream","tagesschau24 Livestream","","","00:00:00","0","","https://tagesschau.akamaized.net/hls/live/2020115/tagesschau/tagesschau_1/master.m3u8","","","","","","","","0","","","false"],"X":["","","ARD-alpha Livestream","","","00:00:00","0","","https://mcdn.br.de/br/fs/ard_alpha/hls/de/master.m3u8","","","","","","","","0","","","false"],"X":["","","one HD Livestream","","","00:00:00","0","","https://mcdn.one.ard.de/ardone/hls/master.m3u8","","","","","","","","0","","","false"],"X":["","","Das Erste HD Livestream","","","00:00:00","0","","https://mcdn.daserste.de/daserste/de/master.m3u8","","","","","","","","0","","","false"],"X":["ARTE.DE","","ARTE HD Livestream","","","00:00:00","0","","https://artesimulcast.akamaized.net/hls/live/2030993/artelive_de/master.m3u8","","","","","","","","0","","","false"],"X":["ARTE.FR","","ARTE HD (FR) Livestream","","","00:00:00","0","","https://artesimulcast.akamaized.net/hls/live/2031003/artelive_fr/master.m3u8","","","","","","","","0","","","false"],"X":["BR","","BR Fernsehen Nord HD Livestream","","","00:00:00","0","","https://mcdn.br.de/br/fs/bfs_nord/hls/de/master.m3u8","","","","","","","","0","","","false"],"X":["","","BR Fernsehen Süd HD Livestream","","","00:00:00","0","","https://brcdn.vo.llnwd.net/br/fs/bfs_sued/hls/de/master.m3u8","","","","","","","","0","","","false"],"X":["3Sat","","3sat Livestream","","","00:00:00","0","","https://zdf-hls-18.akamaized.net/hls/live/2016501/dach/veryhigh/master.m3u8","","","","","","","","0","","","false"],"X":["DW","","Deutsche Welle Livestream","","","00:00:00","0","","https://dwamdstream111.akamaized.net/hls/live/2017972/dwstream111/index.m3u8","","","","","","","","0","","","false"],"X":["","","Deutsche Welle (ES) Livestream","","","00:00:00","0","","https://dwamdstream109.akamaized.net/hls/live/2017970/dwstream109/index.m3u8","","","","","","","","0","","","false"],"X":["","","Deutsche Welle+ Livestream","","","00:00:00","0","","https://dwamdstream110.akamaized.net/hls/live/2017971/dwstream110/index.m3u8","","","","","","","","0","","","false"],"X":["","","Deutsche Welle (EN) Livestream","","","00:00:00","0","","https://dwamdstream107.akamaized.net/hls/live/2017968/dwstream107/index.m3u8","","","","","","","","0","","","false"],"X":["HR","","hr-fernsehen Livestream","","","00:00:00","0","","https://hrhlsde.akamaized.net/hls/live/2024526/hrhlsde/index.m3u8","","","","","","","","0","","","false"],"X":["KiKA","","KiKA Livestream","","","00:00:00","0","","https://kikageohls.akamaized.net/hls/live/2022693/livetvkika_de/master.m3u8","","","","","","","","0","","","false"],"X":["MDR","","MDR Sachsen HD Livestream","","","00:00:00","0","","https://mdrtvsnhls.akamaized.net/hls/live/2016928/mdrtvsn/index.m3u8","","","","","","","","0","","","false"],"X":["","","MDR Thüringen HD Livestream","","","00:00:00","0","","https://mdrtvthhls.akamaized.net/hls/live/2016880/mdrtvth/index.m3u8","","","","","","","","0","","","false"],"X":["","","MDR Sachsen-Anhalt HD Livestream","","","00:00:00","0","","https://mdrtvsahls.akamaized.net/hls/live/2016879/mdrtvsa/index.m3u8","","","","","","","","0","","","false"],"X":["NDR","","NDR Schleswig-Holstein HD Livestream","","","00:00:00","0","","https://mcdn.ndr.de/ndr/hls/ndr_fs/ndr_sh/master.m3u8","","","","","","","","0","","","false"],"X":["","","NDR Mecklenburg-Vorpommern HD Livestream","","","00:00:00","0","","https://mcdn.ndr.de/ndr/hls/ndr_fs/ndr_mv/master.m3u8","","","","","","","","0","","","false"],"X":["","","NDR Niedersachsen HD Livestream","","","00:00:00","0","","https://mcdn.ndr.de/ndr/hls/ndr_fs/ndr_nds/master.m3u8","","","","","","","","0","","","false"],"X":["","","NDR Hamburg HD Livestream","","","00:00:00","0","","https://mcdn.ndr.de/ndr/hls/ndr_fs/ndr_hh/master.m3u8","","","","","","","","0","","","false"],"X":["ORF","","ORF eins HD Livestream","","","00:00:00","0","","https://orf1.mdn.ors.at/out/u/orf1/q6a/manifest_6.m3u8","","","","","","","","0","","","false"],"X":["","","ORF SPORT + Livestream","","","00:00:00","0","","https://orfs.mdn.ors.at/out/u/orfs/q6a/manifest_6.m3u8","","","","","","","","0","","","false"],"X":["","","ORF III HD Livestream","","","00:00:00","0","","https://orf3.mdn.ors.at/out/u/orf3/q6a/manifest_6.m3u8","","","","","","","","0","","","false"],"X":["","","ORF 2 HD Livestream","","","00:00:00","0","","https://orf2.mdn.ors.at/out/u/orf2/qxb/manifest.m3u8","","","","","","","","0","","","false"],"X":["PHOENIX","","phoenix Livestream","","","00:00:00","0","","https://zdf-hls-19.akamaized.net/hls/live/2016502/de/veryhigh/master.m3u8","","","","","","","","0","","","false"],"X":["RBB","","rbb Brandenburg HD Livestream","","","00:00:00","0","","https://rbb-hls-brandenburg.akamaized.net/hls/live/2017825/rbb_brandenburg/master.m3u8","","","","","","","","0","","","false"],"X":["","","rbb Berlin HD Livestream","","","00:00:00","0","","https://rbb-hls-berlin.akamaized.net/hls/live/2017824/rbb_berlin/master.m3u8","","","","","","","","0","","","false"],"X":["SR","","SR Fernsehen HD Livestream","","","00:00:00","0","","https://srfs.akamaized.net/hls/live/689649/srfsgeo/index.m3u8","","","","","","","","0","","","false"],"X":["SWR","","SWR Rheinland-Pfalz HD Livestream","","","00:00:00","0","","https://swrrpd-hls.akamaized.net/hls/live/2018676/swrrpd/master.m3u8","","","","","","","","0","","","false"],"X":["","","SWR Baden-Württemberg HD Livestream","","","00:00:00","0","","https://swrbwd-hls.akamaized.net/hls/live/2018672/swrbwd/master.m3u8","","","","","","","","0","","","false"],"X":["WDR","","WDR Aachen HD Livestream","","","00:00:00","0","","https://wdrlokalzeit.akamaized.net/hls/live/2018019-b/wdrlz_aachen/master.m3u8","","","","","","","","0","","","false"],"X":["","","WDR Köln HD Livestream","","","00:00:00","0","","https://wdrlokalzeit.akamaized.net/hls/live/2023550-b/wdrlz_koeln/master.m3u8","","","","","","","","0","","","false"],"X":["","","WDR Duisburg HD Livestream","","","00:00:00","0","","https://wdrlokalzeit.akamaized.net/hls/live/2018024-b/wdrlz_duisburg/master.m3u8","","","","","","","","0","","","false"],"X":["","","WDR HD Livestream","","","00:00:00","0","","https://wdrfs247.akamaized.net/hls/live/681509/wdr_msl4_fs247/index.m3u8","","","","","","","","0","","","false"],"X":["","","WDR Ruhr HD Livestream","","","00:00:00","0","","https://wdrlokalzeit.akamaized.net/hls/live/2018027-b/wdrlz_essen/master.m3u8","","","","","","","","0","","","false"],"X":["","","WDR Düsseldorf HD Livestream","","","00:00:00","0","","https://wdrlokalzeit.akamaized.net/hls/live/2018023-b/wdrlz_duesseldorf/master.m3u8","","","","","","","","0","","","false"],"X":["","","WDR Bonn HD Livestream","","","00:00:00","0","","https://wdrlokalzeit.akamaized.net/hls/live/2018021-b/wdrlz_bonn/master.m3u8","","","","","","","","0","","","false"],"X":["","","WDR Dortmund HD Livestream","","","00:00:00","0","","https://wdrlokalzeit.akamaized.net/hls/live/2018022-b/wdrlz_dortmund/master.m3u8","","","","","","","","0","","","false"],"X":["","","WDR Bergisches Land HD Livestream","","","00:00:00","0","","https://wdrlokalzeit.akamaized.net/hls/live/2018028-b/wdrlz_wuppertal/master.m3u8","","","","","","","","0","","","false"],"X":["","","WDR Münsterland HD Livestream","","","00:00:00","0","","https://wdrlokalzeit.akamaized.net/hls/live/2018025-b/wdrlz_muensterland/master.m3u8","","","","","","","","0","","","false"],"X":["","","WDR Ostwestfalen HD Livestream","","","00:00:00","0","","https://wdrlokalzeit.akamaized.net/hls/live/2018026-b/wdrlz_bielefeld/master.m3u8","","","","","","","","0","","","false"],"X":["","","WDR Südwestfalen HD Livestream","","","00:00:00","0","","https://wdrlokalzeit.akamaized.net/hls/live/2018020-b/wdrlz_siegen/master.m3u8","","","","","","","","0","","","false"],"X":["ZDF","","ZDF HD Livestream","","","00:00:00","0","","https://zdf-hls-15.akamaized.net/hls/live/2016498/de/veryhigh/master.m3u8","","","","","","","","0","","","false"],"X":["","","ZDFinfo HD Livestream","","","00:00:00","0","","https://zdf-hls-17.akamaized.net/hls/live/2016500/de/veryhigh/master.m3u8","","","","","","","","0","","","false"],"X":["","","ZDFneo HD Livestream","","","00:00:00","0","","https://zdf-hls-16.akamaized.net/hls/live/2016499/de/veryhigh/master.m3u8","","","","","","","","0","","","false"]}