- huge speed improvements

- fixed tests

Author: EchterAlsFake

eporner_api/eporner_api.py

@@ -23,8 +23,8 @@
 from urllib.parse import urljoin
 from functools import cached_property
 from typing import Generator, Union, Optional
-from base_api.base import BaseCore, setup_logger
 from base_api.modules.config import RuntimeConfig
+from base_api.base import BaseCore, setup_logger, Helper
 
 """
 Copyright (c) 2024-2025 Johannes Habel
@@ -58,7 +58,7 @@
 
 
 class Video:
-    def __init__(self, url: str, enable_html_scraping: bool = False, core: Optional[BaseCore] = None):
+    def __init__(self, url: str, enable_html_scraping: bool = True, core: Optional[BaseCore] = None):
         self.core = core
         self.url = url
         self.enable_html = enable_html_scraping
@@ -360,8 +360,9 @@ def download(self, quality, path, callback=None, mode=Encoding.mp4_h264, no_titl
 
 
 
-class Pornstar:
+class Pornstar(Helper):
     def __init__(self, url: str, enable_html_scraping: bool = False, core: Optional[BaseCore] = None):
+        super().__init__(core=core, video=Video)
         self.core = core
         self.url = url
         self.enable_html_scraping = enable_html_scraping
@@ -371,21 +372,18 @@ def __init__(self, url: str, enable_html_scraping: bool = False, core: Optional[
     def enable_logging(self, log_file: str, level, log_ip: str = None, log_port: int = None):
         self.logger = setup_logger(name="EPorner API - [Pornstar]", log_file=log_file, level=level, http_ip=log_ip, http_port=log_port)
 
-    def videos(self, pages: int = 0) -> Generator[Video, None, None]:
+    def videos(self, pages: int = 0, videos_concurrency: int = None, pages_concurrency: int = None) -> Generator[Video, None, None]:
         if pages == 0:
-            pages = int(self.video_amount) / 37 # One page contains 37 videos
+            video_amount = str(self.video_amount).replace(",", "")
+            pages = round(int(video_amount)) / 37 # One page contains 37 videos
 
-        urls = []
-        for page in range(1, pages):
-            response = self.core.fetch(urljoin(self.url + "/", str(page)))
-            extraction = REGEX_SCRAPE_VIDEO_URLS.findall(response)
-            for url in extraction:
-                url = f"https://www.eporner.com{url}"
-                url = url.replace("EPTHBN/", "")
-                urls.append(url)
+        videos_concurrency = videos_concurrency or self.core.config.videos_concurrency
+        pages_concurrency = pages_concurrency or self.core.config.pages_concurrency
 
-        for url in urls:
-            yield Video(url, enable_html_scraping=self.enable_html_scraping)
+        pages = round(pages) # Dont ask
+        page_urls = [urljoin(f"{self.url}/", str(page)) for page in range(1, pages + 1)]
+        yield from self.iterator(page_urls=page_urls, extractor=extractor, pages_concurrency=pages_concurrency,
+                                 videos_concurrency=videos_concurrency)
 
     @cached_property
     def name(self) -> str:
@@ -491,8 +489,9 @@ def biography(self) -> str:
         return REGEX_PORNSTAR_BIOGRAPHY.search(self.html_content).group(1)
 
 
-class Client:
+class Client(Helper):
     def __init__(self, core: Optional[BaseCore] = None):
+        super().__init__(core, video=Video)
         self.core = core or BaseCore(config=RuntimeConfig())
         self.core.initialize_session()
         self.core.session.headers.update(headers)
@@ -517,19 +516,18 @@ def search_videos(self, query: str, sorting_gay: Union[str, Gay], sorting_order:
         json_data = json.loads(response)
         for video_ in json_data.get("videos", []):  # Don't know why this works lmao
             id_ = video_["url"]
-            print(id_)
             yield Video(id_, enable_html_scraping, core=self.core)
 
-    def get_videos_by_category(self, category: Union[str, Category], enable_html_scraping: bool = False)\
-            -> Generator[Video, None, None]:
-        for page in range(100):
-            self.logger.debug(f"Iterating category page ->: {page}")
-            response = self.core.fetch(f"{ROOT_URL}cat/{category}/{page}")
-            extraction = REGEX_SCRAPE_VIDEO_URLS.findall(response)
-            for url in extraction:
-                url = f"https://www.eporner.com{url}"
-                url = url.replace("EPTHBN/", "")
-                yield Video(url, enable_html_scraping=enable_html_scraping, core=self.core)
+    def get_videos_by_category(self, category: Union[str, Category], enable_html_scraping: bool = False,
+                               videos_concurrency: int = None, pages_concurrency: int = None) -> Generator[Video, None, None]:
+
+        page_urls = [f"{ROOT_URL}cat/{category}/{page}" for page in range(1, 100)]
+
+        videos_concurrency = videos_concurrency or self.core.config.videos_concurrency
+        pages_concurrency = pages_concurrency or self.core.config.pages_concurrency
+        yield from self.iterator(page_urls=page_urls, videos_concurrency=videos_concurrency,
+                                 pages_concurrency=pages_concurrency, extractor=extractor)
+
 
     def get_pornstar(self, url: str, enable_html_scraping: bool = True) -> Pornstar:
         self.logger.info(f"Returning Pornstar object for: {url} HTML Scraping -> {enable_html_scraping}")

eporner_api/modules/consts.py

@@ -42,4 +42,15 @@
 
 headers = {
     "Referer": "https://www.eporner.com/"
-}
+}
+
+
+def extractor(content: str):
+    video_urls = []
+    extraction = REGEX_SCRAPE_VIDEO_URLS.findall(content)
+    for url in extraction:
+        url = f"https://www.eporner.com{url}"
+        url = url.replace("EPTHBN/", "")
+        video_urls.append(url)
+
+    return video_urls

eporner_api/tests/test_category.py

@@ -1,12 +1,16 @@
 import time
 
 from ..eporner_api import Client, Category
+from base_api import BaseCore
+core = BaseCore()
+core.config.pages_concurrency = 1
+core.config.videos_concurrency = 1
 
 
 def test_category():
-    videos_1 = Client().get_videos_by_category(category=Category.JAPANESE)
-    videos_2 = Client().get_videos_by_category(category=Category.HD)
-    videos_3 = Client().get_videos_by_category(category=Category.BLONDE)
+    videos_1 = Client(core).get_videos_by_category(category=Category.JAPANESE)
+    videos_2 = Client(core).get_videos_by_category(category=Category.HD)
+    videos_3 = Client(core).get_videos_by_category(category=Category.BLONDE)
 
     for idx, video in enumerate(videos_1):
         if idx == 3:

eporner_api/tests/test_pornstar.py

@@ -1,13 +1,18 @@
 from ..eporner_api import Client
 import time
 url = "https://www.eporner.com/pornstar/riley-reid/"
-pornstar = Client().get_pornstar(url, enable_html_scraping=True)
+from base_api import BaseCore
+core = BaseCore()
+core.config.pages_concurrency = 1
+core.config.videos_concurrency = 1
+
+pornstar = Client(core).get_pornstar(url, enable_html_scraping=True)
+
 
 def test_videos():
     videos = pornstar.videos(pages=1)
 
     for idx, video in enumerate(videos):
-        time.sleep(5) # Lmao
         assert isinstance(video.title, str) and len(video.title) > 3
         if idx == 5:
             break

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "Eporner_API"
-version = "1.9.2"
+version = "1.9.3"
 description = "A Python API for the Porn Site Eporner.com"
 readme = "README.md"
 requires-python = ">=3.6"