exploit.py

import sys
import logging
from modules import hash_sha256, translate_to_english, find_one_document, datetime_to_string, insert_into_mongo, retrieve_key_list, contains_any_word, read_config, initialize_driver, update_post_counts_new
from datetime import datetime
import requests
import pytz
from lxml import html
import time
import requests
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
import time
from random import randint
import json

logging.basicConfig(filename='exploit.log', 
                    filemode='a', 
                    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
                    level=logging.INFO)
# Get the logger instance
logger = logging.getLogger(__name__)

def log_in_to_exploit(driver):
    driver.get("https://forum.exploit.in")
    original_window = driver.current_window_handle
    try:
        WebDriverWait(driver, 20).until(EC.frame_to_be_available_and_switch_to_it((By.XPATH,"//iframe[@title='Widget containing a Cloudflare security challenge']")))
        logger.info("Challenge detected. Attempting to respond.")
        WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//label[@class='ctp-checkbox-label']"))).click()
        time.sleep(20)
    except TimeoutException:
        # Element not found within the time limit
        logger.info("No challenge detected. Continuing...")
    except Exception as e:
        logger.error(e)
        
    input("Press enter after logging in")

def retrieve_cookies(driver, url):
    driver.get(url)
    try:
        WebDriverWait(driver, 20).until(EC.frame_to_be_available_and_switch_to_it((By.XPATH,"//iframe[@title='Widget containing a Cloudflare security challenge']")))
        logger.info("Challenge detected. Attempting to respond.")
        WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//label[@class='ctp-checkbox-label']"))).click()
        time.sleep(20)
    except TimeoutException:
        # Element not found within the time limit
        logger.info("No challenge detected. Continuing with retrieve_cookies.")
    except Exception as e:
        logger.error(e)
    time.sleep(20)
    try:
        cookies = driver.get_cookies()
        for cookie in cookies:
            # Step 1: Load the existing JSON configuration
            config = read_config("config.json")

            # Step 2: Update the JSON configuration with new cookie values
            for cookie in cookies:
                if cookie['name'] in config:
                    config[cookie['name']] = cookie['value']

            # Step 3: Write the updated configuration back to config.json
            with open('config.json', 'w') as file:
                json.dump(config, file, indent=4)
    except Exception as e:
        logger.error(f"Failed to retrieve cookies: {e}")

def send_session_request(url):
    config = read_config("config.json")
    ips4_IPSSessionFront = config.get('ips4_IPSSessionFront')
    ips4_device_key = config.get('ips4_device_key')
    ips4_guestTime = config.get('ips4_guestTime')
    ips4_hasJS = config.get('ips4_hasJS')
    ips4_ipsTimezone = config.get('ips4_ipsTimezone')
    ips4_loggedIn = config.get('ips4_loggedIn')
    ips4_login_key = config.get('ips4_login_key')
    ips4_member_id = config.get('ips4_member_id')
    user_agent = config.get('user_agent_string')
    # Initialize a session
    session = requests.Session()

    # Set custom headers
    headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
        'Accept-Encoding': 'gzip, deflate, br, zstd',
        'Accept-Language': 'en-US,en;q=0.5',
        'User-Agent': user_agent
        # Add other necessary headers here
    }

    # Set cookies received from your session
    cookies = {
        'ips4_IPSSessionFront': ips4_IPSSessionFront,
        'ips4_device_key': ips4_device_key,
        'ips4_guestTime': ips4_guestTime,
        'ips4_hasJS': ips4_hasJS,
        'ips4_ipsTimezone': ips4_ipsTimezone,
        'ips4_loggedIn': ips4_loggedIn,
        'ips4_login_key': ips4_login_key,
        'ips4_member_id': ips4_member_id
    }

    # Update session headers and cookies
    session.headers.update(headers)
    session.cookies.update(cookies)

    # Perform the GET request
    response = session.get(url)

    # Return the response text or other data as needed
    return response.text  

def extract_categories_and_subforums():
    logger.info("Starting extract_categories_and_subforums")
    categories = []
    subforums = []
    response_text = send_session_request('https://forum.exploit.in/')
    tree = html.fromstring(response_text)
    elements = tree.xpath('//ol[@class="ipsList_reset cForumList"]')
    li_elements = elements[0].xpath('.//li[contains(@class, "cForumRow ipsBox ipsSpacer_bottom")]')
    for li_element in li_elements:
        # Finding the <h2> element with the specific class within the current li_element
        h2_element = li_element.xpath('.//h2[contains(@class, "ipsType_sectionTitle ipsType_reset cForumTitle")]')

        # Check if the h2_element list is not empty
        if h2_element:
            # Finding the <a> tag inside the <h2> element, specifically the one containing the href of interest
            a_tag = h2_element[0].xpath('.//a[@href and not(contains(@class, "ipsPos_right ipsJS_show ipsType_noUnderline cForumToggle"))]')

            # Check if the a_tag list is not empty
            if a_tag:
                # Extracting the href attribute
                cat_link = a_tag[0].get('href')

                # Extracting the text, strip() is used to remove leading/trailing whitespace
                cat_name = a_tag[0].text.strip() if a_tag[0].text else ""
                cat_info = {"category": cat_name, "category_link": cat_link}
                categories.append(cat_info)
        ol_element = li_element.xpath('.//ol[contains(@class, "ipsDataList ipsDataList_large ipsDataList_zebra ipsAreaBackground_reset")]')
        li_sub_elements = ol_element[0].xpath('.//li[contains(@class, "cForumRow ipsDataItem ipsDataItem_responsivePhoto ipsDataItem_unread ipsClearfix")]')
        for li_sub_element in li_sub_elements:
            div_elements = li_sub_element.xpath('.//div[contains(@class, "ipsDataItem_main")]')
            h4_elements = div_elements[0].xpath('.//h4[contains(@class, "ipsDataItem_title ipsType_large ipsType_break")]')
            if h4_elements:
                # Extracting the <a> tag
                a_tag = h4_elements[0].xpath('.//a')

                # Check if the a_tag list is not empty
                if a_tag:
                    # Extracting the href attribute
                    sub_link = a_tag[0].get('href')

                    # Extracting the text, strip() is used to remove leading/trailing whitespace
                    subforum_title = a_tag[0].text.strip() if a_tag[0].text else ""
                    subforum_info = {"subforum_link": sub_link, "category": cat_name, "category_link": cat_link, "forum": "Exploit", "forum_link": "https://forum.exploit.in", "subforum_title": subforum_title}
                    subforums.append(subforum_info)
            ul_elements = div_elements[0].xpath('.//ul[contains(@class, "ipsDataItem_subList")]')
            if ul_elements:
                # li_inner_elements = ul_elements[0].xpath('.//li[contains(@class, "ipsDataItem_unread") or not(@class)]')
                li_inner_elements = ul_elements[0].xpath('.//li[contains(@class, "ipsDataItem_unread") or not(@class) or @class=""]')

                for li_inner_element in li_inner_elements:
                    a_tag = li_inner_element.xpath('.//a')
                    if a_tag:
                        sub_sub_link = a_tag[0].get('href')

                        # Using XPath string() function to get all text inside the <a> tag
                        sub_subforum = ''.join(a_tag[0].xpath('string()')).strip()

                        subforum_info = {"subforum_link": sub_sub_link, "category": cat_name, "category_link": cat_link, "forum": "Exploit", "forum_link": "https://forum.exploit.in", "subforum_title": sub_subforum}
                        subforums.append(subforum_info)

    try:
        try:
            existing_document = find_one_document('forum', 'Exploit', 'forums', 'dw_scrapers')

            daily = existing_document.get('daily')
            weekly = existing_document.get('weekly')
            monthly = existing_document.get('monthly')
            total = existing_document.get('total')
            last_updated = existing_document.get('last_updated')
            forum_info = {'forum': 'Exploit', 'forum_link': 'https://forum.exploit.in', 'categories': categories, 'daily': daily, 'weekly':weekly, 'monthly': monthly, 'total': total, 'last_updated': last_updated}
            response = insert_into_mongo('dw_scrapers', 'forums', forum_info, 'forum')
            if 200 <= response.status_code < 300:
                logger.info(response.text)
            else:
                logger.error(f"API call failed with status code {response.status_code}: {response.text}")
        except:
            daily = 0
            weekly = 0
            monthly = 0
            total = 0
            last_updated = datetime_to_string(datetime.now(pytz.utc))
            forum_info = {'forum': 'Exploit', 'forum_link': 'https://forum.exploit.in', 'categories': categories, 'daily': daily, 'weekly':weekly, 'monthly': monthly, 'total': total, 'last_updated': last_updated}
            response = insert_into_mongo('dw_scrapers', 'forums', forum_info, 'forum')
            if 200 <= response.status_code < 300:
                logger.info(response.text)
            else:
                logger.error(f"API call failed with status code {response.status_code}: {response.text}")
    except requests.exceptions.RequestException as e:
        logger.error(f"API request failed: {e}")
    
    try:
        response = insert_into_mongo('dw_scrapers', 'exploit_subforums', subforums, 'subforum_link')
        if 200 <= response.status_code < 300:
            logger.info(response.text)
        else:
            logger.error(f"API call failed with status code {response.status_code}: {response.text}")
    except requests.exceptions.RequestException as e:
        logger.error(f"API request failed: {e}")

def find_next_page(tree, current_page):
    # XPath to find the 'next' page link within a specific <li> class
    next_page_xpath = "//li[contains(@class, 'ipsPagination_next')]/a/@href"

    # Find the href attribute of the next page link
    next_page = tree.xpath(next_page_xpath)

    if next_page:
        # Extract the page number from the 'next page' URL
        next_page_num = int(next_page[0].split('=')[-1])

        # Check if the 'next page' number is greater than the current page number
        if next_page_num > current_page:
            return next_page[0]

    return None

def extract_threads():
    config = read_config("config.json")
    crawl_first = config.get('crawl_threads')
    forum_instance = find_one_document('forum', 'Exploit', 'forums', 'dw_scrapers')
    forum_link = forum_instance.get('forum_link')

    # Removing the last "/" if it exists
    if forum_link.endswith('/'):
        forum_link = forum_link[:-1]      
    categories = retrieve_key_list('exploit_subforums', 'category', 'dw_scrapers')
    cat_links = retrieve_key_list('exploit_subforums', 'category_link', 'dw_scrapers')
    sub_titles = retrieve_key_list('exploit_subforums', 'subforum_title', 'dw_scrapers')
    sub_links = retrieve_key_list('exploit_subforums', 'subforum_link', 'dw_scrapers')
    for cat, cat_link, sub_title, sub_link in zip(categories, cat_links, sub_titles, sub_links):  # Iterate over list of dictionaries
        base_url = sub_link
        page_number = 1
        
        while True:
            try:
                threads_info_list = []
                url = f"{base_url}?page={page_number}" if page_number > 1 else base_url
                logger.info(f"Requesting {url}")
                sleep_time = randint(1, 5)
                time.sleep(sleep_time)                
                response_text = send_session_request(url)
                tree = html.fromstring(response_text)
                try:
                    div_element = tree.xpath("//div[@data-controller='forums.front.forum.forumPage']")
                except Exception as e:
                    logger.error("Failed to extract the div_element")
                    logger.error(e)
                    break
                try:
                    ol_element = div_element[0].xpath(".//ol[contains(@class, 'ipsDataList') and contains(@class, 'cForumTopicTable') and contains(@class, 'cTopicList')]")
                except Exception as e:
                    logger.error("Failed to extract the ol_element")
                    logger.error(e)
                    break
                try:
                    
                    li_elements = ol_element[0].xpath(".//li[contains(@class, 'ipsDataItem') and contains(@class, 'ipsDataItem_responsivePhoto')]")

                except Exception as e:
                    logger.error("Failed to extract li elements")
                    logger.error(e)
                    break
                for li_element in li_elements:
                    try:
                        # Find the <a> element within the <div class="ipsDataItem_main">
                        a_element = li_element.xpath(".//div[contains(@class, 'ipsDataItem_main')]//h4[contains(@class, 'ipsDataItem_title')]//a[@href and not(contains(@class, 'ipsPagination'))]")
                        if a_element:
                            # Extract the href attribute
                            href = a_element[0].get('href')
                            # Extract the text content. Strip() is used to remove leading/trailing whitespace
                            text = a_element[0].text_content().strip()
                            if crawl_first == True:
                                try:
                                    with open('keywords.txt', 'r') as file:
                                        # Read the file content and split by new lines
                                        word_list = file.read().splitlines()
                                    word_found = contains_any_word(text, word_list)
                                    if word_found == False:
                                        continue
                                except Exception as e:
                                    logger.error(f"{e}")
                                    continue
                            thread_title_eng = translate_to_english(text)
                            thread_info = {"thread_link": href, "category": cat, "category_link": cat_link, "forum": "Exploit", "forum_link": forum_link, "subforum_link": sub_link, "subforum_title": sub_title, "thread_title": text, "thread_title_in_english": thread_title_eng}
                            threads_info_list.append(thread_info)
                        else:
                            logger.error("No matching <a> element found within li_element")
                            continue
                    except Exception as e:
                        logger.error("Failed to extract information from li_element")
                        logger.error(e)
                        continue
                
                try:
                    response = insert_into_mongo('dw_scrapers', 'exploit_threads', threads_info_list, 'thread_link')
                    if 200 <= response.status_code < 300:
                        logger.info(response.text)
                    else:
                        logger.error(f"API call failed with status code {response.status_code}: {response.text}")
                except requests.exceptions.RequestException as e:
                    logger.error(f"API request failed: {e}")
                
                next_page = find_next_page(tree, page_number)
                logger.info(next_page)
                if not next_page:  # Check if next page link is not found
                    logger.info("No next link found")
                    break  # Exit the loop if no next page link is found
                page_number += 1
            except Exception as e:
                logger.error(e)
                break

def extract_user_info(author_pane):
    a_tag = author_pane.xpath('.//h3[contains(@class, "cAuthorPane_author")]/strong/a[contains(@class, "ipsType_break")]')

    if a_tag:
        href = a_tag[0].get('href')
        # The username might be within a span tag inside the a tag
        username_span = a_tag[0].xpath('.//span')
        text = username_span[0].text.strip() if username_span else a_tag[0].text_content().strip()
    else:
        href = None
        text = None
        
    ips_type_break = author_pane.xpath('.//li[@class="ipsType_break"]/text()')
    title = ips_type_break[0].strip() if ips_type_break else None
    # Extracting img src
    img_src = author_pane.xpath('.//li[contains(@class, "cAuthorPane_photo")]//img/@src')
    avatar_src = img_src[0] if img_src else None

    # Extracting reputation
    reputation = author_pane.xpath('.//a[contains(@class, "ipsRepBadge")]/text()')
    reputation_text = ''.join(reputation).strip() if reputation else None

    # Extracting post count
    post_count = author_pane.xpath('.//li[contains(text(), "posts")]/text()')
    post_count_text = post_count[0].strip() if post_count else None

    # Extracting joined text
    joined = author_pane.xpath('.//li[contains(text(), "Joined")]/span/text()')
    joined_text = joined[0].strip() if joined else None

    # Extracting activity text
    activity = author_pane.xpath('.//li[contains(text(), "Activity")]/span/text()')
    activity_text = activity[0].strip() if activity else None
    
    user = {"username": text, "joined": joined_text, "messages": post_count_text, "profile_pic": avatar_src, "profile_url": href, "user_title": title, "reputation": reputation_text, "activity": activity_text}
    return user
    
def extract_posts(driver):
    logger.info(f"Starting extract_posts")
    config = read_config("config.json")
    crawl_first = config.get('crawl_content')
    forum_instance = find_one_document('forum', 'Exploit', 'forums', 'dw_scrapers')
    forum_link = forum_instance.get('forum_link')   
    # Removing the last "/" if it exists
    if forum_link.endswith('/'):
        forum_link = forum_link[:-1] 
    categories = retrieve_key_list('exploit_threads', 'category', 'dw_scrapers')
    cat_links = retrieve_key_list('exploit_threads', 'category_link', 'dw_scrapers')
    sub_titles = retrieve_key_list('exploit_threads', 'subforum_title', 'dw_scrapers')
    sub_links = retrieve_key_list('exploit_threads', 'subforum_link', 'dw_scrapers')
    thread_titles = retrieve_key_list('exploit_threads', 'thread_title', 'dw_scrapers')
    thread_titles_english = retrieve_key_list('exploit_threads', 'thread_title_in_english', 'dw_scrapers')
    thread_links = retrieve_key_list('exploit_threads', 'thread_link', 'dw_scrapers')   
    thread_count = 1


    for cat, cat_link, sub_title, sub_link, thrd_title, thrd_title_engl, thrd_link in zip(categories, cat_links, sub_titles, sub_links, thread_titles, thread_titles_english, thread_links):

        try:
            if thread_count % 100 == 0:
                logger.info("Thread count reached 100, retrieving new cookies.")
                retrieve_cookies(driver, forum_link)
        except Exception as e:
            logger.error(f"Selenium error: {e}")
        thread_count += 1
        
        base_url = thrd_link
        page_number = 1
        main_post = []
        replies = []
        posts_plus_replies = []
        word_found = True
        post_number = 1
        users_information = []
        try:
            while True:
                url = f"{base_url}?page={page_number}" if page_number > 1 else base_url
                logger.info(f"Requesting {url}")
                sleep_time = randint(1, 5)
                time.sleep(sleep_time)                
                response_text = send_session_request(url)
                tree = html.fromstring(response_text)
                div_elements = tree.xpath('//div[contains(@data-controller, "commentFeed,forums.front.topic.view")]')
                el_post_feed = div_elements[0].xpath('.//div[@id="elPostFeed"]')
                articles = el_post_feed[0].xpath('.//article[contains(@class, "cPost")]')
                for article in articles:
                    try:
                        target_div = article.xpath('.//div[@class="ipsColumn ipsColumn_fluid"]')
                        target_div = target_div[0]
                    except:
                        continue

                    # Use specific XPath to select only the relevant 'p' and 'pre' tags
                    relevant_texts = target_div.xpath('.//p//text() | .//pre//text() | .//strong/text() | .//span//text()')
                    # Join the extracted texts
                    cleaned_text = ' '.join(relevant_texts)
                    if crawl_first == True:
                        if post_number == 1:
                            try:
                                with open('keywords.txt', 'r') as file:
                                    # Read the file content and split by new lines
                                    word_list = file.read().splitlines()
                                word_found = contains_any_word(cleaned_text, word_list)
                                if word_found == False:
                                    break
                            except Exception as e:
                                # logger.error(f"Error trying to crawl main post {url}")
                                logger.error(f"{e}")
                                break
                    try:
                        content_english = translate_to_english(cleaned_text)
                    except:
                        content_english = None
                    
                    
                    
                    hrefs = [a.get('href') for a in target_div.xpath('.//a')]
                    srcs = [img.get('src') for img in target_div.xpath('.//img')]
                    try:
                        time_element = target_div.xpath('.//time[@datetime]')
                    
                        datetime_value = time_element[0].get('datetime')
                    except Exception as e:
                        logger.error(e)
                        datetime_value = None
                        
                    author_pane = article.xpath('.//aside[contains(@class, "ipsComment_author") and contains(@class, "cAuthorPane")]')
                    user = extract_user_info(author_pane[0])
                    users_information.append(user)    
                    username = user.get("username")
                    profile_url = user.get("profile_url")
                    if post_number == 1:
                            post_type = "post"
                            post_info = {"username": username, "profile_url": profile_url, "post_content": cleaned_text, "post_content_in_english": content_english, "urls_in_post": hrefs, "imgs_in_post": srcs, "post_number": post_number, "post_type": post_type, "date_posted": datetime_value}
                            main_post.append(post_info)
                            posts_plus_replies.append(post_info)
                    else:
                        post_type = "reply"
                        post_info = {"username": username, "profile_url": profile_url, "post_content": cleaned_text, "post_content_in_english": content_english, "urls_in_post": hrefs, "imgs_in_post": srcs, "post_number": post_number, "post_type": post_type, "date_posted": datetime_value}
                        replies.append(post_info)
                        posts_plus_replies.append(post_info)
                    post_number += 1
                if word_found == False:
                    logger.error(f"No keyword found on main post {url}, skipping...")
                    break
                next_page = find_next_page(tree, page_number)
                logger.info(f"Next Link: {next_page}")
                if not next_page:  # Check if next page link is not found
                    logger.info("No next link found")
                    break  # Exit the loop if no next page link is found
                page_number += 1
            if not main_post:
                continue
            posts_hash = hash_sha256(posts_plus_replies)
            existing_document_hash = None
            existing_document = find_one_document('thread_link', 'value=None', 'exploit_posts', 'dw_scrapers', thrd_link)
            if existing_document:
                existing_document_hash = existing_document.get('hash')
            if existing_document_hash == posts_hash:
                logger.info(f"No changes to {thrd_link}")
                continue 
            elif existing_document_hash != posts_hash and existing_document is not None:
                existing_document_date_scraped = existing_document.get('date_scraped')
                existing_document_unix_time = existing_document.get('date_scraped_unix')

                current_datetime = datetime.now()
                updated_datetime = current_datetime.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]  # Truncate to 6 decimal places
                updated_unix_time = current_datetime.timestamp()
            
                post_information = {"thread_link": thrd_link, "category": cat, "category_link": cat_link, "date_scraped": existing_document_date_scraped, "date_scraped_unix": existing_document_unix_time, "forum": "Exploit", "forum_link": forum_link, "hash": posts_hash, "post": main_post, "replies": replies, "subforum_link": sub_link, "subforum_title": sub_title, "thread_title": thrd_title, "thread_title_in_english": thrd_title_engl, "updated_at": updated_datetime, "updated_at_unix_time": updated_unix_time}
                try:
                    response = insert_into_mongo('dw_scrapers', 'exploit_posts', post_information, 'thread_link')
                    if 200 <= response.status_code < 300:
                        logger.info(response.text)
                    else:
                        logger.error(f"API call failed with status code {response.status_code}: {response.text}")
                        continue
                except requests.exceptions.RequestException as e:
                    logger.error(f"API request failed: {e}")
                    continue
                try:
                    insert_into_mongo('dw_scrapers', 'exploit_users', users_information, 'username')
                except Exception as e:
                    logger.error(f"Failed to upload user information")
                    logger.error(f"{e}")
                try:
                    logger.info("trying to update post count")
                    update_post_counts_new(1, 'forum', 'Exploit', 'forums', 'dw_scrapers')
                except Exception as e:
                    logger.error(f"Failed to update count on thread {thrd_link}")
                    logger.error(f"{e}")
            else:
                current_datetime = datetime.now()
                formatted_datetime = current_datetime.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]  # Truncate to 6 decimal places
                unix_time = current_datetime.timestamp()
                post_information = {"thread_link": thrd_link, "category": cat, "category_link": cat_link, "date_scraped": formatted_datetime, "date_scraped_unix": unix_time, "forum": "Exploit", "forum_link": forum_link, "hash": posts_hash, "post": main_post, "replies": replies, "subforum_link": sub_link, "subforum_title": sub_title, "thread_title": thrd_title, "thread_title_in_english": thrd_title_engl}
                try:
                    response = insert_into_mongo('dw_scrapers', 'exploit_posts', post_information, 'thread_link')
                    if 200 <= response.status_code < 300:
                        logger.info(response.text)
                    else:
                        logger.error(f"API call failed with status code {response.status_code}: {response.text}")
                        continue
                except requests.exceptions.RequestException as e:
                    logger.error(f"API request failed: {e}")
                    continue
                
                try:
                    insert_into_mongo('dw_scrapers', 'exploit_users', users_information, 'username')
                except Exception as e:
                    logger.error(f"Failed to upload user information")
                    logger.error(f"{e}")
                
                try:
                    logger.info("trying to update post count")
                    update_post_counts_new(1, 'forum', 'Exploit', 'forums', 'dw_scrapers')
                except Exception as e:
                    logger.error(f"Failed to update count on thread {thrd_link}")
            
        except Exception as e:
                logger.error(f"Failed to parse the thread {thrd_link}")
                logger.error(f"{e}")
                continue

def main():
    driver, service, firefox_options = initialize_driver()
    log_in_to_exploit(driver)
    retrieve_cookies(driver, "https://forum.exploit.in")
    extract_categories_and_subforums()
    extract_threads()
    extract_posts(driver)

if __name__ == "__main__":
    main()