From 8048b6de90a8a614052c89a297da995221d6b46d Mon Sep 17 00:00:00 2001 From: Metfrass <137781779+matteofrassi@users.noreply.github.com> Date: Thu, 19 Feb 2026 16:51:37 +0100 Subject: [PATCH] fix: update broken selectors and switch to DOM-based export Perplexity changed their UI, breaking the original login flow, library page selectors, and API-based thread extraction. Changes: - login.ts: Manual login with OAuth redirect support (Google/Apple) instead of hardcoded email flow with broken selectors - listConversations.ts: Use generic `a[href*="/search/"]` selector instead of removed `data-testid="thread-title"`, dynamically find scrollable container instead of hardcoded class name - exportLibrary.ts: Extract content directly from rendered DOM (`.prose` blocks) and convert HTML to Markdown, replacing the broken `/rest/thread/` API interception approach Tested with 117 conversations, 0 failures. --- src/exportLibrary.ts | 204 +++++++++++++++++++++++++++++++++------ src/listConversations.ts | 82 ++++++++++++---- src/login.ts | 74 ++++++++++---- 3 files changed, 292 insertions(+), 68 deletions(-) diff --git a/src/exportLibrary.ts b/src/exportLibrary.ts index b9ada3f..4b392e0 100644 --- a/src/exportLibrary.ts +++ b/src/exportLibrary.ts @@ -1,11 +1,9 @@ import { promises as fs } from "fs"; import puppeteer from "puppeteer-extra"; -import { Browser } from "puppeteer"; +import { Browser, Page } from "puppeteer"; import StealthPlugin from "puppeteer-extra-plugin-stealth"; -import { ConversationSaver } from "./ConversationSaver"; import { getConversations } from "./listConversations"; import { login } from "./login"; -import renderConversation from "./renderConversation"; import { loadDoneFile, saveDoneFile, sleep } from "./utils"; export interface ExportLibraryOptions { @@ -14,20 +12,149 @@ export interface ExportLibraryOptions { email: string; } +function sanitizeFilename(name: string): string { + return name + .replace(/[/\\?%*:|"<>]/g, "-") + .replace(/\s+/g, " ") + .trim() + .substring(0, 100); +} + +async function extractThreadContent(page: Page): Promise<{ title: string; markdown: string } | null> { + return page.evaluate(() => { + // Helper: convert HTML element to Markdown + function htmlToMd(el: Element): string { + let md = ""; + for (const node of Array.from(el.childNodes)) { + if (node.nodeType === Node.TEXT_NODE) { + md += node.textContent || ""; + } else if (node.nodeType === Node.ELEMENT_NODE) { + const elem = node as Element; + const tag = elem.tagName; + + // Skip citation number spans + if (elem.classList.contains("citation") || elem.classList.contains("citation-nbsp")) { + continue; + } + + if (tag === "P") { + md += "\n\n" + htmlToMd(elem); + } else if (tag === "STRONG" || tag === "B") { + md += "**" + htmlToMd(elem) + "**"; + } else if (tag === "EM" || tag === "I") { + md += "*" + htmlToMd(elem) + "*"; + } else if (tag === "CODE") { + const text = elem.textContent || ""; + if (elem.parentElement?.tagName === "PRE") { + md += text; + } else { + md += "`" + text + "`"; + } + } else if (tag === "PRE") { + const code = elem.querySelector("code"); + const lang = code?.className?.match(/language-(\w+)/)?.[1] || ""; + md += "\n\n```" + lang + "\n" + (code?.textContent || elem.textContent || "") + "\n```\n"; + } else if (tag === "H1") { + md += "\n\n# " + htmlToMd(elem); + } else if (tag === "H2") { + md += "\n\n## " + htmlToMd(elem); + } else if (tag === "H3") { + md += "\n\n### " + htmlToMd(elem); + } else if (tag === "H4") { + md += "\n\n#### " + htmlToMd(elem); + } else if (tag === "UL") { + md += "\n"; + for (const li of Array.from(elem.children)) { + if (li.tagName === "LI") { + md += "\n- " + htmlToMd(li).trim(); + } + } + md += "\n"; + } else if (tag === "OL") { + md += "\n"; + let i = 1; + for (const li of Array.from(elem.children)) { + if (li.tagName === "LI") { + md += "\n" + i + ". " + htmlToMd(li).trim(); + i++; + } + } + md += "\n"; + } else if (tag === "A") { + const href = (elem as HTMLAnchorElement).href; + const text = htmlToMd(elem); + md += "[" + text + "](" + href + ")"; + } else if (tag === "BR") { + md += "\n"; + } else if (tag === "BLOCKQUOTE") { + const inner = htmlToMd(elem).trim().split("\n").map((l: string) => "> " + l).join("\n"); + md += "\n\n" + inner + "\n"; + } else if (tag === "TABLE") { + // Simple table extraction + const rows = Array.from(elem.querySelectorAll("tr")); + if (rows.length > 0) { + md += "\n\n"; + rows.forEach((row, idx) => { + const cells = Array.from(row.querySelectorAll("th, td")); + md += "| " + cells.map((c) => (c.textContent || "").trim()).join(" | ") + " |\n"; + if (idx === 0) { + md += "| " + cells.map(() => "---").join(" | ") + " |\n"; + } + }); + } + } else { + // Default: recurse + md += htmlToMd(elem); + } + } + } + return md; + } + + // Find all Q&A pairs on the page + // The page structure has the query text and then prose blocks for answers + const proseBlocks = Array.from( + document.querySelectorAll(".prose") + ).filter((el) => el.textContent && el.textContent.trim().length > 50); + + if (proseBlocks.length === 0) return null; + + // Get the page title / query from the page + // Usually visible as a heading or the first prominent text + const titleEl = document.querySelector("h1") || + document.querySelector('[class*="query"]') || + document.querySelector("title"); + const title = titleEl?.textContent?.trim() || document.title || "Untitled"; + + // Build markdown from all prose blocks + let fullMd = "# " + title + "\n\n"; + fullMd += "Source: " + window.location.href + "\n\n---\n"; + + for (const block of proseBlocks) { + const content = htmlToMd(block).trim(); + if (content) { + fullMd += "\n\n" + content; + } + } + + // Clean up multiple newlines + fullMd = fullMd.replace(/\n{4,}/g, "\n\n\n"); + + return { title, markdown: fullMd }; + }); +} + export default async function exportLibrary(options: ExportLibraryOptions) { puppeteer.use(StealthPlugin()); - // Create output directory if it doesn't exist await fs.mkdir(options.outputDir, { recursive: true }); - // Load done file const doneFile = await loadDoneFile(options.doneFilePath); console.log( `Loaded ${doneFile.processedUrls.length} processed URLs from done file` ); const browser: Browser = await puppeteer.launch({ - // Authentication is interactive. headless: false, }); @@ -39,34 +166,53 @@ export default async function exportLibrary(options: ExportLibraryOptions) { console.log(`Found ${conversations.length} new conversations to process`); - const conversationSaver = new ConversationSaver(page); - await conversationSaver.initialize(); + let exported = 0; + let failed = 0; - for (const conversation of conversations) { - console.log(`Processing conversation ${conversation.url}`); - - const threadData = await conversationSaver.loadThreadFromURL( - conversation.url + for (let i = 0; i < conversations.length; i++) { + const conv = conversations[i]; + console.log( + `[${i + 1}/${conversations.length}] ${conv.title.substring(0, 60)}` ); - // place the thread data in the output directory - await fs.writeFile( - `${options.outputDir}/${threadData.id}.json`, - JSON.stringify(threadData.conversation, null, 2) - ); - - // render conversation to markdown and save - const markdown = renderConversation(threadData.conversation); - await fs.writeFile(`${options.outputDir}/${threadData.id}.md`, markdown); - - doneFile.processedUrls.push(conversation.url); - // Save after each conversation in case of interruption - await saveDoneFile(doneFile, options.doneFilePath); - - await sleep(2000); // don't do it too fast + try { + await page.goto(conv.url, { waitUntil: "networkidle2", timeout: 30000 }); + // Wait for prose content to appear + await page.waitForSelector(".prose", { timeout: 15000 }); + // Extra time for full render + await sleep(1500); + + const result = await extractThreadContent(page); + + if (result && result.markdown.length > 0) { + const filename = sanitizeFilename(conv.title || `thread-${i + 1}`) + ".md"; + await fs.writeFile( + `${options.outputDir}/${filename}`, + result.markdown, + "utf-8" + ); + exported++; + console.log(` ✓ Saved: ${filename}`); + } else { + failed++; + console.log(` ✗ No content extracted`); + } + + doneFile.processedUrls.push(conv.url); + await saveDoneFile(doneFile, options.doneFilePath); + } catch (error) { + failed++; + console.error(` ✗ Error: ${error}`); + doneFile.processedUrls.push(conv.url); + await saveDoneFile(doneFile, options.doneFilePath); + } + + await sleep(2000); } - console.log("Done"); + console.log( + `\nDone! Exported: ${exported}/${conversations.length}. Failed: ${failed}` + ); } catch (error) { console.error("An error occurred:", error); } finally { diff --git a/src/listConversations.ts b/src/listConversations.ts index a995d78..8f0da2d 100644 --- a/src/listConversations.ts +++ b/src/listConversations.ts @@ -6,20 +6,48 @@ export async function scrollToBottomOfConversations( page: Page, doneFile: DoneFile ): Promise { - // Scroll to bottom and wait for more items until no new items load + // Find the scrollable container for the library + const containerSelector = await page.evaluate(() => { + // Try multiple possible scrollable containers + const candidates = Array.from(document.querySelectorAll("div")).filter((d) => { + const style = window.getComputedStyle(d); + return ( + (style.overflowY === "auto" || style.overflowY === "scroll") && + d.scrollHeight > d.clientHeight + ); + }); + // Return the one that contains thread links + for (const c of candidates) { + if (c.querySelector('a[href*="/search/"]')) { + // Mark it for later use + c.setAttribute("data-perplexport-scroll", "true"); + return true; + } + } + return false; + }); + + if (!containerSelector) { + console.log("Warning: Could not find scrollable container with threads"); + return; + } + let previousHeight = 0; let currentHeight = await page.evaluate(() => { - const container = document.querySelector("div.scrollable-container"); + const container = document.querySelector('[data-perplexport-scroll="true"]'); return container?.scrollHeight || 0; }); - while (previousHeight !== currentHeight) { + let scrollAttempts = 0; + const maxScrollAttempts = 100; // safety limit + + while (previousHeight !== currentHeight && scrollAttempts < maxScrollAttempts) { + scrollAttempts++; + // Check if we've hit any processed URLs const foundProcessed = await page.evaluate((processedUrls) => { - const items = Array.from( - document.querySelectorAll('div[data-testid="thread-title"]') - ).map((div: Element) => div.closest("a") as HTMLAnchorElement); - return items.some((item) => processedUrls.includes(item.href)); + const items = Array.from(document.querySelectorAll('a[href*="/search/"]')); + return items.some((item) => processedUrls.includes((item as HTMLAnchorElement).href)); }, doneFile.processedUrls); if (foundProcessed) { @@ -29,20 +57,26 @@ export async function scrollToBottomOfConversations( // Scroll to bottom await page.evaluate(() => { - const container = document.querySelector("div.scrollable-container"); + const container = document.querySelector('[data-perplexport-scroll="true"]'); if (container) { container.scrollTo(0, container.scrollHeight); } }); - // Wait a bit for content to load await sleep(2000); previousHeight = currentHeight; currentHeight = await page.evaluate(() => { - const container = document.querySelector("div.scrollable-container"); + const container = document.querySelector('[data-perplexport-scroll="true"]'); return container?.scrollHeight || 0; }); + + if (scrollAttempts % 10 === 0) { + const count = await page.evaluate( + () => document.querySelectorAll('a[href*="/search/"]').length + ); + console.log(`Scrolling... found ${count} threads so far`); + } } } @@ -50,21 +84,29 @@ export async function getConversations( page: Page, doneFile: DoneFile ): Promise { - console.log("Navigating to library..."); - await page.goto("https://www.perplexity.ai/library"); + // We're already on /library from the login step + // Just make sure threads are visible + console.log("Reading library threads..."); - await page.waitForSelector('div[data-testid="thread-title"]'); + await page.waitForSelector('a[href*="/search/"]', { timeout: 30000 }); await scrollToBottomOfConversations(page, doneFile); // Get all conversation links const conversations = await page.evaluate(() => { - const items = Array.from( - document.querySelectorAll('div[data-testid="thread-title"]') - ).map((div: Element) => div.closest("a") as HTMLAnchorElement); - return items.map((item) => ({ - title: item.textContent?.trim() || "Untitled", - url: item.href, - })); + const items = Array.from(document.querySelectorAll('a[href*="/search/"]')); + // Deduplicate by href + const seen = new Set(); + return items + .filter((item) => { + const href = (item as HTMLAnchorElement).href; + if (seen.has(href)) return false; + seen.add(href); + return true; + }) + .map((item) => ({ + title: item.textContent?.trim() || "Untitled", + url: (item as HTMLAnchorElement).href, + })); }); // Filter out already processed conversations and reverse the order diff --git a/src/login.ts b/src/login.ts index 3796950..48c23b6 100644 --- a/src/login.ts +++ b/src/login.ts @@ -1,32 +1,68 @@ import { Page } from "puppeteer"; export async function login(page: Page, email: string): Promise { - console.log("Navigating to Perplexity..."); - await page.goto("https://www.perplexity.ai/"); + console.log("Navigating to Perplexity library..."); + console.log("A login modal will appear. Please log in manually."); + console.log("Waiting up to 3 minutes for login to complete..."); - await page.click("button::-p-text('Accept All Cookies')"); + await page.goto("https://www.perplexity.ai/library"); + await new Promise((r) => setTimeout(r, 3000)); - // Wait for email input and enter credentials - await page.waitForSelector('input[type="email"]'); - await page.type('input[type="email"]', email); + const maxWait = 180000; // 3 minutes + const pollInterval = 2000; + let elapsed = 0; + let loggedIn = false; - // Click the login submit button - await page.click("button::-p-text('Continue with email')"); + while (elapsed < maxWait) { + try { + // Check current URL first — after OAuth redirect we may land back on library + const currentUrl = page.url(); - await page.waitForNavigation(); + // If we're on an OAuth page (Google, Apple), just wait + if ( + currentUrl.includes("accounts.google") || + currentUrl.includes("appleid.apple.com") + ) { + await new Promise((r) => setTimeout(r, pollInterval)); + elapsed += pollInterval; + continue; + } - await page.waitForSelector('input[placeholder="Enter Code"]'); + // If we're back on Perplexity, check for threads + if (currentUrl.includes("perplexity.ai")) { + loggedIn = await page.evaluate(() => { + const loginModal = document.querySelector('[data-testid="login-modal"]'); + if (loginModal) return false; + const allLinks = Array.from(document.querySelectorAll("a")); + return allLinks.some( + (a) => a.href && a.href.includes("perplexity.ai/search/") + ); + }); + } + } catch { + // Navigation destroyed context — this is expected during OAuth redirects + } - console.log( - "Check your email and enter code in the window.\nWaiting for you to enter the email code and login to succeed..." - ); + if (loggedIn) break; - await page.waitForNavigation(); + await new Promise((r) => setTimeout(r, pollInterval)); + elapsed += pollInterval; - // Wait for the main chat input to be ready - await page.waitForSelector("#ask-input", { - timeout: 120000, - }); + if (elapsed % 30000 === 0) { + console.log(`Still waiting for login... (${elapsed / 1000}s elapsed)`); + } + } - console.log("Successfully logged in"); + if (!loggedIn) { + throw new Error("Login timed out after 3 minutes. Please try again."); + } + + // Make sure we're on the library page after login + const currentUrl = page.url(); + if (!currentUrl.includes("/library")) { + await page.goto("https://www.perplexity.ai/library"); + await new Promise((r) => setTimeout(r, 3000)); + } + + console.log("Successfully logged in — threads detected in library"); }