Skip to content

fix: update broken selectors and switch to DOM-based export #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
matteofrassi wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: update broken selectors and switch to DOM-based export #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
204 changes: 175 additions & 29 deletions src/exportLibrary.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
import { promises as fs } from "fs";
import puppeteer from "puppeteer-extra";
import { Browser } from "puppeteer";
import { Browser, Page } from "puppeteer";
import StealthPlugin from "puppeteer-extra-plugin-stealth";
import { ConversationSaver } from "./ConversationSaver";
import { getConversations } from "./listConversations";
import { login } from "./login";
import renderConversation from "./renderConversation";
import { loadDoneFile, saveDoneFile, sleep } from "./utils";

export interface ExportLibraryOptions {
Expand All @@ -14,20 +12,149 @@ export interface ExportLibraryOptions {
email: string;
}

function sanitizeFilename(name: string): string {
return name
.replace(/[/\\?%*:|"<>]/g, "-")
.replace(/\s+/g, " ")
.trim()
.substring(0, 100);
}

async function extractThreadContent(page: Page): Promise<{ title: string; markdown: string } | null> {
return page.evaluate(() => {
// Helper: convert HTML element to Markdown
function htmlToMd(el: Element): string {
let md = "";
for (const node of Array.from(el.childNodes)) {
if (node.nodeType === Node.TEXT_NODE) {
md += node.textContent || "";
} else if (node.nodeType === Node.ELEMENT_NODE) {
const elem = node as Element;
const tag = elem.tagName;

// Skip citation number spans
if (elem.classList.contains("citation") || elem.classList.contains("citation-nbsp")) {
continue;
}

if (tag === "P") {
md += "\n\n" + htmlToMd(elem);
} else if (tag === "STRONG" || tag === "B") {
md += "**" + htmlToMd(elem) + "**";
} else if (tag === "EM" || tag === "I") {
md += "*" + htmlToMd(elem) + "*";
} else if (tag === "CODE") {
const text = elem.textContent || "";
if (elem.parentElement?.tagName === "PRE") {
md += text;
} else {
md += "`" + text + "`";
}
} else if (tag === "PRE") {
const code = elem.querySelector("code");
const lang = code?.className?.match(/language-(\w+)/)?.[1] || "";
md += "\n\n```" + lang + "\n" + (code?.textContent || elem.textContent || "") + "\n```\n";
} else if (tag === "H1") {
md += "\n\n# " + htmlToMd(elem);
} else if (tag === "H2") {
md += "\n\n## " + htmlToMd(elem);
} else if (tag === "H3") {
md += "\n\n### " + htmlToMd(elem);
} else if (tag === "H4") {
md += "\n\n#### " + htmlToMd(elem);
} else if (tag === "UL") {
md += "\n";
for (const li of Array.from(elem.children)) {
if (li.tagName === "LI") {
md += "\n- " + htmlToMd(li).trim();
}
}
md += "\n";
} else if (tag === "OL") {
md += "\n";
let i = 1;
for (const li of Array.from(elem.children)) {
if (li.tagName === "LI") {
md += "\n" + i + ". " + htmlToMd(li).trim();
i++;
}
}
md += "\n";
} else if (tag === "A") {
const href = (elem as HTMLAnchorElement).href;
const text = htmlToMd(elem);
md += "[" + text + "](" + href + ")";
} else if (tag === "BR") {
md += "\n";
} else if (tag === "BLOCKQUOTE") {
const inner = htmlToMd(elem).trim().split("\n").map((l: string) => "> " + l).join("\n");
md += "\n\n" + inner + "\n";
} else if (tag === "TABLE") {
// Simple table extraction
const rows = Array.from(elem.querySelectorAll("tr"));
if (rows.length > 0) {
md += "\n\n";
rows.forEach((row, idx) => {
const cells = Array.from(row.querySelectorAll("th, td"));
md += "| " + cells.map((c) => (c.textContent || "").trim()).join(" | ") + " |\n";
if (idx === 0) {
md += "| " + cells.map(() => "---").join(" | ") + " |\n";
}
});
}
} else {
// Default: recurse
md += htmlToMd(elem);
}
}
}
return md;
}

// Find all Q&A pairs on the page
// The page structure has the query text and then prose blocks for answers
const proseBlocks = Array.from(
document.querySelectorAll(".prose")
).filter((el) => el.textContent && el.textContent.trim().length > 50);

if (proseBlocks.length === 0) return null;

// Get the page title / query from the page
// Usually visible as a heading or the first prominent text
const titleEl = document.querySelector("h1") ||
document.querySelector('[class*="query"]') ||
document.querySelector("title");
const title = titleEl?.textContent?.trim() || document.title || "Untitled";

// Build markdown from all prose blocks
let fullMd = "# " + title + "\n\n";
fullMd += "Source: " + window.location.href + "\n\n---\n";

for (const block of proseBlocks) {
const content = htmlToMd(block).trim();
if (content) {
fullMd += "\n\n" + content;
}
}

// Clean up multiple newlines
fullMd = fullMd.replace(/\n{4,}/g, "\n\n\n");

return { title, markdown: fullMd };
});
}

export default async function exportLibrary(options: ExportLibraryOptions) {
puppeteer.use(StealthPlugin());

// Create output directory if it doesn't exist
await fs.mkdir(options.outputDir, { recursive: true });

// Load done file
const doneFile = await loadDoneFile(options.doneFilePath);
console.log(
`Loaded ${doneFile.processedUrls.length} processed URLs from done file`
);

const browser: Browser = await puppeteer.launch({
// Authentication is interactive.
headless: false,
});

Expand All @@ -39,34 +166,53 @@ export default async function exportLibrary(options: ExportLibraryOptions) {

console.log(`Found ${conversations.length} new conversations to process`);

const conversationSaver = new ConversationSaver(page);
await conversationSaver.initialize();
let exported = 0;
let failed = 0;

for (const conversation of conversations) {
console.log(`Processing conversation ${conversation.url}`);

const threadData = await conversationSaver.loadThreadFromURL(
conversation.url
for (let i = 0; i < conversations.length; i++) {
const conv = conversations[i];
console.log(
`[${i + 1}/${conversations.length}] ${conv.title.substring(0, 60)}`
);

// place the thread data in the output directory
await fs.writeFile(
`${options.outputDir}/${threadData.id}.json`,
JSON.stringify(threadData.conversation, null, 2)
);

// render conversation to markdown and save
const markdown = renderConversation(threadData.conversation);
await fs.writeFile(`${options.outputDir}/${threadData.id}.md`, markdown);

doneFile.processedUrls.push(conversation.url);
// Save after each conversation in case of interruption
await saveDoneFile(doneFile, options.doneFilePath);

await sleep(2000); // don't do it too fast
try {
await page.goto(conv.url, { waitUntil: "networkidle2", timeout: 30000 });
// Wait for prose content to appear
await page.waitForSelector(".prose", { timeout: 15000 });
// Extra time for full render
await sleep(1500);

const result = await extractThreadContent(page);

if (result && result.markdown.length > 0) {
const filename = sanitizeFilename(conv.title || `thread-${i + 1}`) + ".md";
await fs.writeFile(
`${options.outputDir}/${filename}`,
result.markdown,
"utf-8"
);
exported++;
console.log(` ✓ Saved: ${filename}`);
} else {
failed++;
console.log(` ✗ No content extracted`);
}

doneFile.processedUrls.push(conv.url);
await saveDoneFile(doneFile, options.doneFilePath);
} catch (error) {
failed++;
console.error(` ✗ Error: ${error}`);
doneFile.processedUrls.push(conv.url);
await saveDoneFile(doneFile, options.doneFilePath);
}

await sleep(2000);
}

console.log("Done");
console.log(
`\nDone! Exported: ${exported}/${conversations.length}. Failed: ${failed}`
);
} catch (error) {
console.error("An error occurred:", error);
} finally {
Expand Down
82 changes: 62 additions & 20 deletions src/listConversations.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,48 @@ export async function scrollToBottomOfConversations(
page: Page,
doneFile: DoneFile
): Promise<void> {
// Scroll to bottom and wait for more items until no new items load
// Find the scrollable container for the library
const containerSelector = await page.evaluate(() => {
// Try multiple possible scrollable containers
const candidates = Array.from(document.querySelectorAll("div")).filter((d) => {
const style = window.getComputedStyle(d);
return (
(style.overflowY === "auto" || style.overflowY === "scroll") &&
d.scrollHeight > d.clientHeight
);
});
// Return the one that contains thread links
for (const c of candidates) {
if (c.querySelector('a[href*="/search/"]')) {
// Mark it for later use
c.setAttribute("data-perplexport-scroll", "true");
return true;
}
}
return false;
});

if (!containerSelector) {
console.log("Warning: Could not find scrollable container with threads");
return;
}

let previousHeight = 0;
let currentHeight = await page.evaluate(() => {
const container = document.querySelector("div.scrollable-container");
const container = document.querySelector('[data-perplexport-scroll="true"]');
return container?.scrollHeight || 0;
});

while (previousHeight !== currentHeight) {
let scrollAttempts = 0;
const maxScrollAttempts = 100; // safety limit

while (previousHeight !== currentHeight && scrollAttempts < maxScrollAttempts) {
scrollAttempts++;

// Check if we've hit any processed URLs
const foundProcessed = await page.evaluate((processedUrls) => {
const items = Array.from(
document.querySelectorAll('div[data-testid="thread-title"]')
).map((div: Element) => div.closest("a") as HTMLAnchorElement);
return items.some((item) => processedUrls.includes(item.href));
const items = Array.from(document.querySelectorAll('a[href*="/search/"]'));
return items.some((item) => processedUrls.includes((item as HTMLAnchorElement).href));
}, doneFile.processedUrls);

if (foundProcessed) {
Expand All @@ -29,42 +57,56 @@ export async function scrollToBottomOfConversations(

// Scroll to bottom
await page.evaluate(() => {
const container = document.querySelector("div.scrollable-container");
const container = document.querySelector('[data-perplexport-scroll="true"]');
if (container) {
container.scrollTo(0, container.scrollHeight);
}
});

// Wait a bit for content to load
await sleep(2000);

previousHeight = currentHeight;
currentHeight = await page.evaluate(() => {
const container = document.querySelector("div.scrollable-container");
const container = document.querySelector('[data-perplexport-scroll="true"]');
return container?.scrollHeight || 0;
});

if (scrollAttempts % 10 === 0) {
const count = await page.evaluate(
() => document.querySelectorAll('a[href*="/search/"]').length
);
console.log(`Scrolling... found ${count} threads so far`);
}
}
}

export async function getConversations(
page: Page,
doneFile: DoneFile
): Promise<Conversation[]> {
console.log("Navigating to library...");
await page.goto("https://www.perplexity.ai/library");
// We're already on /library from the login step
// Just make sure threads are visible
console.log("Reading library threads...");

await page.waitForSelector('div[data-testid="thread-title"]');
await page.waitForSelector('a[href*="/search/"]', { timeout: 30000 });
await scrollToBottomOfConversations(page, doneFile);

// Get all conversation links
const conversations = await page.evaluate(() => {
const items = Array.from(
document.querySelectorAll('div[data-testid="thread-title"]')
).map((div: Element) => div.closest("a") as HTMLAnchorElement);
return items.map((item) => ({
title: item.textContent?.trim() || "Untitled",
url: item.href,
}));
const items = Array.from(document.querySelectorAll('a[href*="/search/"]'));
// Deduplicate by href
const seen = new Set<string>();
return items
.filter((item) => {
const href = (item as HTMLAnchorElement).href;
if (seen.has(href)) return false;
seen.add(href);
return true;
})
.map((item) => ({
title: item.textContent?.trim() || "Untitled",
url: (item as HTMLAnchorElement).href,
}));
});

// Filter out already processed conversations and reverse the order
Expand Down
Loading