Skip to content

Commit 2faeb47

Browse files
committed
fix: format
1 parent 125194b commit 2faeb47

File tree

1 file changed

+20
-14
lines changed

1 file changed

+20
-14
lines changed

src/util/geminiUrlScraper.js

Lines changed: 20 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ async function getGenAIClient() {
1919

2020
/**
2121
* Scrapes URL content using Gemini's urlContext tool
22-
*
22+
*
2323
* @param {string[]} urls - Array of URLs to scrape
2424
* @returns {Promise<Array<{url: string, canonical?: string, title?: string, summary?: string, topImageUrl?: string, html?: string, status: string, error?: string}>>}
2525
*/
@@ -33,7 +33,7 @@ export default async function scrapeUrlsWithGemini(urls) {
3333
try {
3434
// Process all URLs in a single LLM call for efficiency
3535
const urlList = urls.map((url, index) => `${index + 1}. ${url}`).join('\n');
36-
36+
3737
const generateContentArgs = {
3838
model: 'gemini-2.5-flash',
3939
contents: [
@@ -70,21 +70,24 @@ Requirements:
7070
],
7171
config: {
7272
tools: [{ urlContext: {} }],
73-
systemInstruction: 'You are a web content analyzer that extracts structured information from web pages for fact-checking purposes.',
73+
systemInstruction:
74+
'You are a web content analyzer that extracts structured information from web pages for fact-checking purposes.',
7475
responseModalities: ['TEXT'],
7576
temperature: 0.1, // Low temperature for consistent extraction
7677
maxOutputTokens: 4096,
7778
},
7879
};
7980

80-
const response = await genAIClient.models.generateContent(generateContentArgs);
81-
81+
const response = await genAIClient.models.generateContent(
82+
generateContentArgs
83+
);
84+
8285
if (!response.candidates || !response.candidates[0]) {
8386
throw new Error('No response candidates received');
8487
}
8588

8689
const responseText = response.candidates[0].content.parts[0].text;
87-
90+
8891
// Parse the JSON response
8992
let extractedDataArray;
9093
try {
@@ -96,9 +99,12 @@ Requirements:
9699
extractedDataArray = JSON.parse(responseText);
97100
}
98101
} catch (parseError) {
99-
console.warn('[geminiUrlScraper] Failed to parse JSON response:', responseText);
102+
console.warn(
103+
'[geminiUrlScraper] Failed to parse JSON response:',
104+
responseText
105+
);
100106
// Fallback: create error results for all URLs
101-
return urls.map(url => ({
107+
return urls.map((url) => ({
102108
url,
103109
canonical: url,
104110
title: null,
@@ -111,8 +117,9 @@ Requirements:
111117
}
112118

113119
// Ensure we have results for all input URLs
114-
const results = urls.map(url => {
115-
const extracted = extractedDataArray.find(item => item.url === url) || {};
120+
const results = urls.map((url) => {
121+
const extracted =
122+
extractedDataArray.find((item) => item.url === url) || {};
116123
return {
117124
url,
118125
canonical: extracted.canonical || url,
@@ -125,17 +132,16 @@ Requirements:
125132
});
126133

127134
return results;
128-
129135
} catch (error) {
130136
console.error('[geminiUrlScraper] Error processing URLs:', error);
131-
137+
132138
rollbar.error('Gemini URL scraping error', {
133139
urls,
134140
error: error.message,
135141
});
136142

137143
// Return error results for all URLs
138-
return urls.map(url => ({
144+
return urls.map((url) => ({
139145
url,
140146
canonical: url,
141147
title: null,
@@ -146,4 +152,4 @@ Requirements:
146152
error: error.message,
147153
}));
148154
}
149-
}
155+
}

0 commit comments

Comments
 (0)