11"""
22Search service.
3- Integrates Google Custom Search (CSE) and Bing Search API.
3+ Integrates Google Custom Search (CSE), Bing Search API, and DuckDuckGo (free fallback) .
44Configuration is provided via environment variables (see env.example) through config_loader.
55"""
66import requests
77import json
8+ import time
89from typing import List , Dict , Any , Optional
910from app .utils .logger import get_logger
1011from app .utils .config_loader import load_addon_config
1112
1213logger = get_logger (__name__ )
1314
15+ # Track Google API quota status
16+ _google_quota_exhausted = False
17+ _google_quota_reset_time = 0
18+
1419
1520class SearchService :
16- """Search service."""
21+ """Search service with automatic fallback ."""
1722
1823 def __init__ (self ):
1924 self ._config = {}
@@ -28,7 +33,7 @@ def _load_config(self):
2833
2934 def search (self , query : str , num_results : int = None , date_restrict : str = None ) -> List [Dict [str , Any ]]:
3035 """
31- Execute a web search.
36+ Execute a web search with automatic fallback .
3237
3338 Args:
3439 query: Search query
@@ -38,18 +43,45 @@ def search(self, query: str, num_results: int = None, date_restrict: str = None)
3843 Returns:
3944 List of search results
4045 """
46+ global _google_quota_exhausted , _google_quota_reset_time
47+
4148 # 重新加载配置以支持热更新
4249 self ._load_config ()
4350
4451 limit = num_results if num_results else self .max_results
4552
53+ # Check if Google quota has reset (after midnight UTC typically)
54+ if _google_quota_exhausted and time .time () > _google_quota_reset_time :
55+ _google_quota_exhausted = False
56+ logger .info ("Google API quota reset, re-enabling Google search" )
57+
58+ results = []
59+
4660 if self .provider == 'bing' :
47- return self ._search_bing (query , limit )
61+ results = self ._search_bing (query , limit )
62+ elif self .provider == 'duckduckgo' :
63+ results = self ._search_duckduckgo (query , limit )
4864 else :
49- return self ._search_google (query , limit , date_restrict )
65+ # Google with fallback
66+ if not _google_quota_exhausted :
67+ results = self ._search_google (query , limit , date_restrict )
68+
69+ # If Google failed or returned empty, try fallbacks
70+ if not results :
71+ logger .info ("Google search failed or empty, trying fallback search engines..." )
72+ # Try Bing first if configured
73+ results = self ._search_bing (query , limit )
74+
75+ # If Bing also failed, try DuckDuckGo (free, no API key needed)
76+ if not results :
77+ results = self ._search_duckduckgo (query , limit )
78+
79+ return results
5080
5181 def _search_google (self , query : str , num_results : int , date_restrict : str = None ) -> List [Dict [str , Any ]]:
5282 """Google Custom Search (CSE)."""
83+ global _google_quota_exhausted , _google_quota_reset_time
84+
5385 api_key = self ._config .get ('google' , {}).get ('api_key' )
5486 cx = self ._config .get ('google' , {}).get ('cx' )
5587
@@ -71,17 +103,23 @@ def _search_google(self, query: str, num_results: int, date_restrict: str = None
71103 params ['dateRestrict' ] = date_restrict
72104
73105 try :
74- # logger.info(f"正在调用 Google Search API: q={query}, params={params}")
75106 response = requests .get (url , params = params , timeout = 10 )
107+
108+ # Check for quota exceeded (429)
109+ if response .status_code == 429 :
110+ logger .warning ("Google Search API quota exceeded (429). Switching to fallback search engines." )
111+ _google_quota_exhausted = True
112+ # Set reset time to next day midnight UTC
113+ import datetime
114+ tomorrow = datetime .datetime .utcnow ().replace (hour = 0 , minute = 0 , second = 0 , microsecond = 0 ) + datetime .timedelta (days = 1 )
115+ _google_quota_reset_time = tomorrow .timestamp ()
116+ return []
117+
76118 response .raise_for_status ()
77119 data = response .json ()
78120
79- # logger.info(f"Google Search 原始响应: {json.dumps(data, ensure_ascii=False)}") # 打印全部字符
80- # logger.info(f"Google Search 原始响应: {json.dumps(data, ensure_ascii=False)[:500]}...") # 打印前500字符避免日志过大
81-
82121 results = []
83122 if 'items' in data :
84- # logger.info(f"Google Search 返回了 {len(data['items'])} 条结果")
85123 for item in data ['items' ]:
86124 logger .debug (f"Search Item: { item .get ('title' )} - { item .get ('link' )} " )
87125 results .append ({
@@ -96,10 +134,20 @@ def _search_google(self, query: str, num_results: int, date_restrict: str = None
96134
97135 return results
98136
137+ except requests .exceptions .HTTPError as e :
138+ if hasattr (e , 'response' ) and e .response is not None and e .response .status_code == 429 :
139+ logger .warning ("Google Search API quota exceeded. Switching to fallback." )
140+ _google_quota_exhausted = True
141+ import datetime
142+ tomorrow = datetime .datetime .utcnow ().replace (hour = 0 , minute = 0 , second = 0 , microsecond = 0 ) + datetime .timedelta (days = 1 )
143+ _google_quota_reset_time = tomorrow .timestamp ()
144+ else :
145+ logger .error (f"Google search failed: { e } " )
146+ if hasattr (e , 'response' ) and e .response is not None :
147+ logger .error (f"Response: { e .response .text } " )
148+ return []
99149 except Exception as e :
100150 logger .error (f"Google search failed: { e } " )
101- if 'response' in locals ():
102- logger .error (f"Response: { response .text } " )
103151 return []
104152
105153 def _search_bing (self , query : str , num_results : int ) -> List [Dict [str , Any ]]:
@@ -140,3 +188,120 @@ def _search_bing(self, query: str, num_results: int) -> List[Dict[str, Any]]:
140188 logger .error (f"Bing search failed: { e } " )
141189 return []
142190
191+ def _search_duckduckgo (self , query : str , num_results : int ) -> List [Dict [str , Any ]]:
192+ """
193+ DuckDuckGo search (free, no API key required).
194+ Uses the DuckDuckGo HTML search endpoint.
195+ """
196+ try :
197+ # Use DuckDuckGo Instant Answer API
198+ url = "https://api.duckduckgo.com/"
199+ params = {
200+ 'q' : query ,
201+ 'format' : 'json' ,
202+ 'no_html' : 1 ,
203+ 'skip_disambig' : 1
204+ }
205+
206+ response = requests .get (url , params = params , timeout = 10 )
207+ response .raise_for_status ()
208+ data = response .json ()
209+
210+ results = []
211+
212+ # Get results from RelatedTopics
213+ related_topics = data .get ('RelatedTopics' , [])
214+ for topic in related_topics [:num_results ]:
215+ if isinstance (topic , dict ):
216+ if 'FirstURL' in topic :
217+ results .append ({
218+ 'title' : topic .get ('Text' , '' )[:100 ],
219+ 'link' : topic .get ('FirstURL' , '' ),
220+ 'snippet' : topic .get ('Text' , '' ),
221+ 'source' : 'DuckDuckGo' ,
222+ 'published' : ''
223+ })
224+ # Handle nested topics
225+ elif 'Topics' in topic :
226+ for sub_topic in topic ['Topics' ]:
227+ if len (results ) >= num_results :
228+ break
229+ if 'FirstURL' in sub_topic :
230+ results .append ({
231+ 'title' : sub_topic .get ('Text' , '' )[:100 ],
232+ 'link' : sub_topic .get ('FirstURL' , '' ),
233+ 'snippet' : sub_topic .get ('Text' , '' ),
234+ 'source' : 'DuckDuckGo' ,
235+ 'published' : ''
236+ })
237+
238+ # Also check AbstractURL and AbstractText
239+ if data .get ('AbstractURL' ) and len (results ) < num_results :
240+ results .insert (0 , {
241+ 'title' : data .get ('Heading' , query ),
242+ 'link' : data .get ('AbstractURL' , '' ),
243+ 'snippet' : data .get ('AbstractText' , '' ),
244+ 'source' : 'DuckDuckGo' ,
245+ 'published' : ''
246+ })
247+
248+ # If no results from Instant Answer, try HTML scraping as fallback
249+ if not results :
250+ results = self ._search_duckduckgo_html (query , num_results )
251+
252+ if results :
253+ logger .info (f"DuckDuckGo search returned { len (results )} results" )
254+
255+ return results [:num_results ]
256+
257+ except Exception as e :
258+ logger .error (f"DuckDuckGo search failed: { e } " )
259+ # Try HTML fallback
260+ return self ._search_duckduckgo_html (query , num_results )
261+
262+ def _search_duckduckgo_html (self , query : str , num_results : int ) -> List [Dict [str , Any ]]:
263+ """
264+ DuckDuckGo HTML search fallback.
265+ Scrapes the lite HTML version for better results.
266+ """
267+ try :
268+ url = "https://lite.duckduckgo.com/lite/"
269+ headers = {
270+ 'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
271+ }
272+ data = {'q' : query }
273+
274+ response = requests .post (url , headers = headers , data = data , timeout = 10 )
275+ response .raise_for_status ()
276+
277+ results = []
278+
279+ # Simple HTML parsing without BeautifulSoup
280+ html = response .text
281+
282+ # Find all result links (they have class="result-link")
283+ import re
284+
285+ # Pattern to find result entries
286+ link_pattern = r'<a[^>]*class="result-link"[^>]*href="([^"]*)"[^>]*>([^<]*)</a>'
287+ snippet_pattern = r'<td[^>]*class="result-snippet"[^>]*>([^<]*)</td>'
288+
289+ links = re .findall (link_pattern , html )
290+ snippets = re .findall (snippet_pattern , html )
291+
292+ for i , (link , title ) in enumerate (links [:num_results ]):
293+ snippet = snippets [i ] if i < len (snippets ) else ''
294+ if link and title :
295+ results .append ({
296+ 'title' : title .strip (),
297+ 'link' : link ,
298+ 'snippet' : snippet .strip (),
299+ 'source' : 'DuckDuckGo' ,
300+ 'published' : ''
301+ })
302+
303+ return results
304+
305+ except Exception as e :
306+ logger .error (f"DuckDuckGo HTML search failed: { e } " )
307+ return []
0 commit comments