[Bug]: JSON serialization error #1343
Description
crawl4ai version
0.9.7
Expected Behavior
I expect the crawler to informe me that the url doesn't have a valid web page.
Current Behavior
Crawl4ai crash
Is this reproducible?
Yes
Inputs Causing the Bug
{
"urls": ["https://avendre.yaknema.com/articles/1281/"],
"crawler_params": {
# Browser Configuration
"headless": True, # Run in headless mode
"browser_type": "chromium", # chromium/firefox/webkit
# Performance & Behavior
"page_timeout": 30000, # Page load timeout (ms)
"verbose": True, # Enable detailed logging
"semaphore_count": 5, # Concurrent request limit
# Anti-Detection Features
"simulate_user": True, # Simulate human behavior
"magic": True, # Advanced anti-detection
"override_navigator": True, # Override navigator properties
}
}Steps to Reproduce
Use python requests like this
requests
requests.post("http://localhost:11235/crawl", json=[previous_input_data]])Code snippets
OS
Linux
Python version
3.10.12
Browser
No response
Browser version
No response
Error logs & Screenshots (if applicable)
ode context:
758 tag="GOTO",
759 params={"url": url},
760 )
761 response = None
762 else:
763 → raise RuntimeError(f"Failed on navigating
ACS-GOTO:\n{str(e)}")
764
765 await self.execute_hook(
766 "after_goto", page, context=context, url=url,
response=response, config=config
767 )
768
2025-07-25 15:37:04,428 - api - INFO - Memory usage: Start: 124.671875 MB, End: 122.65234375 MB, Delta: -2.01953125 MB, Peak: 124.671875 MB
[2025-07-25 15:37:04 +0000] [13] [ERROR] Exception in ASGI application
+ Exception Group Traceback (most recent call last):
| File "/usr/local/lib/python3.12/site-packages/starlette/_utils.py", line 77, in collapse_excgroups
| yield
| File "/usr/local/lib/python3.12/site-packages/starlette/middleware/base.py", line 183, in __call__
| async with anyio.create_task_group() as task_group:
| ^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/usr/local/lib/python3.12/site-packages/anyio/_backends/_asyncio.py", line 772, in __aexit__
| raise BaseExceptionGroup(
| ExceptionGroup: unhandled errors in a TaskGroup (1 sub-exception)
+-+---------------- 1 ----------------
| Traceback (most recent call last):
| File "/usr/local/lib/python3.12/site-packages/uvicorn/protocols/http/h11_impl.py", line 403, in run_asgi
| result = await app( # type: ignore[func-returns-value]
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/usr/local/lib/python3.12/site-packages/uvicorn/middleware/proxy_headers.py", line 60, in __call__
| return await self.app(scope, receive, send)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/usr/local/lib/python3.12/site-packages/fastapi/applications.py", line 1054, in __call__
| await super().__call__(scope, receive, send)
| File "/usr/local/lib/python3.12/site-packages/starlette/applications.py", line 113, in __call__
| await self.middleware_stack(scope, receive, send)
| File "/usr/local/lib/python3.12/site-packages/starlette/middleware/errors.py", line 186, in __call__
| raise exc
| File "/usr/local/lib/python3.12/site-packages/starlette/middleware/errors.py", line 164, in __call__
| await self.app(scope, receive, _send)
| File "/usr/local/lib/python3.12/site-packages/starlette/middleware/base.py", line 182, in __call__
| with recv_stream, send_stream, collapse_excgroups():
| ^^^^^^^^^^^^^^^^^^^^
| File "/usr/local/lib/python3.12/contextlib.py", line 158, in __exit__
| self.gen.throw(value)
| File "/usr/local/lib/python3.12/site-packages/starlette/_utils.py", line 83, in collapse_excgroups
| raise exc
| File "/usr/local/lib/python3.12/site-packages/starlette/middleware/base.py", line 184, in __call__
| response = await self.dispatch_func(request, call_next)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/app/server.py", line 169, in add_security_headers
| resp = await call_next(request)
| ^^^^^^^^^^^^^^^^^^^^^^^^
| File "/usr/local/lib/python3.12/site-packages/starlette/middleware/base.py", line 159, in call_next
| raise app_exc
| File "/usr/local/lib/python3.12/site-packages/starlette/middleware/base.py", line 144, in coro
| await self.app(scope, receive_or_disconnect, send_no_error)
| File "/usr/local/lib/python3.12/site-packages/prometheus_fastapi_instrumentator/middleware.py", line 177, in __call__
| raise exc
| File "/usr/local/lib/python3.12/site-packages/prometheus_fastapi_instrumentator/middleware.py", line 175, in __call__
| await self.app(scope, receive, send_wrapper)
| File "/usr/local/lib/python3.12/site-packages/starlette/middleware/exceptions.py", line 63, in __call__
| await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
| File "/usr/local/lib/python3.12/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
| raise exc
| File "/usr/local/lib/python3.12/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
| await app(scope, receive, sender)
| File "/usr/local/lib/python3.12/site-packages/starlette/routing.py", line 716, in __call__
| await self.middleware_stack(scope, receive, send)
| File "/usr/local/lib/python3.12/site-packages/starlette/routing.py", line 736, in app
| await route.handle(scope, receive, send)
| File "/usr/local/lib/python3.12/site-packages/starlette/routing.py", line 290, in handle
| await self.app(scope, receive, send)
| File "/usr/local/lib/python3.12/site-packages/starlette/routing.py", line 78, in app
| await wrap_app_handling_exceptions(app, request)(scope, receive, send)
| File "/usr/local/lib/python3.12/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
| raise exc
| File "/usr/local/lib/python3.12/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
| await app(scope, receive, sender)
| File "/usr/local/lib/python3.12/site-packages/starlette/routing.py", line 75, in app
| response = await f(request)
| ^^^^^^^^^^^^^^^^
| File "/usr/local/lib/python3.12/site-packages/fastapi/routing.py", line 302, in app
| raw_response = await run_endpoint_function(
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/usr/local/lib/python3.12/site-packages/fastapi/routing.py", line 213, in run_endpoint_function
| return await dependant.call(**values)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/usr/local/lib/python3.12/site-packages/slowapi/extension.py", line 734, in async_wrapper
| response = await func(*args, **kwargs) # type: ignore
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/app/server.py", line 446, in crawl
| return JSONResponse(res)
| ^^^^^^^^^^^^^^^^^
| File "/usr/local/lib/python3.12/site-packages/starlette/responses.py", line 190, in __init__
| super().__init__(content, status_code, headers, media_type, background)
| File "/usr/local/lib/python3.12/site-packages/starlette/responses.py", line 47, in __init__
| self.body = self.render(content)
| ^^^^^^^^^^^^^^^^^^^^
| File "/usr/local/lib/python3.12/site-packages/starlette/responses.py", line 193, in render
| return json.dumps(
| ^^^^^^^^^^^
| File "/usr/local/lib/python3.12/json/__init__.py", line 238, in dumps
| **kw).encode(obj)
| ^^^^^^^^^^^
| File "/usr/local/lib/python3.12/json/encoder.py", line 200, in encode
| chunks = self.iterencode(o, _one_shot=True)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/usr/local/lib/python3.12/json/encoder.py", line 258, in iterencode
| return _iterencode(o, 0)
| ^^^^^^^^^^^^^^^^^
| File "/usr/local/lib/python3.12/json/encoder.py", line 180, in default
| raise TypeError(f'Object of type {o.__class__.__name__} '
| TypeError: Object of type property is not JSON serializable
+------------------------------------
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.12/site-packages/uvicorn/protocols/http/h11_impl.py", line 403, in run_asgi
result = await app( # type: ignore[func-returns-value]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/uvicorn/middleware/proxy_headers.py", line 60, in __call__
return await self.app(scope, receive, send)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/fastapi/applications.py", line 1054, in __call__
await super().__call__(scope, receive, send)
File "/usr/local/lib/python3.12/site-packages/starlette/applications.py", line 113, in __call__
await self.middleware_stack(scope, receive, send)
File "/usr/local/lib/python3.12/site-packages/starlette/middleware/errors.py", line 186, in __call__
raise exc
File "/usr/local/lib/python3.12/site-packages/starlette/middleware/errors.py", line 164, in __call__
await self.app(scope, receive, _send)
File "/usr/local/lib/python3.12/site-packages/starlette/middleware/base.py", line 182, in __call__
with recv_stream, send_stream, collapse_excgroups():
^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/contextlib.py", line 158, in __exit__
self.gen.throw(value)
File "/usr/local/lib/python3.12/site-packages/starlette/_utils.py", line 83, in collapse_excgroups
raise exc
File "/usr/local/lib/python3.12/site-packages/starlette/middleware/base.py", line 184, in __call__
response = await self.dispatch_func(request, call_next)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/server.py", line 169, in add_security_headers
resp = await call_next(request)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/starlette/middleware/base.py", line 159, in call_next
raise app_exc
File "/usr/local/lib/python3.12/site-packages/starlette/middleware/base.py", line 144, in coro
await self.app(scope, receive_or_disconnect, send_no_error)
File "/usr/local/lib/python3.12/site-packages/prometheus_fastapi_instrumentator/middleware.py", line 177, in __call__
raise exc
File "/usr/local/lib/python3.12/site-packages/prometheus_fastapi_instrumentator/middleware.py", line 175, in __call__
await self.app(scope, receive, send_wrapper)
File "/usr/local/lib/python3.12/site-packages/starlette/middleware/exceptions.py", line 63, in __call__
await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
File "/usr/local/lib/python3.12/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
raise exc
File "/usr/local/lib/python3.12/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/usr/local/lib/python3.12/site-packages/starlette/routing.py", line 716, in __call__
await self.middleware_stack(scope, receive, send)
File "/usr/local/lib/python3.12/site-packages/starlette/routing.py", line 736, in app
await route.handle(scope, receive, send)
File "/usr/local/lib/python3.12/site-packages/starlette/routing.py", line 290, in handle
await self.app(scope, receive, send)
File "/usr/local/lib/python3.12/site-packages/starlette/routing.py", line 78, in app
await wrap_app_handling_exceptions(app, request)(scope, receive, send)
File "/usr/local/lib/python3.12/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
raise exc
File "/usr/local/lib/python3.12/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/usr/local/lib/python3.12/site-packages/starlette/routing.py", line 75, in app
response = await f(request)
^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/fastapi/routing.py", line 302, in app
raw_response = await run_endpoint_function(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/fastapi/routing.py", line 213, in run_endpoint_function
return await dependant.call(**values)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/slowapi/extension.py", line 734, in async_wrapper
response = await func(*args, **kwargs) # type: ignore
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/app/server.py", line 446, in crawl
return JSONResponse(res)
^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/starlette/responses.py", line 190, in __init__
super().__init__(content, status_code, headers, media_type, background)
File "/usr/local/lib/python3.12/site-packages/starlette/responses.py", line 47, in __init__
self.body = self.render(content)
^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/starlette/responses.py", line 193, in render
return json.dumps(
^^^^^^^^^^^
File "/usr/local/lib/python3.12/json/__init__.py", line 238, in dumps
**kw).encode(obj)
^^^^^^^^^^^
File "/usr/local/lib/python3.12/json/encoder.py", line 200, in encode
chunks = self.iterencode(o, _one_shot=True)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/json/encoder.py", line 258, in iterencode
return _iterencode(o, 0)
^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/json/encoder.py", line 180, in default
raise TypeError(f'Object of type {o.__class__.__name__} '
TypeError: Object of type property is not JSON serializable