Faster message serialization #1064
Conversation
…l.parser.isoparse
…ialize_objects for datetime validation
|
The failing tests on Windows appear to be unrelated to this PR: I've run both the main branch and this PR on Windows locally and observe similar errors related to "unclosed sockets". If the test is run individually it passes but when running in groups the error can move around, though generally the reported error is the same. c:\code\jupyter_client\tests\test_multikernelmanager.py::TestKernelManager::test_start_parallel_thread_kernels failed: cls = <class '_pytest.runner.CallInfo'>
func = <function call_and_report.<locals>.<lambda> at 0x0000013B0182FB00>
when = 'call'
reraise = (<class '_pytest.outcomes.Exit'>, <class 'KeyboardInterrupt'>)
@classmethod
def from_call(
cls,
func: Callable[[], TResult],
when: Literal["collect", "setup", "call", "teardown"],
reraise: Optional[
Union[Type[BaseException], Tuple[Type[BaseException], ...]]
] = None,
) -> "CallInfo[TResult]":
"""Call func, wrapping the result in a CallInfo.
:param func:
The function to call. Called without arguments.
:param when:
The phase in which the function is called.
:param reraise:
Exception or exceptions that shall propagate if raised by the
function, instead of being wrapped in the CallInfo.
"""
excinfo = None
start = timing.time()
precise_start = timing.perf_counter()
try:
> result: Optional[TResult] = func()
cls = <class '_pytest.runner.CallInfo'>
duration = 0.7103413999939221
excinfo = <ExceptionInfo PytestUnraisableExceptionWarning('Exception ignored in: <socket.socket fd=-1, family=2, type=1, proto=0...et.socket fd=2124, family=2, type=1, proto=0, laddr=(\'127.0.0.1\', 59458), raddr=(\'127.0.0.1\', 59459)>\n') tblen=11>
func = <function call_and_report.<locals>.<lambda> at 0x0000013B0182FB00>
precise_start = 60518.1109472
precise_stop = 60518.8212886
reraise = (<class '_pytest.outcomes.Exit'>, <class 'KeyboardInterrupt'>)
result = None
start = 1749603690.353999
stop = 1749603691.0643415
when = 'call'
c:\code\jupyter_client\.venv\Lib\site-packages\_pytest\runner.py:340:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
c:\code\jupyter_client\.venv\Lib\site-packages\_pytest\runner.py:240: in <lambda>
lambda: runtest_hook(item=item, **kwds), when=when, reraise=reraise
item = <TestCaseFunction test_start_parallel_thread_kernels>
kwds = {}
runtest_hook = <HookCaller 'pytest_runtest_call'>
c:\code\jupyter_client\.venv\Lib\site-packages\pluggy\_hooks.py:512: in __call__
return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
firstresult = False
kwargs = {'item': <TestCaseFunction test_start_parallel_thread_kernels>}
self = <HookCaller 'pytest_runtest_call'>
c:\code\jupyter_client\.venv\Lib\site-packages\pluggy\_manager.py:120: in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
firstresult = False
hook_name = 'pytest_runtest_call'
kwargs = {'item': <TestCaseFunction test_start_parallel_thread_kernels>}
methods = [<HookImpl plugin_name='runner', plugin=<module '_pytest.runner' from 'c:\\code\\jupyter_client\\.venv\\Lib\\site-pack...nraisableexception' from 'c:\\code\\jupyter_client\\.venv\\Lib\\site-packages\\_pytest\\unraisableexception.py'>>, ...]
self = <_pytest.config.PytestPluginManager object at 0x0000013B7AD034D0>
c:\code\jupyter_client\.venv\Lib\site-packages\_pytest\threadexception.py:87: in pytest_runtest_call
yield from thread_exception_runtest_hook()
c:\code\jupyter_client\.venv\Lib\site-packages\_pytest\threadexception.py:63: in thread_exception_runtest_hook
yield
cm = <_pytest.threadexception.catch_threading_exception object at 0x0000013B018048F0>
c:\code\jupyter_client\.venv\Lib\site-packages\_pytest\unraisableexception.py:90: in pytest_runtest_call
yield from unraisable_exception_runtest_hook()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
def unraisable_exception_runtest_hook() -> Generator[None, None, None]:
with catch_unraisable_exception() as cm:
try:
yield
finally:
if cm.unraisable:
if cm.unraisable.err_msg is not None:
err_msg = cm.unraisable.err_msg
else:
err_msg = "Exception ignored in"
msg = f"{err_msg}: {cm.unraisable.object!r}\n\n"
msg += "".join(
traceback.format_exception(
cm.unraisable.exc_type,
cm.unraisable.exc_value,
cm.unraisable.exc_traceback,
)
)
> warnings.warn(pytest.PytestUnraisableExceptionWarning(msg))
E pytest.PytestUnraisableExceptionWarning: Exception ignored in: <socket.socket fd=-1, family=2, type=1, proto=0>
E
E Traceback (most recent call last):
E File "c:\code\jupyter_client\.venv\Lib\site-packages\traitlets\traitlets.py", line 632, in get
E value = obj._trait_values[self.name]
E ~~~~~~~~~~~~~~~~~^^^^^^^^^^^
E KeyError: 'kernel_spec_manager'
E
E During handling of the above exception, another exception occurred:
E
E Traceback (most recent call last):
E File "c:\code\jupyter_client\.venv\Lib\site-packages\traitlets\config\configurable.py", line 179, in _load_config
E with self.hold_trait_notifications():
E ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^
E ResourceWarning: unclosed <socket.socket fd=2124, family=2, type=1, proto=0, laddr=('127.0.0.1', 59458), raddr=('127.0.0.1', 59459)>
cm = <_pytest.unraisableexception.catch_unraisable_exception object at 0x0000013B018055B0>
err_msg = 'Exception ignored in'
msg = 'Exception ignored in: <socket.socket fd=-1, family=2, type=1, proto=0>\n\nTraceback (most recent call last):\n File ...losed <socket.socket fd=2124, family=2, type=1, proto=0, laddr=(\'127.0.0.1\', 59458), raddr=(\'127.0.0.1\', 59459)>\n'
c:\code\jupyter_client\.venv\Lib\site-packages\_pytest\unraisableexception.py:80: PytestUnraisableExceptionWarning |
|
I've almost got the CI back into a healthy state in #1076 then I'll come back here. I think this looks great! |
Co-authored-by: M Bussonnier <[email protected]>
|
I think you can rebase this one more time. CI is in a happier state now. |
|
I used github to do the merge. |
|
@rgbkrk - can you please re-run ci? |
|
Re-kicked it. I'm also fixing some other CI issues that have been hidden for a while (mostly our "downstream projects" testing). Should be ok in your PR though. |
|
|
I've cleaned up more of the flakiness that would occur with tests so hopefully we're in a real good spot on |
|
Since #aac1ecf it was always using the fallback because the argument name should have been 'option' not 'options' - sorry about that! |
Well, maybe if mypy was running on ci 👀 |
|
oh, it is running via hatch... not sure why it did not catch it. |
|
I find it a bit crazy that github action runs hatch, that runs precommit, that use a mypy-mirror to run an older version of mypy just to not catch errors. |
it's apparently a know issue that mypy in pre-commit is broken. It shoudl be removed and ran directly in GH action. |
|
This PR is only really being tested with minimum version. Would it be better to install orjson in the CI coverage check instead of minimum version, or both? If yes, I could use help to implement it. |
|
Re-triggered CI for failed jobs. |
|
The reason pre-commit fails is due to this line: That import tells mypy to treat |
|
You should add |
|
After adding those changes I restored the typo and it still passes with pre-commit. The typecheckers don't like it when the module could be None. More importantly, I discovered I'd forgotten to return from orjson.loads in |
|
You can protect it with I see errors from pre-commit if you make the edits you mentioned. By the way, pre-commit.ci does not show errors because this is a manual hook, it doesn't run manual hooks. |
|
Also, msgpack doesn't have stubs. You want |
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 5c74d89..fe14057 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -44,7 +44,7 @@ repos:
stages: [manual]
args: ["--install-types", "--non-interactive"]
additional_dependencies:
- ["traitlets>=5.13", "ipykernel>=6.26", "jupyter_core>=5.3.2"]
+ ["traitlets>=5.13", "ipykernel>=6.26", "jupyter_core>=5.3.2", "orjson", "msgpack-types", "types-pexpect", "types-paramiko", "types-netifaces", "types-psutil", "types-python-dateutil"]
- repo: https://github.com/adamchainz/blacken-docs
rev: "1.20.0"
diff --git a/jupyter_client/session.py b/jupyter_client/session.py
index 8191798..33c2183 100644
--- a/jupyter_client/session.py
+++ b/jupyter_client/session.py
@@ -128,9 +128,10 @@ def json_unpacker(s: str | bytes) -> t.Any:
try:
- import orjson # type:ignore[import-not-found]
+ import orjson
+ has_orjson = True
except ModuleNotFoundError:
- orjson = None
+ has_orjson = False
orjson_packer, orjson_unpacker = json_packer, json_unpacker
else:
@@ -139,7 +140,7 @@ else:
) -> bytes:
"""Convert a json object to a bytes using orjson with fallback to json_packer."""
try:
- return orjson.dumps(obj, default=json_default, option=option)
+ return orjson.dumps(obj, default=json_default, options=option)
except Exception:
pass
return json_packer(obj)
@@ -154,10 +155,11 @@ else:
try:
- import msgpack # type:ignore[import-not-found]
+ import msgpack
+ has_msgpack = True
except ModuleNotFoundError:
- msgpack = None
+ has_msgpack = False
else:
msgpack_packer = functools.partial(msgpack.packb, default=json_default)
msgpack_unpacker = msgpack.unpackb
@@ -389,31 +391,31 @@ class Session(Configurable):
# serialization traits:
packer = DottedObjectName(
- "orjson" if orjson else "json",
+ "orjson" if has_orjson else "json",
config=True,
help="""The name of the packer for serializing messages.
Should be one of 'json', 'pickle', or an import name
for a custom callable serializer.""",
)
unpacker = DottedObjectName(
- "orjson" if orjson else "json",
+ "orjson" if has_orjson else "json",
config=True,
help="""The name of the unpacker for unserializing messages.
Only used with custom functions for `packer`.""",
)
- pack = Callable(orjson_packer if orjson else json_packer) # the actual packer function
- unpack = Callable(orjson_unpacker if orjson else json_unpacker) # the actual unpacker function
+ pack = Callable(orjson_packer if has_orjson else json_packer) # the actual packer function
+ unpack = Callable(orjson_unpacker if has_orjson else json_unpacker) # the actual unpacker function
@observe("packer", "unpacker")
def _packer_unpacker_changed(self, change: t.Any) -> None:
new = change["new"].lower()
- if new == "orjson" and orjson:
+ if new == "orjson" and has_orjson:
self.pack, self.unpack = orjson_packer, orjson_unpacker
elif new == "json" or new == "orjson":
self.pack, self.unpack = json_packer, json_unpacker
elif new == "pickle":
self.pack, self.unpack = pickle_packer, pickle_unpacker
- elif new == "msgpack" and msgpack:
+ elif new == "msgpack" and has_msgpack:
self.pack, self.unpack = msgpack_packer, msgpack_unpacker
else:
obj = import_item(str(change["new"]))And this works: |
|
Can you push the changes? |
|
I'd also drop the |
|
Did you give me permissions to push on your repo? I'm not a maintainer here. Here's a patchable diff if that's better, based on the latest changes. Might have gotten the else part wrong, I can't run with diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index f9986dd..c18bafe 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -44,13 +44,11 @@ repos:
stages: [manual]
args: ["--install-types", "--non-interactive"]
additional_dependencies:
- [
- "traitlets>=5.13",
- "ipykernel>=6.26",
- "jupyter_core>=5.3.2",
- "orjson>=3.11.4",
- "msgpack>=1.1.2",
- ]
+ - traitlets>=5.13
+ - ipykernel>=6.26
+ - jupyter_core>=5.3.2
+ - orjson>=3.11.4
+ - msgpack-types
- repo: https://github.com/adamchainz/blacken-docs
rev: "1.20.0"
diff --git a/jupyter_client/session.py b/jupyter_client/session.py
index 2241965..f812ff6 100644
--- a/jupyter_client/session.py
+++ b/jupyter_client/session.py
@@ -128,20 +128,20 @@ def json_unpacker(s: str | bytes) -> t.Any:
return json.loads(s)
-orjson = None
-orjson_packer, orjson_unpacker = json_packer, json_unpacker
-
-if importlib.util.find_spec("orjson"):
+try:
import orjson
-
- assert orjson
+except ModuleNotFoundError:
+ has_orjson = False
+ orjson_packer, orjson_unpacker = json_packer, json_unpacker
+else:
+ has_orjson = True
def orjson_packer(
obj: t.Any, *, option: int | None = orjson.OPT_NAIVE_UTC | orjson.OPT_UTC_Z
) -> bytes:
"""Convert a json object to a bytes using orjson with fallback to json_packer."""
try:
- return orjson.dumps(obj, default=json_default, option=option)
+ return orjson.dumps(obj, default=json_default, options=option)
except Exception:
return json_packer(obj)
@@ -155,10 +155,10 @@ if importlib.util.find_spec("orjson"):
try:
import msgpack
-
except ModuleNotFoundError:
- msgpack = None
+ has_msgpack = False
else:
+ has_msgpack = True
msgpack_packer = functools.partial(msgpack.packb, default=json_default)
msgpack_unpacker = msgpack.unpackb
@@ -389,31 +389,31 @@ class Session(Configurable):
# serialization traits:
packer = DottedObjectName(
- "orjson" if orjson else "json",
+ "orjson" if has_orjson else "json",
config=True,
help="""The name of the packer for serializing messages.
Should be one of 'json', 'pickle', or an import name
for a custom callable serializer.""",
)
unpacker = DottedObjectName(
- "orjson" if orjson else "json",
+ "orjson" if has_orjson else "json",
config=True,
help="""The name of the unpacker for unserializing messages.
Only used with custom functions for `packer`.""",
)
- pack = Callable(orjson_packer if orjson else json_packer) # the actual packer function
- unpack = Callable(orjson_unpacker if orjson else json_unpacker) # the actual unpacker function
+ pack = Callable(orjson_packer if has_orjson else json_packer) # the actual packer function
+ unpack = Callable(orjson_unpacker if has_orjson else json_unpacker) # the actual unpacker function
@observe("packer", "unpacker")
def _packer_unpacker_changed(self, change: t.Any) -> None:
new = change["new"].lower()
- if new == "orjson" and orjson:
+ if new == "orjson" and has_orjson:
self.pack, self.unpack = orjson_packer, orjson_unpacker
elif new == "json" or new == "orjson":
self.pack, self.unpack = json_packer, json_unpacker
elif new == "pickle":
self.pack, self.unpack = pickle_packer, pickle_unpacker
- elif new == "msgpack" and msgpack:
+ elif new == "msgpack" and has_msgpack:
self.pack, self.unpack = msgpack_packer, msgpack_unpacker
else:
obj = import_item(str(change["new"])) |
Signed-off-by: Henry Schreiner <[email protected]>
for more information, see https://pre-commit.ci
|
Oops, should have run the non-mypy parts of pre-commit. :D |
6 participants
This PR provides faster pack and unpack defaults when
orjsona "fast, correct JSON library" is installed.Support is also added for msgpack.
New functions
orjson_packerorjson_unpackermsgpack_packermsgpack_unpackerPerformance comparison
Code