Skip to content

Commit 134d504

Browse files
eustascopybara-github
eustas
authored and
copybara-github
committed
Add py free-threaded support
Modernize testing (-> re-enable tests on Windows) PiperOrigin-RevId: 830892275
1 parent fa925d0 commit 134d504

File tree

4 files changed

+369
-333
lines changed

4 files changed

+369
-333
lines changed

‎.github/workflows/build_test.yml‎

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -133,12 +133,22 @@ jobs:
133133
c_compiler: clang
134134
cxx_compiler: clang++
135135

136-
- name: python3.10-win
136+
- name: python3.14:clang
137137
build_system: python
138-
python_version: "3.10"
139-
# TODO: investigate why win-builds can't run tests
140-
py_setuptools_cmd: build_ext
141-
os: windows-2022
138+
python_version: "3.14"
139+
c_compiler: clang
140+
cxx_compiler: clang++
141+
142+
- name: python3.14t:clang
143+
build_system: python
144+
python_version: "3.14t"
145+
c_compiler: clang
146+
cxx_compiler: clang++
147+
148+
- name: python3.14-win
149+
build_system: python
150+
python_version: "3.14"
151+
os: windows-latest
142152

143153
- name: maven
144154
build_system: maven
@@ -332,9 +342,9 @@ jobs:
332342
if: ${{ matrix.build_system == 'python' }}
333343
run: |
334344
python -VV
335-
python -c "import sys; sys.exit('Invalid python version') if '.'.join(map(str,sys.version_info[0:2])) != '${{ matrix.python_version }}' else True"
336-
pip install setuptools==51.3.3
337-
python setup.py ${{ matrix.py_setuptools_cmd || 'test'}}
345+
pip install "setuptools>=70.0.0" pytest
346+
python setup.py build_ext --inplace
347+
pytest ./python/tests
338348
339349
build_test_dotnet:
340350
name: Build and test with .NET

‎.github/workflows/codeql.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,8 @@ jobs:
7171
- if: matrix.language == 'cpp' || matrix.language == 'python'
7272
name: Build Python
7373
run: |
74+
python -VV
75+
pip install "setuptools>=70.0.0"
7476
python setup.py build_ext
7577
7678
- name: Perform CodeQL Analysis

‎python/_brotli.c‎

Lines changed: 101 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,28 @@
2222
#error "Only Python 3.10+ is supported"
2323
#endif
2424

25+
/*
26+
Decoder / encoder nature does not support concurrent access. Attempt to enter
27+
concurrently will result in an exception.
28+
29+
"Critical" parts used in prologues to ensure that only one thread enters.
30+
For consistency, we use them in epilogues as well. "Critical" is essential for
31+
free-threaded. In GIL environment those rendered as a scope (i.e. `{` and `}`).
32+
33+
NB: `Py_BEGIN_ALLOW_THREADS` / `Py_END_ALLOW_THREADS` are still required to
34+
unblock the stop-the-world GC.
35+
*/
36+
#ifdef Py_GIL_DISABLED
37+
#if PY_MAJOR_VERSION < 3 || (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 13)
38+
#error "Critical sections are only available in Python 3.13+"
39+
#endif
40+
#define BROTLI_CRITICAL_START Py_BEGIN_CRITICAL_SECTION(self)
41+
#define BROTLI_CRITICAL_END Py_END_CRITICAL_SECTION()
42+
#else
43+
#define BROTLI_CRITICAL_START {
44+
#define BROTLI_CRITICAL_END }
45+
#endif
46+
2547
static const char kErrorAttr[] = "error";
2648
static const char kModuleAttr[] = "_module";
2749

@@ -449,6 +471,33 @@ static void brotli_Compressor_dealloc(PyBrotli_Compressor* self) {
449471
Py_TYPE(self)->tp_free((PyObject*)self);
450472
}
451473

474+
static int brotli_compressor_enter(PyBrotli_Compressor* self) {
475+
PyObject* self_type = (PyObject*)Py_TYPE((PyObject*)self);
476+
int ok = 1;
477+
478+
BROTLI_CRITICAL_START;
479+
if (self->healthy == 0) {
480+
set_brotli_exception(self_type, kCompressUnhealthyError);
481+
ok = 0;
482+
}
483+
if (self->processing != 0) {
484+
set_brotli_exception(self_type, kCompressConcurrentError);
485+
ok = 0;
486+
}
487+
if (ok) {
488+
self->processing = 1;
489+
}
490+
BROTLI_CRITICAL_END;
491+
return ok;
492+
}
493+
494+
static void brotli_compressor_leave(PyBrotli_Compressor* self) {
495+
BROTLI_CRITICAL_START;
496+
assert(self->processing == 1);
497+
self->processing = 0;
498+
BROTLI_CRITICAL_END;
499+
}
500+
452501
/*
453502
Compress "utility knife" used for process / flush / finish.
454503
@@ -523,14 +572,7 @@ static PyObject* brotli_Compressor_process(PyBrotli_Compressor* self,
523572
PyObject* input_object = NULL;
524573
Py_buffer input;
525574

526-
if (self->healthy == 0) {
527-
set_brotli_exception(self_type, kCompressUnhealthyError);
528-
return NULL;
529-
}
530-
if (self->processing != 0) {
531-
set_brotli_exception(self_type, kCompressConcurrentError);
532-
return NULL;
533-
}
575+
if (!brotli_compressor_enter(self)) return NULL;
534576

535577
if (!PyArg_ParseTuple(args, "O:process", &input_object)) {
536578
return NULL;
@@ -539,49 +581,33 @@ static PyObject* brotli_Compressor_process(PyBrotli_Compressor* self,
539581
return NULL;
540582
}
541583

542-
self->processing = 1;
543584
ret = compress_stream(self, BROTLI_OPERATION_PROCESS, (uint8_t*)input.buf,
544585
input.len);
545586
PyBuffer_Release(&input);
546-
self->processing = 0;
587+
brotli_compressor_leave(self);
588+
547589
return ret;
548590
}
549591

550592
static PyObject* brotli_Compressor_flush(PyBrotli_Compressor* self) {
551593
PyObject* self_type = (PyObject*)Py_TYPE((PyObject*)self);
552594
PyObject* ret = NULL;
553595

554-
if (self->healthy == 0) {
555-
set_brotli_exception(self_type, kCompressUnhealthyError);
556-
return NULL;
557-
}
558-
if (self->processing != 0) {
559-
set_brotli_exception(self_type, kCompressConcurrentError);
560-
return NULL;
561-
}
562-
563-
self->processing = 1;
596+
if (!brotli_compressor_enter(self)) return NULL;
564597
ret = compress_stream(self, BROTLI_OPERATION_FLUSH, NULL, 0);
565-
self->processing = 0;
598+
brotli_compressor_leave(self);
599+
566600
return ret;
567601
}
568602

569603
static PyObject* brotli_Compressor_finish(PyBrotli_Compressor* self) {
570604
PyObject* self_type = (PyObject*)Py_TYPE((PyObject*)self);
571605
PyObject* ret = NULL;
572606

573-
if (self->healthy == 0) {
574-
set_brotli_exception(self_type, kCompressUnhealthyError);
575-
return NULL;
576-
}
577-
if (self->processing != 0) {
578-
set_brotli_exception(self_type, kCompressConcurrentError);
579-
return NULL;
580-
}
581-
582-
self->processing = 1;
607+
if (!brotli_compressor_enter(self)) return NULL;
583608
ret = compress_stream(self, BROTLI_OPERATION_FINISH, NULL, 0);
584-
self->processing = 0;
609+
brotli_compressor_leave(self);
610+
585611
if (ret != NULL) {
586612
assert(BrotliEncoderIsFinished(self->enc));
587613
}
@@ -639,6 +665,33 @@ static int brotli_Decompressor_init(PyBrotli_Decompressor* self, PyObject* args,
639665
return 0;
640666
}
641667

668+
static int brotli_decompressor_enter(PyBrotli_Decompressor* self) {
669+
PyObject* self_type = (PyObject*)Py_TYPE((PyObject*)self);
670+
int ok = 1;
671+
672+
BROTLI_CRITICAL_START;
673+
if (self->healthy == 0) {
674+
set_brotli_exception(self_type, kDecompressUnhealthyError);
675+
ok = 0;
676+
}
677+
if (self->processing != 0) {
678+
set_brotli_exception(self_type, kDecompressConcurrentError);
679+
ok = 0;
680+
}
681+
if (ok) {
682+
self->processing = 1;
683+
}
684+
BROTLI_CRITICAL_END;
685+
return ok;
686+
}
687+
688+
static void brotli_decompressor_leave(PyBrotli_Decompressor* self) {
689+
BROTLI_CRITICAL_START;
690+
assert(self->processing == 1);
691+
self->processing = 0;
692+
BROTLI_CRITICAL_END;
693+
}
694+
642695
static void brotli_Decompressor_dealloc(PyBrotli_Decompressor* self) {
643696
if (self->dec) BrotliDecoderDestroyInstance(self->dec);
644697
if (self->unconsumed_data) {
@@ -665,14 +718,7 @@ static PyObject* brotli_Decompressor_process(PyBrotli_Decompressor* self,
665718
size_t new_tail_length = 0;
666719
int oom = 0;
667720

668-
if (self->healthy == 0) {
669-
set_brotli_exception(self_type, kDecompressUnhealthyError);
670-
return NULL;
671-
}
672-
if (self->processing != 0) {
673-
set_brotli_exception(self_type, kDecompressConcurrentError);
674-
return NULL;
675-
}
721+
if (!brotli_decompressor_enter(self)) return NULL;
676722

677723
if (!PyArg_ParseTupleAndKeywords(args, keywds, "O|n:process", (char**)kwlist,
678724
&input_object, &output_buffer_limit)) {
@@ -683,7 +729,6 @@ static PyObject* brotli_Decompressor_process(PyBrotli_Decompressor* self,
683729
}
684730

685731
Buffer_Init(&buffer);
686-
self->processing = 1;
687732

688733
if (self->unconsumed_data_length > 0) {
689734
if (input.len > 0) {
@@ -769,21 +814,17 @@ static PyObject* brotli_Decompressor_process(PyBrotli_Decompressor* self,
769814
assert(ret == NULL);
770815
self->healthy = 0;
771816
}
772-
self->processing = 0;
817+
brotli_decompressor_leave(self);
818+
773819
return ret;
774820
}
775821

776822
static PyObject* brotli_Decompressor_is_finished(PyBrotli_Decompressor* self) {
777-
PyObject* self_type = (PyObject*)Py_TYPE((PyObject*)self);
778-
if (self->healthy == 0) {
779-
set_brotli_exception(self_type, kDecompressUnhealthyError);
780-
return NULL;
781-
}
782-
if (self->processing != 0) {
783-
set_brotli_exception(self_type, kDecompressConcurrentError);
784-
return NULL;
785-
}
786-
if (BrotliDecoderIsFinished(self->dec)) {
823+
int result;
824+
if (!brotli_decompressor_enter(self)) return NULL;
825+
result = BrotliDecoderIsFinished(self->dec);
826+
brotli_decompressor_leave(self);
827+
if (result) {
787828
Py_RETURN_TRUE;
788829
} else {
789830
Py_RETURN_FALSE;
@@ -792,16 +833,11 @@ static PyObject* brotli_Decompressor_is_finished(PyBrotli_Decompressor* self) {
792833

793834
static PyObject* brotli_Decompressor_can_accept_more_data(
794835
PyBrotli_Decompressor* self) {
795-
PyObject* self_type = (PyObject*)Py_TYPE((PyObject*)self);
796-
if (self->healthy == 0) {
797-
set_brotli_exception(self_type, kDecompressUnhealthyError);
798-
return NULL;
799-
}
800-
if (self->processing != 0) {
801-
set_brotli_exception(self_type, kDecompressConcurrentError);
802-
return NULL;
803-
}
804-
if (self->unconsumed_data_length > 0) {
836+
int result;
837+
if (!brotli_decompressor_enter(self)) return NULL;
838+
result = (self->unconsumed_data_length > 0);
839+
brotli_decompressor_leave(self);
840+
if (result) {
805841
Py_RETURN_FALSE;
806842
} else {
807843
Py_RETURN_TRUE;
@@ -1003,7 +1039,9 @@ static PyMethodDef brotli_methods[] = {
10031039

10041040
static PyModuleDef_Slot brotli_mod_slots[] = {
10051041
{Py_mod_exec, brotli_init_mod},
1006-
#if (PY_MAJOR_VERSION > 3) || (PY_MINOR_VERSION >= 12)
1042+
#ifdef Py_GIL_DISABLED
1043+
{Py_mod_gil, Py_MOD_GIL_NOT_USED},
1044+
#elif (PY_MAJOR_VERSION > 3) || (PY_MINOR_VERSION >= 12)
10071045
{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
10081046
#endif
10091047
{0, NULL}};

0 commit comments

Comments
 (0)