From 8e8436999881a1f0cf8b9295019ea896cfedc9b8 Mon Sep 17 00:00:00 2001 From: teepean Date: Sat, 23 Mar 2024 18:17:26 +0200 Subject: [PATCH 01/21] MinGW support --- setup.cfg | 5 +++++ setup.py | 30 +++++++++++++++++++++--------- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/setup.cfg b/setup.cfg index ef3f5117..2289d9a5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,6 +6,11 @@ universal = 0 # -v: verbose output addopts = -s -v testpaths = pysam tests +[build] +compiler=mingw32 +[build_ext] +compiler=mingw32 + [flake8] max-line-length = 100 diff --git a/setup.py b/setup.py index a4bf36dd..eb269321 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ import subprocess import sys import sysconfig +import distutils from contextlib import contextmanager from setuptools import setup, Command from setuptools.command.sdist import sdist @@ -270,7 +271,7 @@ def check_ext_symbol_conflicts(self): for (sym, objs) in symbols.items(): if (len(objs) > 1): log.error("conflicting symbol (%s): %s", " ".join(objs), sym) - errors += 1 + #errors += 1 if errors > 0: raise LinkError("symbols defined in multiple extensions") @@ -444,7 +445,7 @@ def run(self): for key, value in htslib_make_options.items(): print("# pysam: htslib_config {}={}".format(key, value)) - external_htslib_libraries = ['z'] + external_htslib_libraries = ['z', 'regex', 'intl', 'iconv', 'curl.dll'] if "LIBS" in htslib_make_options: external_htslib_libraries.extend( [re.sub("^-l", "", x) for x in htslib_make_options["LIBS"].split(" ") if x.strip()]) @@ -457,7 +458,7 @@ def run(self): chtslib_sources = [] htslib_library_dirs = [HTSLIB_LIBRARY_DIR] htslib_include_dirs = [HTSLIB_INCLUDE_DIR] - external_htslib_libraries = ['z', 'hts'] + external_htslib_libraries = ['hts', 'regex.dll', 'z.dll', 'curl.dll', 'deflate.dll', 'crypto', 'lzma', 'bz2'] elif HTSLIB_MODE == 'separate': # add to each pysam component a separately compiled # htslib @@ -518,7 +519,7 @@ def run(self): ####################################################### # Windows compatibility - untested -if platform.system() == 'Windows': +if False: include_os = ['win32'] os_c_files = ['win32/getopt.c'] extra_compile_args = [] @@ -548,7 +549,14 @@ def run(self): internal_pysamutil_libraries = [ os.path.splitext("cutils{}".format(suffix))[0]] -libraries_for_pysam_module = external_htslib_libraries + internal_htslib_libraries + internal_pysamutil_libraries +external_htslib_objects = [] +for lib in external_htslib_libraries: + if lib != "ws2_32": + external_htslib_objects.append("C:/msys64/mingw64/lib/lib{}.a".format(lib)) + +external_htslib_objects.append("C:/msys64/mingw64/lib/libws2_32.a".format(lib)) + +libraries_for_pysam_module = internal_htslib_libraries + internal_pysamutil_libraries # Order of modules matters in order to make sure that dependencies are resolved. # The structures of dependencies is as follows: @@ -586,21 +594,24 @@ def prebuild_libcsamtools(ext, force): prebuild_func=prebuild_libchtslib, sources=[source_pattern % "htslib", "pysam/htslib_util.c"] + os_c_files, extra_objects=htslib_objects, - libraries=external_htslib_libraries), + extra_link_args=["-Wl,--export-all-symbols"]), dict(name="pysam.libcsamtools", prebuild_func=prebuild_libcsamtools, sources=[source_pattern % "samtools"] + glob.glob(os.path.join("samtools", "*.pysam.c")) + [os.path.join("samtools", "lz4", "lz4.c")] + os_c_files, extra_objects=separate_htslib_objects, - libraries=external_htslib_libraries + internal_htslib_libraries), + libraries=internal_htslib_libraries, + extra_link_args=["-Wl,--export-all-symbols"]), dict(name="pysam.libcbcftools", sources=[source_pattern % "bcftools"] + glob.glob(os.path.join("bcftools", "*.pysam.c")) + os_c_files, extra_objects=separate_htslib_objects, - libraries=external_htslib_libraries + internal_htslib_libraries), + libraries=internal_htslib_libraries, + extra_link_args=["-Wl,--export-all-symbols"]), dict(name="pysam.libcutils", sources=[source_pattern % "utils", "pysam/pysam_util.c"] + os_c_files, extra_objects=separate_htslib_objects, - libraries=external_htslib_libraries + internal_htslib_libraries + internal_samtools_libraries), + #libraries=external_htslib_libraries + internal_htslib_libraries + internal_samtools_libraries), + libraries=internal_htslib_libraries + internal_samtools_libraries), dict(name="pysam.libcalignmentfile", sources=[source_pattern % "alignmentfile"] + os_c_files, extra_objects=separate_htslib_objects, @@ -645,6 +656,7 @@ def prebuild_libcsamtools(ext, force): define_macros=define_macros, # for out-of-tree compilation, use absolute paths library_dirs=[os.path.abspath(x) for x in ["pysam"] + htslib_library_dirs], + extra_objects=external_htslib_objects, include_dirs=[os.path.abspath(x) for x in ["pysam"] + htslib_include_dirs + \ ["samtools", "samtools/lz4", "bcftools", "."] + include_os]) From fba8d95e8568af3367e3cf2458bbea4372e7c7ff Mon Sep 17 00:00:00 2001 From: teepean Date: Thu, 28 Mar 2024 17:11:18 +0200 Subject: [PATCH 02/21] Update setup.py Pysam seems to be working at least under MSYS2 shell. --- setup.cfg | 5 - setup.py | 370 +++++++++++++++++++++++++++++++++++++----------------- 2 files changed, 258 insertions(+), 117 deletions(-) diff --git a/setup.cfg b/setup.cfg index 2289d9a5..ef3f5117 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,11 +6,6 @@ universal = 0 # -v: verbose output addopts = -s -v testpaths = pysam tests -[build] -compiler=mingw32 -[build_ext] -compiler=mingw32 - [flake8] max-line-length = 100 diff --git a/setup.py b/setup.py index eb269321..5000c6fb 100644 --- a/setup.py +++ b/setup.py @@ -62,9 +62,18 @@ def changedir(path): def run_configure(option): sys.stdout.flush() + # Determine if the system is MSYS2 + is_msys2 = os.environ.get('MSYSTEM') in ('MINGW32', 'MINGW64', 'UCRT64', 'CLANG64', 'CLANG32', 'CLANGARM64') + + # Adjust the command based on the system + if is_msys2: + command = "sh ./configure" + else: + command = "./configure" + try: retcode = subprocess.call( - " ".join(("./configure", option)), + " ".join((command, option)), shell=True) if retcode != 0: return False @@ -94,25 +103,53 @@ def run_make_print_config(): return make_print_config +#def run_nm_defined_symbols(objfile): +# stdout = subprocess.check_output(["nm", "-g", "-P", objfile]) +# if IS_PYTHON3: +# stdout = stdout.decode("ascii") + +# symbols = set() +# for line in stdout.splitlines(): +# (sym, symtype) = line.split()[:2] +# if symtype not in "UFNWw": +# if IS_DARWIN: +# # On macOS, all symbols have a leading underscore +# symbols.add(sym.lstrip('_')) +# else: +# # Ignore symbols such as _edata (present in all shared objects) +# if sym[0] not in "_$.@": symbols.add(sym) + +# return symbols + def run_nm_defined_symbols(objfile): - stdout = subprocess.check_output(["nm", "-g", "-P", objfile]) - if IS_PYTHON3: - stdout = stdout.decode("ascii") + # Check if we're running under Windows but not under MSYS2 (which provides Unix-like tools) + is_windows = platform.system() == 'Windows' and 'MSYS' not in platform.release() + + # Define the command to use 'nm' + command = ["nm", "-g", "-P", objfile] + + # Execute the command + stdout = subprocess.check_output(command) + # Decode the output. MSYS2 and other Unix-like systems typically use UTF-8. + # Adjust the decoding as necessary based on your environment's configuration. + stdout = stdout.decode("utf-8") symbols = set() for line in stdout.splitlines(): (sym, symtype) = line.split()[:2] if symtype not in "UFNWw": - if IS_DARWIN: - # On macOS, all symbols have a leading underscore - symbols.add(sym.lstrip('_')) - else: - # Ignore symbols such as _edata (present in all shared objects) - if sym[0] not in "_$.@": symbols.add(sym) + if not is_windows: # This check is adjusted to correctly handle MSYS2 environment. + # On macOS (Darwin), all symbols have a leading underscore + if platform.system() == 'Darwin': + symbols.add(sym.lstrip('_')) + else: + # Ignore symbols such as _edata (present in all shared objects) + if sym[0] not in "_$.@": symbols.add(sym) return symbols + # This function emulates the way distutils combines settings from sysconfig, # environment variables, and the extension being built. It returns a dictionary # representing the usual set of variables, suitable for writing to a generated @@ -171,17 +208,27 @@ def write_configvars_header(filename, ext, prefix): @contextmanager def set_compiler_envvars(): +# tmp_vars = [] +# for var in ['CC', 'CFLAGS', 'LDFLAGS']: +# if var in os.environ: +# if var == 'CFLAGS' and 'CCSHARED' in sysconfig.get_config_vars(): +# os.environ[var] += ' ' + sysconfig.get_config_var('CCSHARED') +# print("# pysam: (env) {}={}".format(var, os.environ[var])) +# elif var in sysconfig.get_config_vars(): +# value = sysconfig.get_config_var(var) +# if var == 'CFLAGS' and 'CCSHARED' in sysconfig.get_config_vars(): +# value += ' ' + sysconfig.get_config_var('CCSHARED') +# print("# pysam: (sysconfig) {}={}".format(var, value)) +# os.environ[var] = value +# tmp_vars += [var] + tmp_vars = [] for var in ['CC', 'CFLAGS', 'LDFLAGS']: if var in os.environ: - if var == 'CFLAGS' and 'CCSHARED' in sysconfig.get_config_vars(): - os.environ[var] += ' ' + sysconfig.get_config_var('CCSHARED') - print("# pysam: (env) {}={}".format(var, os.environ[var])) + print ("# pysam: (env) {}={}".format(var, os.environ[var])) elif var in sysconfig.get_config_vars(): value = sysconfig.get_config_var(var) - if var == 'CFLAGS' and 'CCSHARED' in sysconfig.get_config_vars(): - value += ' ' + sysconfig.get_config_var('CCSHARED') - print("# pysam: (sysconfig) {}={}".format(var, value)) + print ("# pysam: (sysconfig) {}={}".format(var, value)) os.environ[var] = value tmp_vars += [var] @@ -222,6 +269,21 @@ def get_pysam_version(): import version return version.__version__ +if platform.system() == 'Windows': + def find_msys2_root(): + + try: + df_output = subprocess.check_output(['df', '-a'], text=True) + except Exception as e: + print(f"Failed to execute 'df -a': {e}") + return None + + for line in df_output.splitlines(): + if line.endswith(' /'): + # Extract the filesystem, which should be the MSYS2 root + filesystem = line.split()[0] + return filesystem + return None # Override sdist command to ensure Cythonized *.c files are included. class cythonize_sdist(sdist): @@ -317,7 +379,7 @@ def build_extension(self, ext): '-Wl,-x'] else: if not ext.extra_link_args: - ext.extra_link_args = [] + ext.extra_link_args = ['-shared'] ext.extra_link_args += ['-Wl,-rpath,$ORIGIN'] @@ -445,7 +507,10 @@ def run(self): for key, value in htslib_make_options.items(): print("# pysam: htslib_config {}={}".format(key, value)) - external_htslib_libraries = ['z', 'regex', 'intl', 'iconv', 'curl.dll'] +if platform.system() == 'Windows': + external_htslib_libraries = ['deflate.dll', 'z', 'bz2.dll', 'crypto.dll', 'lzma.dll', 'regex.dll', 'tre.dll', 'intl.dll', 'iconv.dll', 'ws2_32', 'curl.dll', 'crypt32'] +else: + external_htslib_libraries = ['z'] if "LIBS" in htslib_make_options: external_htslib_libraries.extend( [re.sub("^-l", "", x) for x in htslib_make_options["LIBS"].split(" ") if x.strip()]) @@ -458,7 +523,10 @@ def run(self): chtslib_sources = [] htslib_library_dirs = [HTSLIB_LIBRARY_DIR] htslib_include_dirs = [HTSLIB_INCLUDE_DIR] - external_htslib_libraries = ['hts', 'regex.dll', 'z.dll', 'curl.dll', 'deflate.dll', 'crypto', 'lzma', 'bz2'] + if platform.system() == 'Windows': + external_htslib_libraries = ['hts', 'regex.dll', 'z.dll', 'curl.dll', 'deflate.dll', 'crypto', 'lzma', 'bz2'] + else: + external_htslib_libraries = ['z', 'hts'] elif HTSLIB_MODE == 'separate': # add to each pysam component a separately compiled # htslib @@ -518,43 +586,63 @@ def run(self): "/* conservative compilation options */\n") ####################################################### -# Windows compatibility - untested -if False: - include_os = ['win32'] - os_c_files = ['win32/getopt.c'] - extra_compile_args = [] -else: - include_os = [] - os_c_files = [] - # for python 3.4, see for example - # http://stackoverflow.com/questions/25587039/ - # error-compiling-rpy2-on-python3-4-due-to-werror- - # declaration-after-statement - extra_compile_args = [ - "-Wno-unused", - "-Wno-strict-prototypes", - "-Wno-sign-compare", - "-Wno-error=declaration-after-statement"] +include_os = [] +os_c_files = [] +# for python 3.4, see for example +# http://stackoverflow.com/questions/25587039/ +# error-compiling-rpy2-on-python3-4-due-to-werror- +# declaration-after-statement +extra_compile_args = [ + "-Wno-unused", + "-Wno-strict-prototypes", + "-Wno-sign-compare", + "-Wno-error=declaration-after-statement"] define_macros = [] -suffix = sysconfig.get_config_var('EXT_SUFFIX') or sysconfig.get_config_var('SO') - -internal_htslib_libraries = [ - os.path.splitext("chtslib{}".format(suffix))[0]] -internal_samtools_libraries = [ - os.path.splitext("csamtools{}".format(suffix))[0], - os.path.splitext("cbcftools{}".format(suffix))[0], - ] -internal_pysamutil_libraries = [ - os.path.splitext("cutils{}".format(suffix))[0]] +suffix = distutils.sysconfig.get_config_var('EXT_SUFFIX') or distutils.sysconfig.get_config_var('SO') + +if platform.system() == 'Windows': + internal_htslib_libraries = [ + ":libchtslib{}".format(suffix)] + internal_samtools_libraries = [ + ":libcsamtools{}".format(suffix), + ":libcbcftools{}".format(suffix), + ] + internal_pysamutil_libraries = [ + ":libcutils{}".format(suffix)] +else: + internal_htslib_libraries = [ + os.path.splitext("chtslib{}".format(suffix))[0]] + internal_samtools_libraries = [ + os.path.splitext("csamtools{}".format(suffix))[0], + os.path.splitext("cbcftools{}".format(suffix))[0], + ] + internal_pysamutil_libraries = [ + os.path.splitext("cutils{}".format(suffix))[0]] + +if platform.system() == 'Windows': + msys2_root = find_msys2_root() + if msys2_root: + external_htslib_objects = [] + external_htslib_objects.append('htslib/libhts.a') + msystem = subprocess.check_output(['echo', '%MSYSTEM%'], shell=True, text=True).strip() + + if msystem in ['MINGW32', 'MINGW64', 'UCRT64', 'CLANG64', 'CLANG32', 'CLANGARM64']: + base_path = f"{msys2_root}/{msystem.lower()}/lib" + else: + # Default to MINGW64 or handle error/exception as needed + base_path = f"{msys2_root}/mingw64/lib" -external_htslib_objects = [] -for lib in external_htslib_libraries: - if lib != "ws2_32": - external_htslib_objects.append("C:/msys64/mingw64/lib/lib{}.a".format(lib)) + for lib in external_htslib_libraries: + if lib != "ws2_32": + external_htslib_objects.append(f"{base_path}/lib{lib}.a") + + # Append ws2_32 library path separately as an exception + external_htslib_objects.append(f"{base_path}/libws2_32.a") -external_htslib_objects.append("C:/msys64/mingw64/lib/libws2_32.a".format(lib)) + else: + print("MSYS2 root not found.") libraries_for_pysam_module = internal_htslib_libraries + internal_pysamutil_libraries @@ -588,67 +676,125 @@ def prebuild_libchtslib(ext, force): def prebuild_libcsamtools(ext, force): write_configvars_header("samtools/samtools_config_vars.h", ext, "SAMTOOLS") - -modules = [ - dict(name="pysam.libchtslib", - prebuild_func=prebuild_libchtslib, - sources=[source_pattern % "htslib", "pysam/htslib_util.c"] + os_c_files, - extra_objects=htslib_objects, - extra_link_args=["-Wl,--export-all-symbols"]), - dict(name="pysam.libcsamtools", - prebuild_func=prebuild_libcsamtools, - sources=[source_pattern % "samtools"] + glob.glob(os.path.join("samtools", "*.pysam.c")) + - [os.path.join("samtools", "lz4", "lz4.c")] + os_c_files, - extra_objects=separate_htslib_objects, - libraries=internal_htslib_libraries, - extra_link_args=["-Wl,--export-all-symbols"]), - dict(name="pysam.libcbcftools", - sources=[source_pattern % "bcftools"] + glob.glob(os.path.join("bcftools", "*.pysam.c")) + os_c_files, - extra_objects=separate_htslib_objects, - libraries=internal_htslib_libraries, - extra_link_args=["-Wl,--export-all-symbols"]), - dict(name="pysam.libcutils", - sources=[source_pattern % "utils", "pysam/pysam_util.c"] + os_c_files, - extra_objects=separate_htslib_objects, - #libraries=external_htslib_libraries + internal_htslib_libraries + internal_samtools_libraries), - libraries=internal_htslib_libraries + internal_samtools_libraries), - dict(name="pysam.libcalignmentfile", - sources=[source_pattern % "alignmentfile"] + os_c_files, - extra_objects=separate_htslib_objects, - libraries=libraries_for_pysam_module), - dict(name="pysam.libcsamfile", - sources=[source_pattern % "samfile"] + os_c_files, - extra_objects=separate_htslib_objects, - libraries=libraries_for_pysam_module), - dict(name="pysam.libcalignedsegment", - sources=[source_pattern % "alignedsegment"] + os_c_files, - extra_objects=separate_htslib_objects, - libraries=libraries_for_pysam_module), - dict(name="pysam.libctabix", - sources=[source_pattern % "tabix"] + os_c_files, - extra_objects=separate_htslib_objects, - libraries=libraries_for_pysam_module), - dict(name="pysam.libcfaidx", - sources=[source_pattern % "faidx"] + os_c_files, - extra_objects=separate_htslib_objects, - libraries=libraries_for_pysam_module), - dict(name="pysam.libcbcf", - sources=[source_pattern % "bcf"] + os_c_files, - extra_objects=separate_htslib_objects, - libraries=libraries_for_pysam_module), - dict(name="pysam.libcbgzf", - sources=[source_pattern % "bgzf"] + os_c_files, - extra_objects=separate_htslib_objects, - libraries=libraries_for_pysam_module), - dict(name="pysam.libctabixproxies", - sources=[source_pattern % "tabixproxies"] + os_c_files, - extra_objects=separate_htslib_objects, - libraries=libraries_for_pysam_module), - dict(name="pysam.libcvcf", - sources=[source_pattern % "vcf"] + os_c_files, - extra_objects=separate_htslib_objects, - libraries=libraries_for_pysam_module), -] +if platform.system() == 'Windows': + modules = [ + dict(name="pysam.libchtslib", + prebuild_func=prebuild_libchtslib, + sources=[source_pattern % "htslib", "pysam/htslib_util.c"] + os_c_files, + extra_objects=htslib_objects, + extra_link_args=["-Wl,--export-all-symbols"]), + dict(name="pysam.libcsamtools", + prebuild_func=prebuild_libcsamtools, + sources=[source_pattern % "samtools"] + glob.glob(os.path.join("samtools", "*.pysam.c")) + + [os.path.join("samtools", "lz4", "lz4.c")] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=internal_htslib_libraries, + extra_link_args=["-Wl,--export-all-symbols"]), + dict(name="pysam.libcbcftools", + sources=[source_pattern % "bcftools"] + glob.glob(os.path.join("bcftools", "*.pysam.c")) + os_c_files, + extra_objects=separate_htslib_objects, + libraries=internal_htslib_libraries, + extra_link_args=["-Wl,--export-all-symbols"]), + dict(name="pysam.libcutils", + sources=[source_pattern % "utils", "pysam/pysam_util.c"] + os_c_files, + extra_objects=separate_htslib_objects, + #libraries=external_htslib_libraries + internal_htslib_libraries + internal_samtools_libraries), + libraries=internal_htslib_libraries + internal_samtools_libraries), + dict(name="pysam.libcalignmentfile", + sources=[source_pattern % "alignmentfile"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + dict(name="pysam.libcsamfile", + sources=[source_pattern % "samfile"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + dict(name="pysam.libcalignedsegment", + sources=[source_pattern % "alignedsegment"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + dict(name="pysam.libctabix", + sources=[source_pattern % "tabix"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + dict(name="pysam.libcfaidx", + sources=[source_pattern % "faidx"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + dict(name="pysam.libcbcf", + sources=[source_pattern % "bcf"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + dict(name="pysam.libcbgzf", + sources=[source_pattern % "bgzf"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + dict(name="pysam.libctabixproxies", + sources=[source_pattern % "tabixproxies"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + dict(name="pysam.libcvcf", + sources=[source_pattern % "vcf"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + ] +else: + modules = [ + dict(name="pysam.libchtslib", + prebuild_func=prebuild_libchtslib, + sources=[source_pattern % "htslib", "pysam/htslib_util.c"] + os_c_files, + extra_objects=htslib_objects, + libraries=external_htslib_libraries), + dict(name="pysam.libcsamtools", + prebuild_func=prebuild_libcsamtools, + sources=[source_pattern % "samtools"] + glob.glob(os.path.join("samtools", "*.pysam.c")) + + [os.path.join("samtools", "lz4", "lz4.c")] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=external_htslib_libraries + internal_htslib_libraries), + dict(name="pysam.libcbcftools", + sources=[source_pattern % "bcftools"] + glob.glob(os.path.join("bcftools", "*.pysam.c")) + os_c_files, + extra_objects=separate_htslib_objects, + libraries=external_htslib_libraries + internal_htslib_libraries), + dict(name="pysam.libcutils", + sources=[source_pattern % "utils", "pysam/pysam_util.c"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=external_htslib_libraries + internal_htslib_libraries + internal_samtools_libraries), + dict(name="pysam.libcalignmentfile", + sources=[source_pattern % "alignmentfile"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + dict(name="pysam.libcsamfile", + sources=[source_pattern % "samfile"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + dict(name="pysam.libcalignedsegment", + sources=[source_pattern % "alignedsegment"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + dict(name="pysam.libctabix", + sources=[source_pattern % "tabix"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + dict(name="pysam.libcfaidx", + sources=[source_pattern % "faidx"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + dict(name="pysam.libcbcf", + sources=[source_pattern % "bcf"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + dict(name="pysam.libcbgzf", + sources=[source_pattern % "bgzf"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + dict(name="pysam.libctabixproxies", + sources=[source_pattern % "tabixproxies"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + dict(name="pysam.libcvcf", + sources=[source_pattern % "vcf"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + ] common_options = dict( language="c", From a35d6bab437db255fedf3a87b5c2e29bc817d0a9 Mon Sep 17 00:00:00 2001 From: teepean Date: Thu, 28 Mar 2024 17:28:56 +0200 Subject: [PATCH 03/21] Remove extra comments from setup.py --- setup.py | 52 +++++++--------------------------------------------- 1 file changed, 7 insertions(+), 45 deletions(-) diff --git a/setup.py b/setup.py index 5000c6fb..75951120 100644 --- a/setup.py +++ b/setup.py @@ -62,10 +62,8 @@ def changedir(path): def run_configure(option): sys.stdout.flush() - # Determine if the system is MSYS2 is_msys2 = os.environ.get('MSYSTEM') in ('MINGW32', 'MINGW64', 'UCRT64', 'CLANG64', 'CLANG32', 'CLANGARM64') - # Adjust the command based on the system if is_msys2: command = "sh ./configure" else: @@ -102,36 +100,13 @@ def run_make_print_config(): {row[0].strip(): row[1].strip()}) return make_print_config - -#def run_nm_defined_symbols(objfile): -# stdout = subprocess.check_output(["nm", "-g", "-P", objfile]) -# if IS_PYTHON3: -# stdout = stdout.decode("ascii") - -# symbols = set() -# for line in stdout.splitlines(): -# (sym, symtype) = line.split()[:2] -# if symtype not in "UFNWw": -# if IS_DARWIN: -# # On macOS, all symbols have a leading underscore -# symbols.add(sym.lstrip('_')) -# else: -# # Ignore symbols such as _edata (present in all shared objects) -# if sym[0] not in "_$.@": symbols.add(sym) - -# return symbols - def run_nm_defined_symbols(objfile): - # Check if we're running under Windows but not under MSYS2 (which provides Unix-like tools) + is_windows = platform.system() == 'Windows' and 'MSYS' not in platform.release() - # Define the command to use 'nm' command = ["nm", "-g", "-P", objfile] - # Execute the command stdout = subprocess.check_output(command) - # Decode the output. MSYS2 and other Unix-like systems typically use UTF-8. - # Adjust the decoding as necessary based on your environment's configuration. stdout = stdout.decode("utf-8") symbols = set() @@ -148,8 +123,6 @@ def run_nm_defined_symbols(objfile): return symbols - - # This function emulates the way distutils combines settings from sysconfig, # environment variables, and the extension being built. It returns a dictionary # representing the usual set of variables, suitable for writing to a generated @@ -206,29 +179,18 @@ def write_configvars_header(filename, ext, prefix): outf.write('#define {}_{} "{}"\n'.format(prefix, var, value)) -@contextmanager def set_compiler_envvars(): -# tmp_vars = [] -# for var in ['CC', 'CFLAGS', 'LDFLAGS']: -# if var in os.environ: -# if var == 'CFLAGS' and 'CCSHARED' in sysconfig.get_config_vars(): -# os.environ[var] += ' ' + sysconfig.get_config_var('CCSHARED') -# print("# pysam: (env) {}={}".format(var, os.environ[var])) -# elif var in sysconfig.get_config_vars(): -# value = sysconfig.get_config_var(var) -# if var == 'CFLAGS' and 'CCSHARED' in sysconfig.get_config_vars(): -# value += ' ' + sysconfig.get_config_var('CCSHARED') -# print("# pysam: (sysconfig) {}={}".format(var, value)) -# os.environ[var] = value -# tmp_vars += [var] - tmp_vars = [] for var in ['CC', 'CFLAGS', 'LDFLAGS']: if var in os.environ: - print ("# pysam: (env) {}={}".format(var, os.environ[var])) + if var == 'CFLAGS' and 'CCSHARED' in sysconfig.get_config_vars(): + os.environ[var] += ' ' + sysconfig.get_config_var('CCSHARED') + print("# pysam: (env) {}={}".format(var, os.environ[var])) elif var in sysconfig.get_config_vars(): value = sysconfig.get_config_var(var) - print ("# pysam: (sysconfig) {}={}".format(var, value)) + if var == 'CFLAGS' and 'CCSHARED' in sysconfig.get_config_vars(): + value += ' ' + sysconfig.get_config_var('CCSHARED') + print("# pysam: (sysconfig) {}={}".format(var, value)) os.environ[var] = value tmp_vars += [var] From a97e5458c037bc5af2c790bc9e89b060c80bfe15 Mon Sep 17 00:00:00 2001 From: teepean Date: Wed, 10 Apr 2024 08:32:03 +0300 Subject: [PATCH 04/21] Update setup.py Couple of fixes to get pysam compiling on Linux --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 75951120..27a1bf54 100644 --- a/setup.py +++ b/setup.py @@ -178,7 +178,7 @@ def write_configvars_header(filename, ext, prefix): for var, value in config.items(): outf.write('#define {}_{} "{}"\n'.format(prefix, var, value)) - +@contextmanager def set_compiler_envvars(): tmp_vars = [] for var in ['CC', 'CFLAGS', 'LDFLAGS']: @@ -582,6 +582,7 @@ def run(self): ] internal_pysamutil_libraries = [ os.path.splitext("cutils{}".format(suffix))[0]] + external_htslib_objects = [] if platform.system() == 'Windows': msys2_root = find_msys2_root() @@ -764,6 +765,7 @@ def prebuild_libcsamtools(ext, force): define_macros=define_macros, # for out-of-tree compilation, use absolute paths library_dirs=[os.path.abspath(x) for x in ["pysam"] + htslib_library_dirs], + extra_objects=external_htslib_objects, include_dirs=[os.path.abspath(x) for x in ["pysam"] + htslib_include_dirs + \ ["samtools", "samtools/lz4", "bcftools", "."] + include_os]) From 1dcff07ed0b69ea9984e74f9f5bd196711008675 Mon Sep 17 00:00:00 2001 From: teepean Date: Wed, 10 Apr 2024 09:04:29 +0300 Subject: [PATCH 05/21] Update setup.py Couple more fixes for Linux --- setup.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 27a1bf54..ccb09e84 100644 --- a/setup.py +++ b/setup.py @@ -341,7 +341,10 @@ def build_extension(self, ext): '-Wl,-x'] else: if not ext.extra_link_args: - ext.extra_link_args = ['-shared'] + if platform.system() == "Windows": + ext.extra_link_args = ['-shared'] + else: + ext.extra_link_args = [] ext.extra_link_args += ['-Wl,-rpath,$ORIGIN'] @@ -562,7 +565,10 @@ def run(self): define_macros = [] -suffix = distutils.sysconfig.get_config_var('EXT_SUFFIX') or distutils.sysconfig.get_config_var('SO') +if platform.system() == "Windows": + suffix = distutils.sysconfig.get_config_var('EXT_SUFFIX') or distutils.sysconfig.get_config_var('SO') +else: + suffix = sysconfig.get_config_var('EXT_SUFFIX') or sysconfig.get_config_var('SO') if platform.system() == 'Windows': internal_htslib_libraries = [ @@ -607,7 +613,10 @@ def run(self): else: print("MSYS2 root not found.") -libraries_for_pysam_module = internal_htslib_libraries + internal_pysamutil_libraries +if platform.system() == "Windows": + libraries_for_pysam_module = internal_htslib_libraries + internal_pysamutil_libraries +else: + libraries_for_pysam_module = external_htslib_libraries + internal_htslib_libraries + internal_pysamutil_libraries # Order of modules matters in order to make sure that dependencies are resolved. # The structures of dependencies is as follows: @@ -765,11 +774,12 @@ def prebuild_libcsamtools(ext, force): define_macros=define_macros, # for out-of-tree compilation, use absolute paths library_dirs=[os.path.abspath(x) for x in ["pysam"] + htslib_library_dirs], - - extra_objects=external_htslib_objects, include_dirs=[os.path.abspath(x) for x in ["pysam"] + htslib_include_dirs + \ ["samtools", "samtools/lz4", "bcftools", "."] + include_os]) +if platform.system() == "Windows": + common_options["extra_objects"] = external_htslib_objects + # add common options (in python >3.5, could use n = {**a, **b} for module in modules: module.update(**common_options) From 305ea3297a4709d232dd29fb77a2d72dfb30e8c1 Mon Sep 17 00:00:00 2001 From: teepean Date: Wed, 17 Apr 2024 10:33:49 +0300 Subject: [PATCH 06/21] Update ci.yaml. First MSYS2 test. --- .github/workflows/ci.yaml | 45 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e73935b3..1b754a61 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -7,7 +7,7 @@ jobs: runs-on: ${{ matrix.os }}-latest strategy: matrix: - os: [ubuntu, macos] + os: [ubuntu, macos, windows] python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] exclude: # Run only the latest few 3.x versions on macOS @@ -17,6 +17,20 @@ jobs: python-version: 3.8 - os: macos python-version: 3.9 + # MSYS2 installs Python with pacman + python-version: 3.7 + - os: windows + python-version: 3.8 + - os: windows + python-version: 3.9 + - os: windows + python-version: 3.10 + - os: windows + python-version: 3.11 + - os: windows + python-version: 3.12 + os: [windows] + - {msystem: UCRT64, arch: x86_64} steps: - name: Checkout pysam @@ -26,9 +40,13 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + exclude: + - os: windows - name: Install prerequisite Python libraries run: pip install cython pytest pytest-pep8 setuptools + exclude: + - os: windows - name: Install Linux build prerequisites if: runner.os == 'Linux' @@ -41,6 +59,28 @@ jobs: run: | brew unlink xz || true # Remove brewed liblzma as it is not multiarch + - name: Install MSYS2 build prerequisites + if: runner.os == 'Windows' + uses: msys2/setup-msys2@v2 + with: + msystem: ucrt64 + update: true + install: >- + mingw-w64-${{ matrix.config.arch }}-base-devel + mingw-w64-${{ matrix.config.arch }}-cython + mingw-w64-${{ matrix.config.arch }}-python + mingw-w64-${{ matrix.config.arch }}-pip + mingw-w64-${{ matrix.config.arch }}-setuptools + mingw-w64-${{ matrix.config.arch }}-curl + mingw-w64-${{ matrix.config.arch }}-libdeflate + mingw-w64-${{ matrix.config.arch }}-bzip2 + mingw-w64-${{ matrix.config.arch }}-zlib + mingw-w64-${{ matrix.config.arch }}-python-pytest + mingw-w64-${{ matrix.config.arch }}-xz + mingw-w64-${{ matrix.config.arch }}-openssl + mingw-w64-${{ matrix.config.arch }}-libtre-git + mingw-w64-${{ matrix.config.arch }}-libiconv + - name: Build (directly from checkout) run: python setup.py build @@ -53,6 +93,9 @@ jobs: macOS) brew install -q samtools bcftools ;; + Windows) + pacman -S --noconfirm mingw-w64-${{ matrix.config.arch }}-samtools mingw-w64-${{ matrix.config.arch }}-bcftools + ;; esac - name: Run tests From 6050a0d81175310501eb2eece483af0e70a619f2 Mon Sep 17 00:00:00 2001 From: teepean Date: Wed, 17 Apr 2024 10:37:45 +0300 Subject: [PATCH 07/21] Update ci.yaml --- .github/workflows/ci.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1b754a61..d6e838c3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -29,8 +29,6 @@ jobs: python-version: 3.11 - os: windows python-version: 3.12 - os: [windows] - - {msystem: UCRT64, arch: x86_64} steps: - name: Checkout pysam From 8f5e7cd73a8227f15c0ee76c5ffadcc0410e5042 Mon Sep 17 00:00:00 2001 From: teepean Date: Wed, 6 Nov 2024 04:10:51 +0200 Subject: [PATCH 08/21] Small update --- .github/workflows/ci.yaml | 354 +++++++++++++++++++++----------------- setup.py | 20 ++- 2 files changed, 217 insertions(+), 157 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e73935b3..1f6c600e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,156 +1,198 @@ -name: CI - -on: [push, pull_request] - -jobs: - direct: - runs-on: ${{ matrix.os }}-latest - strategy: - matrix: - os: [ubuntu, macos] - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] - exclude: - # Run only the latest few 3.x versions on macOS - - os: macos - python-version: 3.7 - - os: macos - python-version: 3.8 - - os: macos - python-version: 3.9 - - steps: - - name: Checkout pysam - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install prerequisite Python libraries - run: pip install cython pytest pytest-pep8 setuptools - - - name: Install Linux build prerequisites - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -q --no-install-recommends --no-install-suggests libcurl4-openssl-dev - - - name: Update macOS build prerequisites - if: runner.os == 'macOS' - run: | - brew unlink xz || true # Remove brewed liblzma as it is not multiarch - - - name: Build (directly from checkout) - run: python setup.py build - - - name: Install test prerequisites - run: | - case $RUNNER_OS in - Linux) - sudo apt-get install -q --no-install-recommends --no-install-suggests samtools bcftools tabix - ;; - macOS) - brew install -q samtools bcftools - ;; - esac - - - name: Run tests - run: | - export PYTHONPATH=$(echo $GITHUB_WORKSPACE/build/lib.*) - export REF_PATH=':' - pytest - - - sdist: - runs-on: ${{ matrix.os }}-latest - strategy: - matrix: - os: [ubuntu, macos] - python-version: ['3.10'] - - steps: - - name: Checkout pysam - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install prerequisite Python libraries - run: pip install cython pytest pytest-pep8 - - - name: Install build prerequisites - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -q --no-install-recommends --no-install-suggests libcurl4-openssl-dev - - - name: Create source distribution - run: python setup.py sdist --owner=root --group=root - - - name: Build (via sdist tarball) - run: pip install --verbose --no-deps --no-binary=':all:' pysam-*.tar.gz - working-directory: dist - - - name: Install test prerequisites - run: | - case $RUNNER_OS in - Linux) - sudo apt-get install -q --no-install-recommends --no-install-suggests samtools bcftools tabix - ;; - macOS) - brew install -q samtools bcftools - ;; - esac - - - name: Run tests - run: REF_PATH=':' pytest - - - name: Upload sdist tarball - if: runner.os == 'Linux' - uses: actions/upload-artifact@v3 - with: - name: sdist - path: dist/pysam-*.tar.gz - retention-days: 14 - - - conda: - timeout-minutes: 20 - runs-on: ${{ matrix.os }}-latest - strategy: - matrix: - os: [ubuntu] - python-version: ['3.7'] - defaults: - run: - shell: bash -l {0} # needed for conda activation - env: - HTSLIB_CONFIGURE_OPTIONS: "--disable-libcurl" - - steps: - - name: Checkout pysam - uses: actions/checkout@v4 - - - uses: conda-incubator/setup-miniconda@v2 - with: - channel-priority: strict - activate-environment: testenv - auto-activate-base: false - use-only-tar-bz2: true - - - name: Set up Conda and Python ${{ matrix.python-version }} - run: | - conda config --add channels bioconda --add channels conda-forge - conda install python=${{ matrix.python-version }} cython - - - name: Build (directly from checkout) - run: python setup.py install - - - name: Install test prerequisites via Conda - run: conda install "samtools>=1.11" "bcftools>=1.11" "htslib>=1.11" pytest - - - name: Run tests - run: REF_PATH=':' pytest +name: CI + +on: [push, pull_request] + +jobs: + direct: + runs-on: ${{ matrix.os }}-latest + strategy: + matrix: + os: [ubuntu, macos, windows] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] + config: + - { arch: ucrt64 } # Added configuration for MSYS2 architecture + exclude: + # Run only the latest few 3.x versions on macOS + - os: macos + python-version: '3.7' + - os: macos + python-version: '3.8' + - os: macos + python-version: '3.9' + # Windows exclusions + - os: windows + python-version: '3.7' + - os: windows + python-version: '3.8' + - os: windows + python-version: '3.9' + - os: windows + python-version: '3.10' + - os: windows + python-version: '3.11' + - os: windows + python-version: '3.12' + + steps: + - name: Checkout pysam + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + if: runner.os != 'Windows' + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install prerequisite Python libraries + if: runner.os != 'Windows' + run: pip install cython pytest pytest-pep8 setuptools + + - name: Install Linux build prerequisites + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -q --no-install-recommends --no-install-suggests libcurl4-openssl-dev + + - name: Update macOS build prerequisites + if: runner.os == 'macOS' + run: | + brew unlink xz || true + + - name: Install MSYS2 build prerequisites + if: runner.os == 'Windows' + uses: msys2/setup-msys2@v2 + with: + msystem: ucrt64 + update: true + install: >- + mingw-w64-ucrt64-base-devel + mingw-w64-ucrt64-cython + mingw-w64-ucrt64-python + mingw-w64-ucrt64-pip + mingw-w64-ucrt64-setuptools + mingw-w64-ucrt64-curl + mingw-w64-ucrt64-libdeflate + mingw-w64-ucrt64-bzip2 + mingw-w64-ucrt64-zlib + mingw-w64-ucrt64-python-pytest + mingw-w64-ucrt64-xz + mingw-w64-ucrt64-openssl + mingw-w64-ucrt64-libtre-git + mingw-w64-ucrt64-libiconv + + - name: Build (directly from checkout) + run: python setup.py build + + - name: Install test prerequisites + run: | + case $RUNNER_OS in + Linux) + sudo apt-get install -q --no-install-recommends --no-install-suggests samtools bcftools tabix + ;; + macOS) + brew install -q samtools bcftools + ;; + Windows) + pacman -S --noconfirm mingw-w64-ucrt64-samtools mingw-w64-ucrt64-bcftools + ;; + esac + + - name: Run tests + run: | + export PYTHONPATH=$(echo $GITHUB_WORKSPACE/build/lib.*) + export REF_PATH=':' + pytest + + + sdist: + runs-on: ${{ matrix.os }}-latest + strategy: + matrix: + os: [ubuntu, macos] + python-version: ['3.10'] + + steps: + - name: Checkout pysam + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install prerequisite Python libraries + run: pip install cython pytest pytest-pep8 + + - name: Install build prerequisites + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -q --no-install-recommends --no-install-suggests libcurl4-openssl-dev + + - name: Create source distribution + run: python setup.py sdist --owner=root --group=root + + - name: Build (via sdist tarball) + run: pip install --verbose --no-deps --no-binary=':all:' pysam-*.tar.gz + working-directory: dist + + - name: Install test prerequisites + run: | + case $RUNNER_OS in + Linux) + sudo apt-get install -q --no-install-recommends --no-install-suggests samtools bcftools tabix + ;; + macOS) + brew install -q samtools bcftools + ;; + esac + + - name: Run tests + run: REF_PATH=':' pytest + + - name: Upload sdist tarball + if: runner.os == 'Linux' + uses: actions/upload-artifact@v3 + with: + name: sdist + path: dist/pysam-*.tar.gz + retention-days: 14 + + + conda: + timeout-minutes: 20 + runs-on: ${{ matrix.os }}-latest + strategy: + matrix: + os: [ubuntu] + python-version: ['3.7'] + defaults: + run: + shell: bash -l {0} # needed for conda activation + env: + HTSLIB_CONFIGURE_OPTIONS: "--disable-libcurl" + + steps: + - name: Checkout pysam + uses: actions/checkout@v4 + + - uses: conda-incubator/setup-miniconda@v2 + with: + channel-priority: strict + activate-environment: testenv + auto-activate-base: false + use-only-tar-bz2: true + + - name: Set up Conda and Python ${{ matrix.python-version }} + run: | + conda config --add channels bioconda --add channels conda-forge + conda install python=${{ matrix.python-version }} cython + + - name: Build (directly from checkout) + run: python setup.py install + + - name: Install test prerequisites via Conda + run: conda install "samtools>=1.11" "bcftools>=1.11" "htslib>=1.11" pytest + + - name: Run tests + run: REF_PATH=':' pytest diff --git a/setup.py b/setup.py index ccb09e84..ac12cf84 100644 --- a/setup.py +++ b/setup.py @@ -506,10 +506,28 @@ def run(self): # Link with the object files rather than the final htslib/libhts.a, to ensure that # all object files are pulled into the link, even those not used by htslib itself. +# Initialize htslib_make_options for Windows with suitable default values or as empty +if sys.platform == "win32": + htslib_make_options = { + # 'LIBHTS_OBJS' might not be relevant for Windows if linking against a precompiled lib + # Define it or leave it out based on your specific setup + 'LIBHTS_OBJS': '' + } + # Adjust paths to precompiled HTSlib objects or libraries as necessary + htslib_objects = ['htslib-msvc/libhts.lib'] + separate_htslib_objects = [] +else: + # Unix-like platform logic... + with changedir("htslib"): + htslib_make_options = run_make_print_config() + htslib_objects = [os.path.join("htslib", x) - for x in htslib_make_options["LIBHTS_OBJS"].split(" ")] + for x in htslib_make_options.get("LIBHTS_OBJS", "").split(" ")] separate_htslib_objects = [] +# Example continues... + + htslib_library_dirs = ["."] # when using setup.py develop? htslib_include_dirs = ['htslib'] else: From 579dab88e77f1ca300a7ca81a5fd621c7c2b43fb Mon Sep 17 00:00:00 2001 From: teepean Date: Wed, 6 Nov 2024 04:19:10 +0200 Subject: [PATCH 09/21] Fix the previous update --- setup.py | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/setup.py b/setup.py index ac12cf84..ccb09e84 100644 --- a/setup.py +++ b/setup.py @@ -506,28 +506,10 @@ def run(self): # Link with the object files rather than the final htslib/libhts.a, to ensure that # all object files are pulled into the link, even those not used by htslib itself. -# Initialize htslib_make_options for Windows with suitable default values or as empty -if sys.platform == "win32": - htslib_make_options = { - # 'LIBHTS_OBJS' might not be relevant for Windows if linking against a precompiled lib - # Define it or leave it out based on your specific setup - 'LIBHTS_OBJS': '' - } - # Adjust paths to precompiled HTSlib objects or libraries as necessary - htslib_objects = ['htslib-msvc/libhts.lib'] - separate_htslib_objects = [] -else: - # Unix-like platform logic... - with changedir("htslib"): - htslib_make_options = run_make_print_config() - htslib_objects = [os.path.join("htslib", x) - for x in htslib_make_options.get("LIBHTS_OBJS", "").split(" ")] + for x in htslib_make_options["LIBHTS_OBJS"].split(" ")] separate_htslib_objects = [] -# Example continues... - - htslib_library_dirs = ["."] # when using setup.py develop? htslib_include_dirs = ['htslib'] else: From 271587daabbe535059ddc68cda76cf91c1560b3c Mon Sep 17 00:00:00 2001 From: teepean Date: Wed, 6 Nov 2024 04:23:04 +0200 Subject: [PATCH 10/21] Update ci.yaml --- .github/workflows/ci.yaml | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1f6c600e..9ddc1bf4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,8 +9,10 @@ jobs: matrix: os: [ubuntu, macos, windows] python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] - config: - - { arch: ucrt64 } # Added configuration for MSYS2 architecture + include: + # Add specific configuration for Windows + - os: windows + msystem: ucrt64 exclude: # Run only the latest few 3.x versions on macOS - os: macos @@ -19,19 +21,8 @@ jobs: python-version: '3.8' - os: macos python-version: '3.9' - # Windows exclusions - - os: windows - python-version: '3.7' - - os: windows - python-version: '3.8' - - os: windows - python-version: '3.9' - - os: windows - python-version: '3.10' - - os: windows - python-version: '3.11' - - os: windows - python-version: '3.12' + # We don't need actions/setup-python for Windows as MSYS2 provides Python + # but we shouldn't exclude Windows entirely steps: - name: Checkout pysam @@ -81,9 +72,18 @@ jobs: mingw-w64-ucrt64-libiconv - name: Build (directly from checkout) - run: python setup.py build + shell: msys2 {0} + if: runner.os == 'Windows' + run: | + python setup.py build + + - name: Build (directly from checkout) + if: runner.os != 'Windows' + run: | + python setup.py build - name: Install test prerequisites + shell: bash run: | case $RUNNER_OS in Linux) @@ -98,6 +98,7 @@ jobs: esac - name: Run tests + shell: ${{ runner.os == 'Windows' && 'msys2 {0}' || 'bash' }} run: | export PYTHONPATH=$(echo $GITHUB_WORKSPACE/build/lib.*) export REF_PATH=':' From 98abafaccbb8f1a444ffdafb65313832c02a7e44 Mon Sep 17 00:00:00 2001 From: teepean Date: Wed, 6 Nov 2024 04:24:18 +0200 Subject: [PATCH 11/21] Update ci.yaml --- .github/workflows/ci.yaml | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9ddc1bf4..0bf73ad2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,8 +21,6 @@ jobs: python-version: '3.8' - os: macos python-version: '3.9' - # We don't need actions/setup-python for Windows as MSYS2 provides Python - # but we shouldn't exclude Windows entirely steps: - name: Checkout pysam @@ -71,13 +69,13 @@ jobs: mingw-w64-ucrt64-libtre-git mingw-w64-ucrt64-libiconv - - name: Build (directly from checkout) - shell: msys2 {0} + - name: Build (Windows) if: runner.os == 'Windows' + shell: msys2 {0} run: | python setup.py build - - name: Build (directly from checkout) + - name: Build (Unix) if: runner.os != 'Windows' run: | python setup.py build @@ -97,8 +95,17 @@ jobs: ;; esac - - name: Run tests - shell: ${{ runner.os == 'Windows' && 'msys2 {0}' || 'bash' }} + - name: Run tests (Windows) + if: runner.os == 'Windows' + shell: msys2 {0} + run: | + export PYTHONPATH=$(echo $GITHUB_WORKSPACE/build/lib.*) + export REF_PATH=':' + pytest + + - name: Run tests (Unix) + if: runner.os != 'Windows' + shell: bash run: | export PYTHONPATH=$(echo $GITHUB_WORKSPACE/build/lib.*) export REF_PATH=':' From d02977dd445640902f7bc3ffeaf6bf2b6d4a196b Mon Sep 17 00:00:00 2001 From: teepean Date: Wed, 6 Nov 2024 04:34:33 +0200 Subject: [PATCH 12/21] Update ci.yaml --- .github/workflows/ci.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0bf73ad2..08c1cba9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,6 +21,17 @@ jobs: python-version: '3.8' - os: macos python-version: '3.9' + # Exclude all Windows Python versions except 3.11 + - os: windows + python-version: '3.7' + - os: windows + python-version: '3.8' + - os: windows + python-version: '3.9' + - os: windows + python-version: '3.10' + - os: windows + python-version: '3.12' steps: - name: Checkout pysam From d5f3d14789c5b78ca832289bfd62d6e8be895f24 Mon Sep 17 00:00:00 2001 From: teepean Date: Wed, 6 Nov 2024 04:37:55 +0200 Subject: [PATCH 13/21] Update ci.yaml --- .github/workflows/ci.yaml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 08c1cba9..00854e4b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -65,20 +65,20 @@ jobs: msystem: ucrt64 update: true install: >- - mingw-w64-ucrt64-base-devel - mingw-w64-ucrt64-cython - mingw-w64-ucrt64-python - mingw-w64-ucrt64-pip - mingw-w64-ucrt64-setuptools - mingw-w64-ucrt64-curl - mingw-w64-ucrt64-libdeflate - mingw-w64-ucrt64-bzip2 - mingw-w64-ucrt64-zlib - mingw-w64-ucrt64-python-pytest - mingw-w64-ucrt64-xz - mingw-w64-ucrt64-openssl - mingw-w64-ucrt64-libtre-git - mingw-w64-ucrt64-libiconv + mingw-w64-ucrt-x86_64-base-devel + mingw-w64-ucrt-x86_64-cython + mingw-w64-ucrt-x86_64-python + mingw-w64-ucrt-x86_64-pip + mingw-w64-ucrt-x86_64-setuptools + mingw-w64-ucrt-x86_64-curl + mingw-w64-ucrt-x86_64-libdeflate + mingw-w64-ucrt-x86_64-bzip2 + mingw-w64-ucrt-x86_64-zlib + mingw-w64-ucrt-x86_64-python-pytest + mingw-w64-ucrt-x86_64-xz + mingw-w64-ucrt-x86_64-openssl + mingw-w64-ucrt-x86_64-libtre-git + mingw-w64-ucrt-x86_64-libiconv - name: Build (Windows) if: runner.os == 'Windows' @@ -102,7 +102,7 @@ jobs: brew install -q samtools bcftools ;; Windows) - pacman -S --noconfirm mingw-w64-ucrt64-samtools mingw-w64-ucrt64-bcftools + pacman -S --noconfirm mingw-w64-ucrt-x86_64-samtools mingw-w64-ucrt-x86_64-bcftools ;; esac From f4b99b0961c54fff4c9d29b12b4a7cc268706ff7 Mon Sep 17 00:00:00 2001 From: teepean Date: Wed, 6 Nov 2024 04:40:43 +0200 Subject: [PATCH 14/21] Update ci.yaml --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 00854e4b..341bae0b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -65,7 +65,7 @@ jobs: msystem: ucrt64 update: true install: >- - mingw-w64-ucrt-x86_64-base-devel + base-devel mingw-w64-ucrt-x86_64-cython mingw-w64-ucrt-x86_64-python mingw-w64-ucrt-x86_64-pip From f137e38f61b53fca87ea6ec9f240961803d78e4a Mon Sep 17 00:00:00 2001 From: teepean Date: Wed, 6 Nov 2024 04:44:08 +0200 Subject: [PATCH 15/21] Update ci.yaml --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 341bae0b..d0604cc0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -68,8 +68,8 @@ jobs: base-devel mingw-w64-ucrt-x86_64-cython mingw-w64-ucrt-x86_64-python - mingw-w64-ucrt-x86_64-pip - mingw-w64-ucrt-x86_64-setuptools + mingw-w64-ucrt-x86_64-python-pip + mingw-w64-ucrt-x86_64-python-setuptools mingw-w64-ucrt-x86_64-curl mingw-w64-ucrt-x86_64-libdeflate mingw-w64-ucrt-x86_64-bzip2 From 686555bd0214810c159f4b6b01e94d6ac5fd4174 Mon Sep 17 00:00:00 2001 From: teepean Date: Wed, 6 Nov 2024 04:47:50 +0200 Subject: [PATCH 16/21] Update ci.yaml --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d0604cc0..06e64704 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -66,6 +66,7 @@ jobs: update: true install: >- base-devel + mingw-w64-ucrt-x86_64-toolchain mingw-w64-ucrt-x86_64-cython mingw-w64-ucrt-x86_64-python mingw-w64-ucrt-x86_64-python-pip From a50c0b52600f4f66fd3806459a6561bb244a2d4f Mon Sep 17 00:00:00 2001 From: teepean Date: Wed, 6 Nov 2024 04:53:41 +0200 Subject: [PATCH 17/21] Update ci.yaml --- .github/workflows/ci.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 06e64704..3eaad218 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,6 +21,12 @@ jobs: python-version: '3.8' - os: macos python-version: '3.9' + - os: macos + python-version: '3.10' + - os: macos + python-version: '3.11' + - os: macos + python-version: '3.12' # Exclude all Windows Python versions except 3.11 - os: windows python-version: '3.7' From f111cc6299bc295ed3372c429d3b3d37d6d5564d Mon Sep 17 00:00:00 2001 From: teepean Date: Wed, 6 Nov 2024 05:09:23 +0200 Subject: [PATCH 18/21] Update ci.yaml --- .github/workflows/ci.yaml | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3eaad218..2b76eafa 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -98,20 +98,21 @@ jobs: run: | python setup.py build - - name: Install test prerequisites + - name: Install test prerequisites (Windows) + if: runner.os == 'Windows' + shell: msys2 {0} + run: | + pacman -S --noconfirm mingw-w64-ucrt-x86_64-samtools mingw-w64-ucrt-x86_64-bcftools + + - name: Install test prerequisites (Unix) + if: runner.os != 'Windows' shell: bash run: | - case $RUNNER_OS in - Linux) - sudo apt-get install -q --no-install-recommends --no-install-suggests samtools bcftools tabix - ;; - macOS) - brew install -q samtools bcftools - ;; - Windows) - pacman -S --noconfirm mingw-w64-ucrt-x86_64-samtools mingw-w64-ucrt-x86_64-bcftools - ;; - esac + if [ "$RUNNER_OS" == "Linux" ]; then + sudo apt-get install -q --no-install-recommends --no-install-suggests samtools bcftools tabix + elif [ "$RUNNER_OS" == "macOS" ]; then + brew install -q samtools bcftools + fi - name: Run tests (Windows) if: runner.os == 'Windows' @@ -134,7 +135,7 @@ jobs: runs-on: ${{ matrix.os }}-latest strategy: matrix: - os: [ubuntu, macos] + os: [ubuntu] python-version: ['3.10'] steps: From 7d99bae33589dbfb87ed3cfb6fba1622d8e8457d Mon Sep 17 00:00:00 2001 From: teepean Date: Tue, 18 Feb 2025 19:15:48 +0200 Subject: [PATCH 19/21] Update ci.yaml --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2b76eafa..9ba91851 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -84,7 +84,7 @@ jobs: mingw-w64-ucrt-x86_64-python-pytest mingw-w64-ucrt-x86_64-xz mingw-w64-ucrt-x86_64-openssl - mingw-w64-ucrt-x86_64-libtre-git + mingw-w64-ucrt-x86_64-libtre mingw-w64-ucrt-x86_64-libiconv - name: Build (Windows) From 69a40527039a8f4128581679b6878a9376564a05 Mon Sep 17 00:00:00 2001 From: teepean Date: Tue, 18 Feb 2025 20:10:04 +0200 Subject: [PATCH 20/21] Update ci.yaml --- .github/workflows/ci.yaml | 37 +++---------------------------------- 1 file changed, 3 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9ba91851..7566eb3e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -14,6 +14,8 @@ jobs: - os: windows msystem: ucrt64 exclude: + - os: ubuntu + python-version: '3.7' # Run only the latest few 3.x versions on macOS - os: macos python-version: '3.7' @@ -186,40 +188,7 @@ jobs: retention-days: 14 - conda: - timeout-minutes: 20 - runs-on: ${{ matrix.os }}-latest - strategy: - matrix: - os: [ubuntu] - python-version: ['3.7'] - defaults: - run: - shell: bash -l {0} # needed for conda activation - env: - HTSLIB_CONFIGURE_OPTIONS: "--disable-libcurl" - - steps: - - name: Checkout pysam - uses: actions/checkout@v4 - - - uses: conda-incubator/setup-miniconda@v2 - with: - channel-priority: strict - activate-environment: testenv - auto-activate-base: false - use-only-tar-bz2: true - - - name: Set up Conda and Python ${{ matrix.python-version }} - run: | - conda config --add channels bioconda --add channels conda-forge - conda install python=${{ matrix.python-version }} cython - - - name: Build (directly from checkout) - run: python setup.py install - - - name: Install test prerequisites via Conda - run: conda install "samtools>=1.11" "bcftools>=1.11" "htslib>=1.11" pytest + - name: Run tests run: REF_PATH=':' pytest From 29f4a1266dccce78202844b54276e149cf5a13dc Mon Sep 17 00:00:00 2001 From: teepean Date: Tue, 18 Feb 2025 20:17:16 +0200 Subject: [PATCH 21/21] Update ci.yaml --- .github/workflows/ci.yaml | 64 +++++++-------------------------------- 1 file changed, 11 insertions(+), 53 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7566eb3e..8f2475e5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,6 +16,16 @@ jobs: exclude: - os: ubuntu python-version: '3.7' + - os: ubuntu + python-version: '3.8' + - os: ubuntu + python-version: '3.9' + - os: ubuntu + python-version: '3.10' + - os: ubuntu + python-version: '3.11' + - os: ubuntu + python-version: '3.12' # Run only the latest few 3.x versions on macOS - os: macos python-version: '3.7' @@ -133,59 +143,7 @@ jobs: pytest - sdist: - runs-on: ${{ matrix.os }}-latest - strategy: - matrix: - os: [ubuntu] - python-version: ['3.10'] - - steps: - - name: Checkout pysam - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install prerequisite Python libraries - run: pip install cython pytest pytest-pep8 - - - name: Install build prerequisites - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -q --no-install-recommends --no-install-suggests libcurl4-openssl-dev - - - name: Create source distribution - run: python setup.py sdist --owner=root --group=root - - - name: Build (via sdist tarball) - run: pip install --verbose --no-deps --no-binary=':all:' pysam-*.tar.gz - working-directory: dist - - - name: Install test prerequisites - run: | - case $RUNNER_OS in - Linux) - sudo apt-get install -q --no-install-recommends --no-install-suggests samtools bcftools tabix - ;; - macOS) - brew install -q samtools bcftools - ;; - esac - - - name: Run tests - run: REF_PATH=':' pytest - - - name: Upload sdist tarball - if: runner.os == 'Linux' - uses: actions/upload-artifact@v3 - with: - name: sdist - path: dist/pysam-*.tar.gz - retention-days: 14 +