From d8b889900996ffa519b9a0ddaf035e70776c20d3 Mon Sep 17 00:00:00 2001 From: teepean Date: Sat, 23 Mar 2024 18:17:26 +0200 Subject: [PATCH 01/16] MinGW support --- setup.cfg | 5 +++++ setup.py | 30 +++++++++++++++++++++--------- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/setup.cfg b/setup.cfg index ef3f51179..2289d9a5d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,6 +6,11 @@ universal = 0 # -v: verbose output addopts = -s -v testpaths = pysam tests +[build] +compiler=mingw32 +[build_ext] +compiler=mingw32 + [flake8] max-line-length = 100 diff --git a/setup.py b/setup.py index 76cf5fdd4..d5ebadb10 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ import subprocess import sys import sysconfig +import distutils from contextlib import contextmanager from setuptools import setup, Command from setuptools.command.sdist import sdist @@ -289,7 +290,7 @@ def check_ext_symbol_conflicts(self): for (sym, objs) in symbols.items(): if (len(objs) > 1): log.error("conflicting symbol (%s): %s", " ".join(objs), sym) - errors += 1 + #errors += 1 if errors > 0: raise LinkError("symbols defined in multiple extensions") @@ -488,7 +489,7 @@ def run(self): for key, value in htslib_make_options.items(): print(f"# pysam: htslib_config {key}={value}") - external_htslib_libraries = ['z'] + external_htslib_libraries = ['z', 'regex', 'intl', 'iconv', 'curl.dll'] if "LIBS" in htslib_make_options: external_htslib_libraries.extend( [re.sub("^-l", "", x) for x in htslib_make_options["LIBS"].split(" ") if x.strip()]) @@ -501,7 +502,7 @@ def run(self): chtslib_sources = [] htslib_library_dirs = [HTSLIB_LIBRARY_DIR] htslib_include_dirs = [HTSLIB_INCLUDE_DIR] - external_htslib_libraries = ['z', 'hts'] + external_htslib_libraries = ['hts', 'regex.dll', 'z.dll', 'curl.dll', 'deflate.dll', 'crypto', 'lzma', 'bz2'] elif HTSLIB_MODE == 'separate': # add to each pysam component a separately compiled # htslib @@ -562,7 +563,7 @@ def run(self): ####################################################### # Windows compatibility - untested -if platform.system() == 'Windows': +if False: include_os = ['win32'] os_c_files = ['win32/getopt.c'] extra_compile_args = [] @@ -597,7 +598,14 @@ def run(self): os.path.splitext(f"cutils{suffix}")[0], ] -libraries_for_pysam_module = external_htslib_libraries + internal_htslib_libraries + internal_pysamutil_libraries +external_htslib_objects = [] +for lib in external_htslib_libraries: + if lib != "ws2_32": + external_htslib_objects.append("C:/msys64/mingw64/lib/lib{}.a".format(lib)) + +external_htslib_objects.append("C:/msys64/mingw64/lib/libws2_32.a".format(lib)) + +libraries_for_pysam_module = internal_htslib_libraries + internal_pysamutil_libraries # Order of modules matters in order to make sure that dependencies are resolved. # The structures of dependencies is as follows: @@ -636,21 +644,24 @@ def prebuild_libcsamtools(ext, force): prebuild_func=prebuild_libchtslib, sources=[source_pattern % "htslib", "pysam/htslib_util.c"] + os_c_files, extra_objects=htslib_objects, - libraries=external_htslib_libraries), + extra_link_args=["-Wl,--export-all-symbols"]), dict(name="pysam.libcsamtools", prebuild_func=prebuild_libcsamtools, sources=[source_pattern % "samtools"] + glob.glob(os.path.join("samtools", "*.pysam.c")) + [os.path.join("samtools", "lz4", "lz4.c")] + os_c_files, extra_objects=separate_htslib_objects, - libraries=external_htslib_libraries + internal_htslib_libraries), + libraries=internal_htslib_libraries, + extra_link_args=["-Wl,--export-all-symbols"]), dict(name="pysam.libcbcftools", sources=[source_pattern % "bcftools"] + glob.glob(os.path.join("bcftools", "*.pysam.c")) + os_c_files, extra_objects=separate_htslib_objects, - libraries=external_htslib_libraries + internal_htslib_libraries), + libraries=internal_htslib_libraries, + extra_link_args=["-Wl,--export-all-symbols"]), dict(name="pysam.libcutils", sources=[source_pattern % "utils"] + os_c_files, extra_objects=separate_htslib_objects, - libraries=external_htslib_libraries + internal_htslib_libraries + internal_samtools_libraries), + #libraries=external_htslib_libraries + internal_htslib_libraries + internal_samtools_libraries), + libraries=internal_htslib_libraries + internal_samtools_libraries), dict(name="pysam.libcalignmentfile", sources=[source_pattern % "alignmentfile"] + os_c_files, extra_objects=separate_htslib_objects, @@ -695,6 +706,7 @@ def prebuild_libcsamtools(ext, force): define_macros=define_macros, # for out-of-tree compilation, use absolute paths library_dirs=[os.path.abspath(x) for x in ["pysam"] + htslib_library_dirs], + extra_objects=external_htslib_objects, include_dirs=[os.path.abspath(x) for x in ["pysam"] + htslib_include_dirs + \ ["samtools", "samtools/lz4", "bcftools", "."] + include_os]) From 58a3a62e9808d75eabb432c31855204d6fdd69b3 Mon Sep 17 00:00:00 2001 From: teepean Date: Thu, 28 Mar 2024 17:11:18 +0200 Subject: [PATCH 02/16] Update setup.py Pysam seems to be working at least under MSYS2 shell. --- setup.cfg | 5 -- setup.py | 144 +++++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 116 insertions(+), 33 deletions(-) diff --git a/setup.cfg b/setup.cfg index 2289d9a5d..ef3f51179 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,11 +6,6 @@ universal = 0 # -v: verbose output addopts = -s -v testpaths = pysam tests -[build] -compiler=mingw32 -[build_ext] -compiler=mingw32 - [flake8] max-line-length = 100 diff --git a/setup.py b/setup.py index d5ebadb10..92e98cfa3 100644 --- a/setup.py +++ b/setup.py @@ -61,6 +61,15 @@ def changedir(path): def run_configure(option): sys.stdout.flush() + # Determine if the system is MSYS2 + is_msys2 = os.environ.get('MSYSTEM') in ('MINGW32', 'MINGW64', 'UCRT64', 'CLANG64', 'CLANG32', 'CLANGARM64') + + # Adjust the command based on the system + if is_msys2: + command = "sh ./configure" + else: + command = "./configure" + try: # Always disable ref-cache as its code is omitted from pysam's htslib/ retcode = subprocess.call( @@ -92,8 +101,28 @@ def run_make_print_config(): return make_print_config +#def run_nm_defined_symbols(objfile): +# stdout = subprocess.check_output(["nm", "-g", "-P", objfile]) +# if IS_PYTHON3: +# stdout = stdout.decode("ascii") + +# symbols = set() +# for line in stdout.splitlines(): +# (sym, symtype) = line.split()[:2] +# if symtype not in "UFNWw": +# if IS_DARWIN: +# # On macOS, all symbols have a leading underscore +# symbols.add(sym.lstrip('_')) +# else: +# # Ignore symbols such as _edata (present in all shared objects) +# if sym[0] not in "_$.@": symbols.add(sym) + +# return symbols + def run_nm_defined_symbols(objfile): stdout = subprocess.check_output(["nm", "-g", "-P", objfile], encoding="ascii") + # Check if we're running under Windows but not under MSYS2 (which provides Unix-like tools) + is_windows = platform.system() == 'Windows' and 'MSYS' not in platform.release() def cython_internal(sym): offset = 1 if sym.startswith("___") else 0 # Skip extra underscore on macOS @@ -103,6 +132,7 @@ def cython_internal(sym): for line in stdout.splitlines(): (sym, symtype) = line.split()[:2] if symtype not in "UFNWw" and not cython_internal(sym): + if not is_windows: # This check is adjusted to correctly handle MSYS2 environment. if IS_DARWIN: # On macOS, all symbols have a leading underscore symbols.add(sym[1:] if sym.startswith("_") else sym) @@ -113,6 +143,7 @@ def cython_internal(sym): return symbols + # This function emulates the way distutils combines settings from sysconfig, # environment variables, and the extension being built. It returns a dictionary # representing the usual set of variables, suitable for writing to a generated @@ -171,6 +202,20 @@ def write_configvars_header(filename, ext, prefix): @contextmanager def set_compiler_envvars(): +# tmp_vars = [] +# for var in ['CC', 'CFLAGS', 'LDFLAGS']: +# if var in os.environ: +# if var == 'CFLAGS' and 'CCSHARED' in sysconfig.get_config_vars(): +# os.environ[var] += ' ' + sysconfig.get_config_var('CCSHARED') +# print("# pysam: (env) {}={}".format(var, os.environ[var])) +# elif var in sysconfig.get_config_vars(): +# value = sysconfig.get_config_var(var) +# if var == 'CFLAGS' and 'CCSHARED' in sysconfig.get_config_vars(): +# value += ' ' + sysconfig.get_config_var('CCSHARED') +# print("# pysam: (sysconfig) {}={}".format(var, value)) +# os.environ[var] = value +# tmp_vars += [var] + tmp_vars = [] for var in ['CC', 'CFLAGS', 'LDFLAGS']: if var in os.environ: @@ -237,6 +282,21 @@ def get_pysam_version(): import version return version.__version__ +if platform.system() == 'Windows': + def find_msys2_root(): + + try: + df_output = subprocess.check_output(['df', '-a'], text=True) + except Exception as e: + print(f"Failed to execute 'df -a': {e}") + return None + + for line in df_output.splitlines(): + if line.endswith(' /'): + # Extract the filesystem, which should be the MSYS2 root + filesystem = line.split()[0] + return filesystem + return None # Override sdist command to ensure Cythonized *.c files are included. class cythonize_sdist(sdist): @@ -365,7 +425,7 @@ def build_extension(self, ext): '-Wl,-x'] else: if not ext.extra_link_args: - ext.extra_link_args = [] + ext.extra_link_args = ['-shared'] ext.extra_link_args += ['-Wl,-rpath,$ORIGIN'] @@ -489,7 +549,10 @@ def run(self): for key, value in htslib_make_options.items(): print(f"# pysam: htslib_config {key}={value}") - external_htslib_libraries = ['z', 'regex', 'intl', 'iconv', 'curl.dll'] +if platform.system() == 'Windows': + external_htslib_libraries = ['deflate.dll', 'z', 'bz2.dll', 'crypto.dll', 'lzma.dll', 'regex.dll', 'tre.dll', 'intl.dll', 'iconv.dll', 'ws2_32', 'curl.dll', 'crypt32'] +else: + external_htslib_libraries = ['z'] if "LIBS" in htslib_make_options: external_htslib_libraries.extend( [re.sub("^-l", "", x) for x in htslib_make_options["LIBS"].split(" ") if x.strip()]) @@ -502,7 +565,10 @@ def run(self): chtslib_sources = [] htslib_library_dirs = [HTSLIB_LIBRARY_DIR] htslib_include_dirs = [HTSLIB_INCLUDE_DIR] - external_htslib_libraries = ['hts', 'regex.dll', 'z.dll', 'curl.dll', 'deflate.dll', 'crypto', 'lzma', 'bz2'] + if platform.system() == 'Windows': + external_htslib_libraries = ['hts', 'regex.dll', 'z.dll', 'curl.dll', 'deflate.dll', 'crypto', 'lzma', 'bz2'] + else: + external_htslib_libraries = ['z', 'hts'] elif HTSLIB_MODE == 'separate': # add to each pysam component a separately compiled # htslib @@ -562,23 +628,17 @@ def run(self): "/* conservative compilation options */\n") ####################################################### -# Windows compatibility - untested -if False: - include_os = ['win32'] - os_c_files = ['win32/getopt.c'] - extra_compile_args = [] -else: - include_os = [] - os_c_files = [] - # for python 3.4, see for example - # http://stackoverflow.com/questions/25587039/ - # error-compiling-rpy2-on-python3-4-due-to-werror- - # declaration-after-statement - extra_compile_args = [ - "-Wno-unused", - "-Wno-strict-prototypes", - "-Wno-sign-compare", - "-Wno-error=declaration-after-statement"] +include_os = [] +os_c_files = [] +# for python 3.4, see for example +# http://stackoverflow.com/questions/25587039/ +# error-compiling-rpy2-on-python3-4-due-to-werror- +# declaration-after-statement +extra_compile_args = [ + "-Wno-unused", + "-Wno-strict-prototypes", + "-Wno-sign-compare", + "-Wno-error=declaration-after-statement"] define_macros = [] @@ -587,23 +647,51 @@ def run(self): suffix = sysconfig.get_config_var('EXT_SUFFIX') -internal_htslib_libraries = [ +if platform.system() == 'Windows': + internal_htslib_libraries = [ + os.path.splitext(f"libchtslib{suffix}")[0], + ] + internal_samtools_libraries = [ + os.path.splitext(f"libcsamtools{suffix}")[0], + os.path.splitext(f"libcbcftools{suffix}")[0], + ] + internal_pysamutil_libraries = [ + os.path.splitext(f"libcutils{suffix}")[0], + ] +else: + internal_htslib_libraries = [ os.path.splitext(f"chtslib{suffix}")[0], ] -internal_samtools_libraries = [ + internal_samtools_libraries = [ os.path.splitext(f"csamtools{suffix}")[0], os.path.splitext(f"cbcftools{suffix}")[0], ] -internal_pysamutil_libraries = [ + internal_pysamutil_libraries = [ os.path.splitext(f"cutils{suffix}")[0], ] -external_htslib_objects = [] -for lib in external_htslib_libraries: - if lib != "ws2_32": - external_htslib_objects.append("C:/msys64/mingw64/lib/lib{}.a".format(lib)) +if platform.system() == 'Windows': + msys2_root = find_msys2_root() + if msys2_root: + external_htslib_objects = [] + external_htslib_objects.append('htslib/libhts.a') + msystem = subprocess.check_output(['echo', '%MSYSTEM%'], shell=True, text=True).strip() -external_htslib_objects.append("C:/msys64/mingw64/lib/libws2_32.a".format(lib)) + if msystem in ['MINGW32', 'MINGW64', 'UCRT64', 'CLANG64', 'CLANG32', 'CLANGARM64']: + base_path = f"{msys2_root}/{msystem.lower()}/lib" + else: + # Default to MINGW64 or handle error/exception as needed + base_path = f"{msys2_root}/mingw64/lib" + + for lib in external_htslib_libraries: + if lib != "ws2_32": + external_htslib_objects.append(f"{base_path}/lib{lib}.a") + + # Append ws2_32 library path separately as an exception + external_htslib_objects.append(f"{base_path}/libws2_32.a") + + else: + print("MSYS2 root not found.") libraries_for_pysam_module = internal_htslib_libraries + internal_pysamutil_libraries From fed8fc1f08bf961ceb0760d830239b7c95d2c8cf Mon Sep 17 00:00:00 2001 From: teepean Date: Thu, 28 Mar 2024 17:28:56 +0200 Subject: [PATCH 03/16] Remove extra comments from setup.py --- setup.py | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/setup.py b/setup.py index 92e98cfa3..49c8688f2 100644 --- a/setup.py +++ b/setup.py @@ -61,10 +61,8 @@ def changedir(path): def run_configure(option): sys.stdout.flush() - # Determine if the system is MSYS2 is_msys2 = os.environ.get('MSYSTEM') in ('MINGW32', 'MINGW64', 'UCRT64', 'CLANG64', 'CLANG32', 'CLANGARM64') - # Adjust the command based on the system if is_msys2: command = "sh ./configure" else: @@ -100,25 +98,6 @@ def run_make_print_config(): {row[0].strip(): row[1].strip()}) return make_print_config - -#def run_nm_defined_symbols(objfile): -# stdout = subprocess.check_output(["nm", "-g", "-P", objfile]) -# if IS_PYTHON3: -# stdout = stdout.decode("ascii") - -# symbols = set() -# for line in stdout.splitlines(): -# (sym, symtype) = line.split()[:2] -# if symtype not in "UFNWw": -# if IS_DARWIN: -# # On macOS, all symbols have a leading underscore -# symbols.add(sym.lstrip('_')) -# else: -# # Ignore symbols such as _edata (present in all shared objects) -# if sym[0] not in "_$.@": symbols.add(sym) - -# return symbols - def run_nm_defined_symbols(objfile): stdout = subprocess.check_output(["nm", "-g", "-P", objfile], encoding="ascii") # Check if we're running under Windows but not under MSYS2 (which provides Unix-like tools) @@ -142,8 +121,6 @@ def cython_internal(sym): return symbols - - # This function emulates the way distutils combines settings from sysconfig, # environment variables, and the extension being built. It returns a dictionary # representing the usual set of variables, suitable for writing to a generated @@ -200,22 +177,7 @@ def write_configvars_header(filename, ext, prefix): outf.write(f'#define {prefix}_{var} "{value}"\n') -@contextmanager def set_compiler_envvars(): -# tmp_vars = [] -# for var in ['CC', 'CFLAGS', 'LDFLAGS']: -# if var in os.environ: -# if var == 'CFLAGS' and 'CCSHARED' in sysconfig.get_config_vars(): -# os.environ[var] += ' ' + sysconfig.get_config_var('CCSHARED') -# print("# pysam: (env) {}={}".format(var, os.environ[var])) -# elif var in sysconfig.get_config_vars(): -# value = sysconfig.get_config_var(var) -# if var == 'CFLAGS' and 'CCSHARED' in sysconfig.get_config_vars(): -# value += ' ' + sysconfig.get_config_var('CCSHARED') -# print("# pysam: (sysconfig) {}={}".format(var, value)) -# os.environ[var] = value -# tmp_vars += [var] - tmp_vars = [] for var in ['CC', 'CFLAGS', 'LDFLAGS']: if var in os.environ: From b9db109dc21fc4c3b7b31f8304d24f781fdbd13a Mon Sep 17 00:00:00 2001 From: teepean Date: Wed, 10 Apr 2024 08:32:03 +0300 Subject: [PATCH 04/16] Update setup.py Couple of fixes to get pysam compiling on Linux --- setup.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 49c8688f2..35b7799e6 100644 --- a/setup.py +++ b/setup.py @@ -176,7 +176,7 @@ def write_configvars_header(filename, ext, prefix): for var, value in config.items(): outf.write(f'#define {prefix}_{var} "{value}"\n') - +@contextmanager def set_compiler_envvars(): tmp_vars = [] for var in ['CC', 'CFLAGS', 'LDFLAGS']: @@ -629,8 +629,8 @@ def run(self): os.path.splitext(f"cbcftools{suffix}")[0], ] internal_pysamutil_libraries = [ - os.path.splitext(f"cutils{suffix}")[0], - ] + os.path.splitext("cutils{}".format(suffix))[0]] + external_htslib_objects = [] if platform.system() == 'Windows': msys2_root = find_msys2_root() @@ -756,6 +756,7 @@ def prebuild_libcsamtools(ext, force): define_macros=define_macros, # for out-of-tree compilation, use absolute paths library_dirs=[os.path.abspath(x) for x in ["pysam"] + htslib_library_dirs], + extra_objects=external_htslib_objects, include_dirs=[os.path.abspath(x) for x in ["pysam"] + htslib_include_dirs + \ ["samtools", "samtools/lz4", "bcftools", "."] + include_os]) From 7d4f69b7a0eb5244bfac38a7d2dfa6128b88c4c5 Mon Sep 17 00:00:00 2001 From: teepean Date: Wed, 10 Apr 2024 09:04:29 +0300 Subject: [PATCH 05/16] Update setup.py Couple more fixes for Linux --- setup.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 35b7799e6..5a8e6c9fb 100644 --- a/setup.py +++ b/setup.py @@ -387,7 +387,10 @@ def build_extension(self, ext): '-Wl,-x'] else: if not ext.extra_link_args: - ext.extra_link_args = ['-shared'] + if platform.system() == "Windows": + ext.extra_link_args = ['-shared'] + else: + ext.extra_link_args = [] ext.extra_link_args += ['-Wl,-rpath,$ORIGIN'] @@ -607,7 +610,7 @@ def run(self): if os.environ.get("CIBUILDWHEEL", "0") == "1": define_macros.append(("BUILDING_WHEEL", None)) -suffix = sysconfig.get_config_var('EXT_SUFFIX') +suffix = sysconfig.get_config_var('EXT_SUFFIX') or sysconfig.get_config_var('SO') if platform.system() == 'Windows': internal_htslib_libraries = [ @@ -655,7 +658,10 @@ def run(self): else: print("MSYS2 root not found.") -libraries_for_pysam_module = internal_htslib_libraries + internal_pysamutil_libraries +if platform.system() == "Windows": + libraries_for_pysam_module = internal_htslib_libraries + internal_pysamutil_libraries +else: + libraries_for_pysam_module = external_htslib_libraries + internal_htslib_libraries + internal_pysamutil_libraries # Order of modules matters in order to make sure that dependencies are resolved. # The structures of dependencies is as follows: @@ -756,11 +762,12 @@ def prebuild_libcsamtools(ext, force): define_macros=define_macros, # for out-of-tree compilation, use absolute paths library_dirs=[os.path.abspath(x) for x in ["pysam"] + htslib_library_dirs], - - extra_objects=external_htslib_objects, include_dirs=[os.path.abspath(x) for x in ["pysam"] + htslib_include_dirs + \ ["samtools", "samtools/lz4", "bcftools", "."] + include_os]) +if platform.system() == "Windows": + common_options["extra_objects"] = external_htslib_objects + # add common options (in python >3.5, could use n = {**a, **b} for module in modules: module.update(**common_options) From c09d8824950b2ba7059892e8c91f42b47f8de9fb Mon Sep 17 00:00:00 2001 From: teepean Date: Wed, 6 Nov 2024 04:10:51 +0200 Subject: [PATCH 06/16] Small update --- setup.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5a8e6c9fb..45570715d 100644 --- a/setup.py +++ b/setup.py @@ -548,10 +548,28 @@ def run(self): # Link with the object files rather than the final htslib/libhts.a, to ensure that # all object files are pulled into the link, even those not used by htslib itself. +# Initialize htslib_make_options for Windows with suitable default values or as empty +if sys.platform == "win32": + htslib_make_options = { + # 'LIBHTS_OBJS' might not be relevant for Windows if linking against a precompiled lib + # Define it or leave it out based on your specific setup + 'LIBHTS_OBJS': '' + } + # Adjust paths to precompiled HTSlib objects or libraries as necessary + htslib_objects = ['htslib-msvc/libhts.lib'] + separate_htslib_objects = [] +else: + # Unix-like platform logic... + with changedir("htslib"): + htslib_make_options = run_make_print_config() + htslib_objects = [os.path.join("htslib", x) - for x in htslib_make_options["LIBHTS_OBJS"].split(" ")] + for x in htslib_make_options.get("LIBHTS_OBJS", "").split(" ")] separate_htslib_objects = [] +# Example continues... + + htslib_library_dirs = ["."] # when using setup.py develop? htslib_include_dirs = ['htslib'] else: From 2b84e55cf7479e6b35eafb4872928eeb739f333d Mon Sep 17 00:00:00 2001 From: Andrew Robbins Date: Mon, 17 Mar 2025 17:35:41 -0400 Subject: [PATCH 07/16] add windows to ci --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bd8e9797c..e2e67df43 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -7,7 +7,7 @@ jobs: runs-on: ${{ matrix.os }}-latest strategy: matrix: - os: [ubuntu, macos] + os: [ubuntu, macos, windows] python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14'] exclude: # Avoid spurious "upgrade pip" error annotation From 8be4453de532280579ace66c7b82abe43fdb317c Mon Sep 17 00:00:00 2001 From: Andrew Robbins Date: Mon, 17 Mar 2025 17:58:54 -0400 Subject: [PATCH 08/16] add setup-msys2 --- .github/workflows/ci.yaml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e2e67df43..2c037fced 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -37,6 +37,29 @@ jobs: run: | brew unlink xz || true # Remove brewed liblzma as it is not multiarch + - name: Install MSYS2 build prerequisites + if: runner.os == 'Windows' + uses: msys2/setup-msys2@v2 + with: + msystem: ucrt64 + update: true + install: >- + base-devel + mingw-w64-ucrt-x86_64-toolchain + mingw-w64-ucrt-x86_64-cython + mingw-w64-ucrt-x86_64-python + mingw-w64-ucrt-x86_64-python-pip + mingw-w64-ucrt-x86_64-python-setuptools + mingw-w64-ucrt-x86_64-curl + mingw-w64-ucrt-x86_64-libdeflate + mingw-w64-ucrt-x86_64-bzip2 + mingw-w64-ucrt-x86_64-zlib + mingw-w64-ucrt-x86_64-python-pytest + mingw-w64-ucrt-x86_64-xz + mingw-w64-ucrt-x86_64-openssl + mingw-w64-ucrt-x86_64-libtre + mingw-w64-ucrt-x86_64-libiconv + - name: Build (directly from checkout) run: python setup.py build From cc149ee836cf0a45e7066d1b6df8ad1bd089f9dd Mon Sep 17 00:00:00 2001 From: Andrew Robbins Date: Mon, 17 Mar 2025 18:25:20 -0400 Subject: [PATCH 09/16] split windows run env --- .github/workflows/ci.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2c037fced..21f5106f4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -38,6 +38,7 @@ jobs: brew unlink xz || true # Remove brewed liblzma as it is not multiarch - name: Install MSYS2 build prerequisites + id: msys2 if: runner.os == 'Windows' uses: msys2/setup-msys2@v2 with: @@ -61,6 +62,12 @@ jobs: mingw-w64-ucrt-x86_64-libiconv - name: Build (directly from checkout) + if: runner.os != 'Windows' + run: python setup.py build + + - name: Build (directly from checkout) + if: runner.os == 'Windows' + shell: msys2 {0} run: python setup.py build - name: Install test prerequisites From 8ecb540c010b5efd220d023bcfda359af987a55e Mon Sep 17 00:00:00 2001 From: Andrew Robbins Date: Mon, 17 Mar 2025 20:07:39 -0400 Subject: [PATCH 10/16] do not install msys python --- .github/workflows/ci.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 21f5106f4..ad47e6a79 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -47,15 +47,10 @@ jobs: install: >- base-devel mingw-w64-ucrt-x86_64-toolchain - mingw-w64-ucrt-x86_64-cython - mingw-w64-ucrt-x86_64-python - mingw-w64-ucrt-x86_64-python-pip - mingw-w64-ucrt-x86_64-python-setuptools mingw-w64-ucrt-x86_64-curl mingw-w64-ucrt-x86_64-libdeflate mingw-w64-ucrt-x86_64-bzip2 mingw-w64-ucrt-x86_64-zlib - mingw-w64-ucrt-x86_64-python-pytest mingw-w64-ucrt-x86_64-xz mingw-w64-ucrt-x86_64-openssl mingw-w64-ucrt-x86_64-libtre From 34996f8d21fe4aab7f734bbb423f5f137f1a743e Mon Sep 17 00:00:00 2001 From: Andrew Robbins Date: Mon, 17 Mar 2025 20:11:12 -0400 Subject: [PATCH 11/16] do not import distutils --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 45570715d..f4ee93fc3 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,6 @@ import subprocess import sys import sysconfig -import distutils from contextlib import contextmanager from setuptools import setup, Command from setuptools.command.sdist import sdist From 070a8c9082aa1cb095753ad2933b6991efdd5a37 Mon Sep 17 00:00:00 2001 From: Andrew Robbins Date: Mon, 17 Mar 2025 21:57:01 -0400 Subject: [PATCH 12/16] replace toolchian with subset --- .github/workflows/ci.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ad47e6a79..0bb2c5d63 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -46,7 +46,14 @@ jobs: update: true install: >- base-devel - mingw-w64-ucrt-x86_64-toolchain + mingw-w64-ucrt-x86_64-binutils + mingw-w64-ucrt-x86_64-headers-git + mingw-w64-ucrt-x86_64-gcc + mingw-w64-ucrt-x86_64-libmangle-git + mingw-w64-ucrt-x86_64-winpthreads-git + mingw-w64-ucrt-x86_64-make + mingw-w64-ucrt-x86_64-pkgconf + mingw-w64-ucrt-x86_64-tools-git mingw-w64-ucrt-x86_64-curl mingw-w64-ucrt-x86_64-libdeflate mingw-w64-ucrt-x86_64-bzip2 From 3f8b6dcce0a60cf3c66d29d9149cb6741263ebe7 Mon Sep 17 00:00:00 2001 From: Andrew Robbins Date: Mon, 17 Mar 2025 22:01:23 -0400 Subject: [PATCH 13/16] use path to setup-python python --- .github/workflows/ci.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0bb2c5d63..752aa16aa 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,7 +19,8 @@ jobs: uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@v5 + id: setup-python with: python-version: ${{ matrix.python-version }} @@ -70,7 +71,7 @@ jobs: - name: Build (directly from checkout) if: runner.os == 'Windows' shell: msys2 {0} - run: python setup.py build + run: ${{ steps.setup-python.outputs.python-path }} setup.py build - name: Install test prerequisites run: | From 9248ace4568887fabe6a69e06a3ae5e6c829c96b Mon Sep 17 00:00:00 2001 From: Andrew Robbins Date: Mon, 17 Mar 2025 22:08:48 -0400 Subject: [PATCH 14/16] cygpath needed --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 752aa16aa..7bfd59441 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -71,7 +71,7 @@ jobs: - name: Build (directly from checkout) if: runner.os == 'Windows' shell: msys2 {0} - run: ${{ steps.setup-python.outputs.python-path }} setup.py build + run: sh -c '$(cygpath "${{ steps.setup-python.outputs.python-path }}") setup.py build' - name: Install test prerequisites run: | From 95bd9cd1806fcce810bb7de4c51594462aaf266c Mon Sep 17 00:00:00 2001 From: Andrew Robbins Date: Mon, 17 Mar 2025 22:33:20 -0400 Subject: [PATCH 15/16] need systre --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7bfd59441..d33128408 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -61,7 +61,7 @@ jobs: mingw-w64-ucrt-x86_64-zlib mingw-w64-ucrt-x86_64-xz mingw-w64-ucrt-x86_64-openssl - mingw-w64-ucrt-x86_64-libtre + mingw-w64-ucrt-x86_64-libsystre mingw-w64-ucrt-x86_64-libiconv - name: Build (directly from checkout) From 1b607839755846ed8400c82d3ccb7150eacd4075 Mon Sep 17 00:00:00 2001 From: Andrew Robbins Date: Tue, 18 Mar 2025 17:33:47 -0400 Subject: [PATCH 16/16] build for windows --- setup.cfg | 3 + setup.py | 252 +++++++++++++++++++++++++++++++++++++----------------- 2 files changed, 178 insertions(+), 77 deletions(-) diff --git a/setup.cfg b/setup.cfg index ef3f51179..947fcc510 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,9 @@ [bdist_wheel] universal = 0 +[build_ext] +compiler = mingw32 + [tool:pytest] # -s: do not capture stdout, conflicts with pysam.dispatch # -v: verbose output diff --git a/setup.py b/setup.py index f4ee93fc3..e42c6b782 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ import subprocess import sys import sysconfig + from contextlib import contextmanager from setuptools import setup, Command from setuptools.command.sdist import sdist @@ -70,7 +71,7 @@ def run_configure(option): try: # Always disable ref-cache as its code is omitted from pysam's htslib/ retcode = subprocess.call( - " ".join(("./configure", "--disable-ref-cache", option)), + " ".join((command, "--disable-ref-cache", option)), shell=True) if retcode != 0: return False @@ -548,16 +549,16 @@ def run(self): # Link with the object files rather than the final htslib/libhts.a, to ensure that # all object files are pulled into the link, even those not used by htslib itself. # Initialize htslib_make_options for Windows with suitable default values or as empty -if sys.platform == "win32": + if sys.platform == "win32": htslib_make_options = { # 'LIBHTS_OBJS' might not be relevant for Windows if linking against a precompiled lib # Define it or leave it out based on your specific setup 'LIBHTS_OBJS': '' } # Adjust paths to precompiled HTSlib objects or libraries as necessary - htslib_objects = ['htslib-msvc/libhts.lib'] + htslib_objects = ['htslib/hts-3.lib'] separate_htslib_objects = [] -else: + else: # Unix-like platform logic... with changedir("htslib"): htslib_make_options = run_make_print_config() @@ -569,8 +570,8 @@ def run(self): # Example continues... - htslib_library_dirs = ["."] # when using setup.py develop? - htslib_include_dirs = ['htslib'] + htslib_library_dirs = ["."] # when using setup.py develop? + htslib_include_dirs = ['htslib'] else: raise ValueError(f"unknown HTSLIB value {HTSLIB_MODE!r}") @@ -630,15 +631,16 @@ def run(self): suffix = sysconfig.get_config_var('EXT_SUFFIX') or sysconfig.get_config_var('SO') if platform.system() == 'Windows': + base_path_win = 'build/lib.' + sysconfig.get_platform() + '-' + sys.implementation.cache_tag + '/pysam' internal_htslib_libraries = [ - os.path.splitext(f"libchtslib{suffix}")[0], + f"{base_path_win}/libchtslib{suffix}" ] internal_samtools_libraries = [ - os.path.splitext(f"libcsamtools{suffix}")[0], - os.path.splitext(f"libcbcftools{suffix}")[0], + f"{base_path_win}/libcsamtools{suffix}", + f"{base_path_win}/libcbcftools{suffix}", ] internal_pysamutil_libraries = [ - os.path.splitext(f"libcutils{suffix}")[0], + f"{base_path_win}/libcutils{suffix}" ] else: internal_htslib_libraries = [ @@ -656,7 +658,7 @@ def run(self): msys2_root = find_msys2_root() if msys2_root: external_htslib_objects = [] - external_htslib_objects.append('htslib/libhts.a') + external_htslib_objects.append('htslib/hts-3.lib') msystem = subprocess.check_output(['echo', '%MSYSTEM%'], shell=True, text=True).strip() if msystem in ['MINGW32', 'MINGW64', 'UCRT64', 'CLANG64', 'CLANG32', 'CLANGARM64']: @@ -676,7 +678,7 @@ def run(self): print("MSYS2 root not found.") if platform.system() == "Windows": - libraries_for_pysam_module = internal_htslib_libraries + internal_pysamutil_libraries + libraries_for_pysam_module = [] else: libraries_for_pysam_module = external_htslib_libraries + internal_htslib_libraries + internal_pysamutil_libraries @@ -703,7 +705,11 @@ def prebuild_libchtslib(ext, force): # extra flags for configure instead of hacking on ALL_CPPFLAGS. args = " ".join(ext.extra_compile_args) defines = " ".join([format_macro_option(*pair) for pair in ext.define_macros]) - run_make(["ALL_CPPFLAGS=-I. " + args + " " + defines + " $(CPPFLAGS)", "lib-static"]) + if platform.system() == "Windows": + run_make(["ALL_CPPFLAGS=-I. " + args + " " + defines + " $(CPPFLAGS)", "lib-shared"]) + else: + run_make(["ALL_CPPFLAGS=-I. " + args + " " + defines + " $(CPPFLAGS)", "lib-static"]) + else: log.warning("skipping 'libhts.a' (already built)") @@ -711,67 +717,162 @@ def prebuild_libchtslib(ext, force): def prebuild_libcsamtools(ext, force): write_configvars_header("samtools/samtools_config_vars.h", ext, "SAMTOOLS") - -modules = [ - dict(name="pysam.libchtslib", - prebuild_func=prebuild_libchtslib, - sources=[source_pattern % "htslib", "pysam/htslib_util.c"] + os_c_files, - extra_objects=htslib_objects, - extra_link_args=["-Wl,--export-all-symbols"]), - dict(name="pysam.libcsamtools", - prebuild_func=prebuild_libcsamtools, - sources=[source_pattern % "samtools"] + glob.glob(os.path.join("samtools", "*.pysam.c")) + - [os.path.join("samtools", "lz4", "lz4.c")] + os_c_files, - extra_objects=separate_htslib_objects, - libraries=internal_htslib_libraries, - extra_link_args=["-Wl,--export-all-symbols"]), - dict(name="pysam.libcbcftools", - sources=[source_pattern % "bcftools"] + glob.glob(os.path.join("bcftools", "*.pysam.c")) + os_c_files, - extra_objects=separate_htslib_objects, - libraries=internal_htslib_libraries, - extra_link_args=["-Wl,--export-all-symbols"]), - dict(name="pysam.libcutils", - sources=[source_pattern % "utils"] + os_c_files, - extra_objects=separate_htslib_objects, - #libraries=external_htslib_libraries + internal_htslib_libraries + internal_samtools_libraries), - libraries=internal_htslib_libraries + internal_samtools_libraries), - dict(name="pysam.libcalignmentfile", - sources=[source_pattern % "alignmentfile"] + os_c_files, - extra_objects=separate_htslib_objects, - libraries=libraries_for_pysam_module), - dict(name="pysam.libcsamfile", - sources=[source_pattern % "samfile"] + os_c_files, - extra_objects=separate_htslib_objects, - libraries=libraries_for_pysam_module), - dict(name="pysam.libcalignedsegment", - sources=[source_pattern % "alignedsegment"] + os_c_files, - extra_objects=separate_htslib_objects, - libraries=libraries_for_pysam_module), - dict(name="pysam.libctabix", - sources=[source_pattern % "tabix"] + os_c_files, - extra_objects=separate_htslib_objects, - libraries=libraries_for_pysam_module), - dict(name="pysam.libcfaidx", - sources=[source_pattern % "faidx"] + os_c_files, - extra_objects=separate_htslib_objects, - libraries=libraries_for_pysam_module), - dict(name="pysam.libcbcf", - sources=[source_pattern % "bcf"] + os_c_files, - extra_objects=separate_htslib_objects, - libraries=libraries_for_pysam_module), - dict(name="pysam.libcbgzf", - sources=[source_pattern % "bgzf"] + os_c_files, - extra_objects=separate_htslib_objects, - libraries=libraries_for_pysam_module), - dict(name="pysam.libctabixproxies", - sources=[source_pattern % "tabixproxies"] + os_c_files, - extra_objects=separate_htslib_objects, - libraries=libraries_for_pysam_module), - dict(name="pysam.libcvcf", - sources=[source_pattern % "vcf"] + os_c_files, - extra_objects=separate_htslib_objects, - libraries=libraries_for_pysam_module), -] +if platform.system() == 'Windows': + modules = [ + dict( + name="pysam.libchtslib", + prebuild_func=prebuild_libchtslib, + sources=[source_pattern % "htslib", "pysam/htslib_util.c"] + os_c_files, + extra_objects=htslib_objects + external_htslib_objects, + extra_link_args=["-Wl,--export-all-symbols"], + ), + dict( + name="pysam.libcsamtools", + prebuild_func=prebuild_libcsamtools, + sources=[source_pattern % "samtools"] + + glob.glob(os.path.join("samtools", "*.pysam.c")) + + [os.path.join("samtools", "lz4", "lz4.c")] + + os_c_files, + extra_objects=separate_htslib_objects + + internal_htslib_libraries + + external_htslib_objects, + extra_link_args=["-Wl,--export-all-symbols"], + ), + dict( + name="pysam.libcbcftools", + sources=[source_pattern % "bcftools"] + + glob.glob(os.path.join("bcftools", "*.pysam.c")) + + os_c_files, + extra_objects=separate_htslib_objects + + internal_htslib_libraries + + external_htslib_objects, + extra_link_args=["-Wl,--export-all-symbols"], + ), + dict( + name="pysam.libcutils", + sources=[source_pattern % "utils"] + os_c_files, + extra_objects=separate_htslib_objects + + external_htslib_objects + + internal_htslib_libraries + + internal_samtools_libraries, + # libraries=external_htslib_libraries + internal_htslib_libraries + internal_samtools_libraries), + ), + dict( + name="pysam.libcalignmentfile", + sources=[source_pattern % "alignmentfile"] + os_c_files, + extra_objects=separate_htslib_objects + external_htslib_objects + internal_htslib_libraries, + libraries=libraries_for_pysam_module, + ), + dict( + name="pysam.libcsamfile", + sources=[source_pattern % "samfile"] + os_c_files, + extra_objects=separate_htslib_objects + external_htslib_objects + internal_htslib_libraries, + libraries=libraries_for_pysam_module, + ), + dict( + name="pysam.libcalignedsegment", + sources=[source_pattern % "alignedsegment"] + os_c_files, + extra_objects=separate_htslib_objects + internal_htslib_libraries + external_htslib_objects, + libraries=libraries_for_pysam_module, + ), + dict( + name="pysam.libctabix", + sources=[source_pattern % "tabix"] + os_c_files, + extra_objects=separate_htslib_objects + external_htslib_objects, + libraries=libraries_for_pysam_module, + ), + dict( + name="pysam.libcfaidx", + sources=[source_pattern % "faidx"] + os_c_files, + extra_objects=separate_htslib_objects + external_htslib_objects, + libraries=libraries_for_pysam_module, + ), + dict( + name="pysam.libcbcf", + sources=[source_pattern % "bcf"] + os_c_files, + extra_objects=separate_htslib_objects + external_htslib_objects, + libraries=libraries_for_pysam_module, + ), + dict( + name="pysam.libcbgzf", + sources=[source_pattern % "bgzf"] + os_c_files, + extra_objects=separate_htslib_objects + external_htslib_objects, + libraries=libraries_for_pysam_module, + ), + dict( + name="pysam.libctabixproxies", + sources=[source_pattern % "tabixproxies"] + os_c_files, + extra_objects=separate_htslib_objects + external_htslib_objects, + libraries=libraries_for_pysam_module, + ), + dict( + name="pysam.libcvcf", + sources=[source_pattern % "vcf"] + os_c_files, + extra_objects=separate_htslib_objects + external_htslib_objects, + libraries=libraries_for_pysam_module, + ), + ] +else: + modules = [ + dict(name="pysam.libchtslib", + prebuild_func=prebuild_libchtslib, + sources=[source_pattern % "htslib", "pysam/htslib_util.c"] + os_c_files, + extra_objects=htslib_objects, + extra_link_args=["-Wl,--export-all-symbols"]), + dict(name="pysam.libcsamtools", + prebuild_func=prebuild_libcsamtools, + sources=[source_pattern % "samtools"] + glob.glob(os.path.join("samtools", "*.pysam.c")) + + [os.path.join("samtools", "lz4", "lz4.c")] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=internal_htslib_libraries, + extra_link_args=["-Wl,--export-all-symbols"]), + dict(name="pysam.libcbcftools", + sources=[source_pattern % "bcftools"] + glob.glob(os.path.join("bcftools", "*.pysam.c")) + os_c_files, + extra_objects=separate_htslib_objects, + libraries=internal_htslib_libraries, + extra_link_args=["-Wl,--export-all-symbols"]), + dict(name="pysam.libcutils", + sources=[source_pattern % "utils"] + os_c_files, + extra_objects=separate_htslib_objects, + #libraries=external_htslib_libraries + internal_htslib_libraries + internal_samtools_libraries), + libraries=internal_htslib_libraries + internal_samtools_libraries), + dict(name="pysam.libcalignmentfile", + sources=[source_pattern % "alignmentfile"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + dict(name="pysam.libcsamfile", + sources=[source_pattern % "samfile"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + dict(name="pysam.libcalignedsegment", + sources=[source_pattern % "alignedsegment"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + dict(name="pysam.libctabix", + sources=[source_pattern % "tabix"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + dict(name="pysam.libcfaidx", + sources=[source_pattern % "faidx"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + dict(name="pysam.libcbcf", + sources=[source_pattern % "bcf"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + dict(name="pysam.libcbgzf", + sources=[source_pattern % "bgzf"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + dict(name="pysam.libctabixproxies", + sources=[source_pattern % "tabixproxies"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + dict(name="pysam.libcvcf", + sources=[source_pattern % "vcf"] + os_c_files, + extra_objects=separate_htslib_objects, + libraries=libraries_for_pysam_module), + ] common_options = dict( language="c", @@ -782,9 +883,6 @@ def prebuild_libcsamtools(ext, force): include_dirs=[os.path.abspath(x) for x in ["pysam"] + htslib_include_dirs + \ ["samtools", "samtools/lz4", "bcftools", "."] + include_os]) -if platform.system() == "Windows": - common_options["extra_objects"] = external_htslib_objects - # add common options (in python >3.5, could use n = {**a, **b} for module in modules: module.update(**common_options)