diff --git a/pygit2/__init__.py b/pygit2/__init__.py index 8d4346ff..54a06548 100644 --- a/pygit2/__init__.py +++ b/pygit2/__init__.py @@ -365,7 +365,7 @@ from .settings import Settings from .submodules import Submodule from .transaction import ReferenceTransaction -from .utils import to_bytes, to_str +from .utils import decode_fs_path, encode_fs_path, to_bytes, to_str # Features features = enums.Feature(C.git_libgit2_features()) @@ -430,7 +430,7 @@ def init_repository( options.mode = mode if workdir_path: - workdir_path_ref = ffi.new('char []', to_bytes(workdir_path)) + workdir_path_ref = ffi.new('char []', encode_fs_path(workdir_path)) options.workdir_path = workdir_path_ref if description: @@ -438,7 +438,7 @@ def init_repository( options.description = description_ref if template_path: - template_path_ref = ffi.new('char []', to_bytes(template_path)) + template_path_ref = ffi.new('char []', encode_fs_path(template_path)) options.template_path = template_path_ref if initial_head: @@ -451,7 +451,7 @@ def init_repository( # Call crepository = ffi.new('git_repository **') - err = C.git_repository_init_ext(crepository, to_bytes(path), options) + err = C.git_repository_init_ext(crepository, encode_fs_path(path), options) check_error(err) # Ok @@ -536,7 +536,7 @@ def clone_repository( with git_fetch_options(payload, opts=opts.fetch_opts): with git_proxy_options(payload, opts.fetch_opts.proxy_opts, proxy): crepo = ffi.new('git_repository **') - err = C.git_clone(crepo, to_bytes(url), to_bytes(path), opts) + err = C.git_clone(crepo, to_bytes(url), encode_fs_path(path), opts) payload.check_error(err) # Ok diff --git a/pygit2/blame.py b/pygit2/blame.py index 9854f3af..b677f3b4 100644 --- a/pygit2/blame.py +++ b/pygit2/blame.py @@ -30,7 +30,7 @@ # Import from pygit2 from .ffi import C, ffi -from .utils import GenericIterator +from .utils import GenericIterator, maybe_string if TYPE_CHECKING: from ._libgit2.ffi import GitBlameC, GitHunkC, GitSignatureC @@ -110,7 +110,7 @@ def orig_path(self) -> None | str: if not path: return None - return ffi.string(path).decode('utf-8') + return maybe_string(path) class Blame: diff --git a/pygit2/callbacks.py b/pygit2/callbacks.py index 1072f8cf..a0aa8da2 100644 --- a/pygit2/callbacks.py +++ b/pygit2/callbacks.py @@ -74,7 +74,14 @@ from .enums import CheckoutNotify, CheckoutStrategy, CredentialType, StashApplyProgress from .errors import Passthrough, check_error from .ffi import C, ffi -from .utils import StrArray, maybe_string, ptr_to_bytes, to_bytes +from .utils import ( + StrArray, + decode_fs_path, + encode_fs_path, + maybe_string, + ptr_to_bytes, + to_bytes, +) _Credentials = Username | UserPass | Keypair @@ -784,7 +791,7 @@ def get_credentials(fn, url, username, allowed): def _checkout_notify_cb( why, path_cstr, baseline, target, workdir, data: CheckoutCallbacks ): - pypath = maybe_string(path_cstr) + pypath = decode_fs_path(path_cstr) pybaseline = DiffFile.from_c(ptr_to_bytes(baseline)) pytarget = DiffFile.from_c(ptr_to_bytes(target)) pyworkdir = DiffFile.from_c(ptr_to_bytes(workdir)) @@ -805,7 +812,7 @@ def _checkout_notify_cb( @libgit2_callback_void def _checkout_progress_cb(path, completed_steps, total_steps, data: CheckoutCallbacks): - data.checkout_progress(maybe_string(path), completed_steps, total_steps) # type: ignore[arg-type] + data.checkout_progress(decode_fs_path(path), completed_steps, total_steps) # type: ignore[arg-type] def _git_checkout_options( @@ -839,7 +846,7 @@ def _git_checkout_options( opts.checkout_strategy = int(strategy) if directory: - target_dir = ffi.new('char[]', to_bytes(directory)) + target_dir = ffi.new('char[]', encode_fs_path(directory)) refs.append(target_dir) opts.target_directory = target_dir diff --git a/pygit2/config.py b/pygit2/config.py index e7522ffe..877bf0f0 100644 --- a/pygit2/config.py +++ b/pygit2/config.py @@ -35,7 +35,7 @@ # Import from pygit2 from .errors import check_error from .ffi import C, ffi -from .utils import to_bytes +from .utils import encode_fs_path, to_bytes if TYPE_CHECKING: from ._libgit2.ffi import GitConfigC, GitConfigEntryC @@ -116,7 +116,7 @@ def __init__(self, path: PathLike | str | None = None) -> None: if not path: err = C.git_config_new(cconfig) else: - path_bytes = to_bytes(path) + path_bytes = encode_fs_path(path) err = C.git_config_open_ondisk(cconfig, path_bytes) check_error(err, io=True) @@ -282,7 +282,7 @@ def add_file(self, path: str | PathLike, level: int = 0, force: int = 0) -> None """Add a config file instance to an existing config.""" err = C.git_config_add_file_ondisk( - self._config, to_bytes(path), level, ffi.NULL, force + self._config, encode_fs_path(path), level, ffi.NULL, force ) check_error(err) diff --git a/pygit2/filter.py b/pygit2/filter.py index 6fc953e3..9c53b471 100644 --- a/pygit2/filter.py +++ b/pygit2/filter.py @@ -32,7 +32,7 @@ from ._pygit2 import Blob, FilterSource from .errors import check_error from .ffi import C, ffi -from .utils import to_bytes +from .utils import encode_fs_path, to_bytes if TYPE_CHECKING: from ._libgit2.ffi import GitFilterListC @@ -178,7 +178,7 @@ def apply_to_file(self, repo: BaseRepository, path: str) -> bytes: Return the filtered contents. """ buf = ffi.new('git_buf *') - c_path = to_bytes(path) + c_path = encode_fs_path(path) err = C.git_filter_list_apply_to_file(buf, self._pointer, repo._repo, c_path) check_error(err) try: diff --git a/pygit2/index.py b/pygit2/index.py index e80273ce..6af3b933 100644 --- a/pygit2/index.py +++ b/pygit2/index.py @@ -33,7 +33,7 @@ from .enums import DiffOption, FileMode from .errors import check_error from .ffi import C, ffi -from .utils import GenericIterator, StrArray, to_bytes, to_str +from .utils import GenericIterator, StrArray, decode_fs_path, encode_fs_path if typing.TYPE_CHECKING: from .repository import Repository @@ -52,7 +52,7 @@ def __init__(self, path: str | PathLike[str] | None = None) -> None: to read from and write to. """ cindex = ffi.new('git_index **') - err = C.git_index_open(cindex, to_bytes(path)) + err = C.git_index_open(cindex, encode_fs_path(path)) check_error(err) self._repo = None @@ -79,7 +79,7 @@ def __len__(self) -> int: return C.git_index_entrycount(self._index) def __contains__(self, path) -> bool: - err = C.git_index_find(ffi.NULL, self._index, to_bytes(path)) + err = C.git_index_find(ffi.NULL, self._index, encode_fs_path(path)) if err == C.GIT_ENOTFOUND: return False @@ -89,7 +89,7 @@ def __contains__(self, path) -> bool: def __getitem__(self, key: str | int | PathLike[str]) -> 'IndexEntry': centry = ffi.NULL if isinstance(key, str) or hasattr(key, '__fspath__'): - centry = C.git_index_get_bypath(self._index, to_bytes(key), 0) + centry = C.git_index_get_bypath(self._index, encode_fs_path(key), 0) elif isinstance(key, int): if key >= 0: centry = C.git_index_get_byindex(self._index, key) @@ -180,12 +180,12 @@ def write_tree(self, repo: 'Repository | None' = None) -> Oid: def remove(self, path: PathLike[str] | str, level: int = 0) -> None: """Remove an entry from the Index.""" - err = C.git_index_remove(self._index, to_bytes(path), level) + err = C.git_index_remove(self._index, encode_fs_path(path), level) check_error(err, io=True) def remove_directory(self, path: PathLike[str] | str, level: int = 0) -> None: """Remove a directory from the Index.""" - err = C.git_index_remove_directory(self._index, to_bytes(path), level) + err = C.git_index_remove_directory(self._index, encode_fs_path(path), level) check_error(err, io=True) def remove_all(self, pathspecs: typing.Sequence[str | PathLike[str]]) -> None: @@ -221,7 +221,7 @@ def add(self, path_or_entry: 'IndexEntry | str | PathLike[str]') -> None: err = C.git_index_add(self._index, centry) elif isinstance(path_or_entry, str) or hasattr(path_or_entry, '__fspath__'): path = path_or_entry - err = C.git_index_add_bypath(self._index, to_bytes(path)) + err = C.git_index_add_bypath(self._index, encode_fs_path(path)) else: raise TypeError('argument must be string, Path or IndexEntry') @@ -420,7 +420,7 @@ def _from_c(cls, centry): return None automergeable = centry.automergeable != 0 - path = to_str(ffi.string(centry.path)) if centry.path else None + path = decode_fs_path(ffi.string(centry.path)) if centry.path else None mode = FileMode(centry.mode) contents = ffi.string(centry.ptr, centry.len).decode('utf-8') @@ -475,7 +475,7 @@ def _to_c(self) -> tuple['ffi.GitIndexEntryC', 'ffi.ArrayC[ffi.char]']: # basically memcpy() ffi.buffer(ffi.addressof(centry, 'id'))[:] = self.id.raw[:] centry.mode = int(self.mode) - path = ffi.new('char[]', to_bytes(self.path)) + path = ffi.new('char[]', encode_fs_path(self.path)) centry.path = path return centry, path @@ -486,7 +486,7 @@ def _from_c(cls, centry): return None entry = cls.__new__(cls) - entry.path = to_str(ffi.string(centry.path)) + entry.path = decode_fs_path(ffi.string(centry.path)) entry.mode = FileMode(centry.mode) entry.id = Oid(raw=bytes(ffi.buffer(ffi.addressof(centry, 'id'))[:])) @@ -503,7 +503,7 @@ def __getitem__(self, path): ctheirs = ffi.new('git_index_entry **') err = C.git_index_conflict_get( - cancestor, cours, ctheirs, self._index._index, to_bytes(path) + cancestor, cours, ctheirs, self._index._index, encode_fs_path(path) ) check_error(err) @@ -514,7 +514,7 @@ def __getitem__(self, path): return ancestor, ours, theirs def __delitem__(self, path): - err = C.git_index_conflict_remove(self._index._index, to_bytes(path)) + err = C.git_index_conflict_remove(self._index._index, encode_fs_path(path)) check_error(err) def __iter__(self): @@ -526,7 +526,7 @@ def __contains__(self, path): ctheirs = ffi.new('git_index_entry **') err = C.git_index_conflict_get( - cancestor, cours, ctheirs, self._index._index, to_bytes(path) + cancestor, cours, ctheirs, self._index._index, encode_fs_path(path) ) if err == C.GIT_ENOTFOUND: return False diff --git a/pygit2/options.py b/pygit2/options.py index ad96253b..5d4405f5 100644 --- a/pygit2/options.py +++ b/pygit2/options.py @@ -34,7 +34,7 @@ from .errors import check_error from .ffi import C, ffi -from .utils import to_bytes, to_str +from .utils import decode_fs_path, encode_fs_path, to_bytes, to_str if TYPE_CHECKING: from ._libgit2.ffi import NULL_TYPE, ArrayC, char, char_pointer @@ -348,7 +348,7 @@ def option(option_type: Option, arg1: Any = NOT_PASSED, arg2: Any = NOT_PASSED) try: if buf.ptr != ffi.NULL: - result = to_str(ffi.string(buf.ptr)) + result = decode_fs_path(ffi.string(buf.ptr)) else: result = None finally: @@ -366,7 +366,7 @@ def option(option_type: Option, arg1: Any = NOT_PASSED, arg2: Any = NOT_PASSED) if path is None: path_cdata = ffi.NULL else: - path_bytes = to_bytes(path) + path_bytes = encode_fs_path(path) path_cdata = ffi.new('char[]', path_bytes) err = C.git_libgit2_opts(option_type, ffi.cast('int', level), path_cdata) @@ -423,14 +423,14 @@ def option(option_type: Option, arg1: Any = NOT_PASSED, arg2: Any = NOT_PASSED) if cert_file is None: cert_file_cdata = ffi.NULL else: - cert_file_bytes = to_bytes(cert_file) + cert_file_bytes = encode_fs_path(cert_file) cert_file_cdata = ffi.new('char[]', cert_file_bytes) cert_dir_cdata: ArrayC[char] | NULL_TYPE if cert_dir is None: cert_dir_cdata = ffi.NULL else: - cert_dir_bytes = to_bytes(cert_dir) + cert_dir_bytes = encode_fs_path(cert_dir) cert_dir_cdata = ffi.new('char[]', cert_dir_bytes) err = C.git_libgit2_opts(option_type, cert_file_cdata, cert_dir_cdata) @@ -476,7 +476,7 @@ def option(option_type: Option, arg1: Any = NOT_PASSED, arg2: Any = NOT_PASSED) try: if buf.ptr != ffi.NULL: - result = to_str(ffi.string(buf.ptr)) + result = decode_fs_path(ffi.string(buf.ptr)) else: result = None finally: @@ -492,7 +492,7 @@ def option(option_type: Option, arg1: Any = NOT_PASSED, arg2: Any = NOT_PASSED) if path is None: template_path_cdata = ffi.NULL else: - path_bytes = to_bytes(path) + path_bytes = encode_fs_path(path) template_path_cdata = ffi.new('char[]', path_bytes) err = C.git_libgit2_opts(option_type, template_path_cdata) @@ -688,7 +688,7 @@ def option(option_type: Option, arg1: Any = NOT_PASSED, arg2: Any = NOT_PASSED) try: if buf.ptr != ffi.NULL: - result = to_str(ffi.string(buf.ptr)) + result = decode_fs_path(ffi.string(buf.ptr)) else: result = None finally: @@ -705,7 +705,7 @@ def option(option_type: Option, arg1: Any = NOT_PASSED, arg2: Any = NOT_PASSED) if path is None: homedir_cdata = ffi.NULL else: - path_bytes = to_bytes(path) + path_bytes = encode_fs_path(path) homedir_cdata = ffi.new('char[]', path_bytes) err = C.git_libgit2_opts(option_type, homedir_cdata) diff --git a/pygit2/packbuilder.py b/pygit2/packbuilder.py index baa54630..f9b77c5d 100644 --- a/pygit2/packbuilder.py +++ b/pygit2/packbuilder.py @@ -29,7 +29,7 @@ # Import from pygit2 from .errors import check_error from .ffi import C, ffi -from .utils import to_bytes +from .utils import encode_fs_path if TYPE_CHECKING: from pygit2 import Oid, Repository @@ -76,7 +76,7 @@ def set_threads(self, n_threads: int) -> int: return C.git_packbuilder_set_threads(self._packbuilder, n_threads) def write(self, path: str | bytes | PathLike[str] | None = None) -> None: - path_bytes = ffi.NULL if path is None else to_bytes(path) + path_bytes = ffi.NULL if path is None else encode_fs_path(path) err = C.git_packbuilder_write( self._packbuilder, path_bytes, 0, ffi.NULL, ffi.NULL ) diff --git a/pygit2/repository.py b/pygit2/repository.py index f3c34e81..3fa9a43e 100644 --- a/pygit2/repository.py +++ b/pygit2/repository.py @@ -81,7 +81,7 @@ from .remotes import RemoteCollection from .submodules import SubmoduleCollection from .transaction import ReferenceTransaction -from .utils import StrArray, maybe_string, to_bytes +from .utils import StrArray, decode_fs_path, encode_fs_path, maybe_string, to_bytes if TYPE_CHECKING: from pygit2._libgit2.ffi import ( @@ -219,7 +219,7 @@ def hashfile( If this is `None` and the `path` parameter is a file within the repository's working directory, then the `path` will be used. """ - c_path = to_bytes(path) + c_path = encode_fs_path(path) c_as_path: ffi.NULL_TYPE | bytes if as_path is None: @@ -1824,7 +1824,7 @@ def __init__( if hasattr(path, '__fspath__'): path = path.__fspath__() if not isinstance(path, str): - path = path.decode('utf-8') + path = decode_fs_path(path) path_backend = init_file_backend(path, int(flags)) super().__init__(path_backend) else: diff --git a/pygit2/submodules.py b/pygit2/submodules.py index 4ff00980..67a32882 100644 --- a/pygit2/submodules.py +++ b/pygit2/submodules.py @@ -138,14 +138,12 @@ def reload(self, force: bool = False) -> None: @property def name(self): """Name of the submodule.""" - name = C.git_submodule_name(self._subm) - return ffi.string(name).decode('utf-8') + return maybe_string(C.git_submodule_name(self._subm)) @property def path(self): """Path of the submodule.""" - path = C.git_submodule_path(self._subm) - return ffi.string(path).decode('utf-8') + return maybe_string(C.git_submodule_path(self._subm)) @property def url(self) -> str | None: @@ -164,8 +162,7 @@ def url(self, url: str) -> None: @property def branch(self): """Branch that is to be tracked by the submodule.""" - branch = C.git_submodule_branch(self._subm) - return ffi.string(branch).decode('utf-8') + return maybe_string(C.git_submodule_branch(self._subm)) @property def head_id(self) -> Oid | None: diff --git a/pygit2/utils.py b/pygit2/utils.py index 4a36a06f..3fe71a64 100644 --- a/pygit2/utils.py +++ b/pygit2/utils.py @@ -51,6 +51,46 @@ def maybe_string(ptr: 'char_pointer | None') -> str | None: return ffi.string(ptr).decode('utf8', errors='surrogateescape') +@overload +def decode_fs_path(ptr: 'char_pointer') -> str: ... +@overload +def decode_fs_path(ptr: None) -> None: ... +@overload +def decode_fs_path(data: bytes) -> str: ... +def decode_fs_path(ptr_or_bytes: 'char_pointer | bytes | None') -> str | None: + if ptr_or_bytes is None: + return None + + if isinstance(ptr_or_bytes, bytes): + return os.fsdecode(ptr_or_bytes) + + if not ptr_or_bytes: + return None + + return os.fsdecode(ffi.string(ptr_or_bytes)) + + +@overload +def encode_fs_path( + s: str | bytes | os.PathLike[str] | os.PathLike[bytes], +) -> bytes: ... +@overload +def encode_fs_path(s: Union['ffi.NULL_TYPE', None]) -> Union['ffi.NULL_TYPE']: ... +def encode_fs_path( + s: Union[str, bytes, 'ffi.NULL_TYPE', os.PathLike[str], os.PathLike[bytes], None], +) -> Union[bytes, 'ffi.NULL_TYPE']: + if s == ffi.NULL or s is None: + return ffi.NULL + + if hasattr(s, '__fspath__'): + return os.fsencode(s) + + if isinstance(s, bytes): + return s + + return os.fsencode(s) + + @overload def to_bytes( s: str | bytes | os.PathLike[str] | os.PathLike[bytes], diff --git a/src/diff.c b/src/diff.c index 2ae542a7..b580737f 100644 --- a/src/diff.c +++ b/src/diff.c @@ -228,9 +228,19 @@ DiffFile_mode__get__(DiffFile *self) return pygit2_enum(FileModeEnum, self->mode); } +PyDoc_STRVAR(DiffFile_path__doc__, "Path to the entry."); + +PyObject * +DiffFile_path__get__(DiffFile *self) +{ + if (self->path == NULL) + Py_RETURN_NONE; + + return PyUnicode_DecodeFSDefault(self->path); +} + PyMemberDef DiffFile_members[] = { MEMBER(DiffFile, id, T_OBJECT, "Oid of the item."), - MEMBER(DiffFile, path, T_STRING, "Path to the entry."), MEMBER(DiffFile, raw_path, T_OBJECT, "Path to the entry (bytes)."), MEMBER(DiffFile, size, T_LONG, "Size of the entry."), {NULL} @@ -244,6 +254,7 @@ static PyMethodDef DiffFile_methods[] = { PyGetSetDef DiffFile_getsetters[] = { GETTER(DiffFile, flags), GETTER(DiffFile, mode), + GETTER(DiffFile, path), {NULL}, }; diff --git a/src/repository.c b/src/repository.c index 2276eb77..2057569e 100644 --- a/src/repository.c +++ b/src/repository.c @@ -1797,7 +1797,12 @@ Repository_status(Repository *self, PyObject *args, PyObject *kw) if (status == NULL) goto error; - err = PyDict_SetItemString(dict, path, status); + PyObject *py_path = PyUnicode_DecodeFSDefault(path); + if (py_path == NULL) + goto error; + + err = PyDict_SetItem(dict, py_path, status); + Py_DECREF(py_path); Py_CLEAR(status); if (err < 0) diff --git a/test/test_nonunicode.py b/test/test_nonunicode.py index 04d3d8a8..76cfb7a0 100644 --- a/test/test_nonunicode.py +++ b/test/test_nonunicode.py @@ -25,6 +25,7 @@ """Tests for non unicode byte strings""" +import os import shutil import sys from pathlib import Path @@ -33,6 +34,8 @@ import pygit2 from pygit2 import Repository +from pygit2.enums import FileStatus +from pygit2.utils import encode_fs_path from . import utils @@ -57,3 +60,16 @@ def test_nonunicode_branchname(testrepo: Repository, tmp_path: Path) -> None: (ref.split('/')[-1]).encode('utf8', 'surrogateescape') for ref in newrepo.listall_references() ] # Remote branch among references: 'refs/remotes/origin/\udcc3master' + + +@works_in_linux +def test_nonunicode_status_path(tmp_path: Path) -> None: + repo = pygit2.init_repository(str(tmp_path / 'repo'), bare=False) + path_bytes = 'éléphant'.encode('latin1') + filepath = Path(repo.workdir) / path_bytes.decode('utf-8', 'surrogateescape') + filepath.write_bytes(b'dummy') + git_status = repo.status() + path_key = os.fsdecode(path_bytes) + assert path_bytes in [encode_fs_path(path) for path in git_status] + assert git_status[path_key] & FileStatus.WT_NEW + assert encode_fs_path(path_key) == path_bytes