diff --git a/conan/internal/api/export.py b/conan/internal/api/export.py index d8d4e3a0bb4..0e55711f90d 100644 --- a/conan/internal/api/export.py +++ b/conan/internal/api/export.py @@ -2,7 +2,7 @@ import shutil from conan.tools.files import copy -from conan.api.output import ConanOutput, Color +from conan.api.output import ConanOutput from conan.tools.scm import Git from conan.internal.errors import conanfile_exception_formatter from conan.errors import ConanException @@ -159,8 +159,8 @@ def _export_source(conanfile, destination_source_folder): conanfile.exports_sources = (conanfile.exports_sources,) included_sources, excluded_sources = _classify_patterns(conanfile.exports_sources) - for pattern in included_sources: - copy(conanfile, pattern, src=conanfile.recipe_folder, + if included_sources: + copy(conanfile, included_sources, src=conanfile.recipe_folder, dst=destination_source_folder, excludes=excluded_sources) conanfile.folders.set_base_export_sources(destination_source_folder) @@ -184,9 +184,9 @@ def _export_recipe(conanfile, destination_folder): included_exports, excluded_exports = _classify_patterns(conanfile.exports) - for pattern in included_exports: - copy(conanfile, pattern, conanfile.recipe_folder, destination_folder, - excludes=excluded_exports) + if included_exports: + copy(conanfile, included_exports, conanfile.recipe_folder, + destination_folder, excludes=excluded_exports) conanfile.folders.set_base_export(destination_folder) _run_method(conanfile, "export") diff --git a/conan/tools/files/copy_pattern.py b/conan/tools/files/copy_pattern.py index f1640264fb6..c8c674330a3 100644 --- a/conan/tools/files/copy_pattern.py +++ b/conan/tools/files/copy_pattern.py @@ -13,8 +13,10 @@ def copy(conanfile, pattern, src, dst, keep_path=True, excludes=None, Copy the files matching the pattern (fnmatch) at the src folder to a dst folder. :param conanfile: The current recipe object. Always use ``self``. - :param pattern: (Required) An fnmatch file pattern of the files that should be copied. - It must not start with ``..`` relative path or an exception will be raised. + :param pattern: (Required) An fnmatch file pattern, or a list/tuple of fnmatch patterns, + of the files that should be copied. When a list is given the src folder is walked + only once and files matching any of the patterns are copied. Patterns must not + start with ``..`` relative path or an exception will be raised. :param src: (Required) Source folder in which those files will be searched. This folder will be stripped from the dst parameter. E.g., lib/Debug/x86. :param dst: (Required) Destination local folder. It must be different from src value or an @@ -31,17 +33,21 @@ def copy(conanfile, pattern, src, dst, keep_path=True, excludes=None, different modification time) :return: list of copied files """ - if src == dst: - raise ConanException("copy() 'src' and 'dst' arguments must have different values") - if pattern.startswith(".."): - raise ConanException("copy() it is not possible to use relative patterns starting with '..'") if src is None: raise ConanException("copy() received 'src=None' argument") + if src == dst: + raise ConanException("copy() 'src' and 'dst' arguments must have different values") + + patterns = [pattern] if isinstance(pattern, str) else list(pattern) + for p in patterns: + if p.startswith(".."): + raise ConanException("copy() it is not possible to use relative " + "patterns starting with '..'") # This is necessary to add the trailing / so it is not reported as symlink src = os.path.join(src, "") excluded_folder = dst - files_to_copy, files_symlinked_to_folders = _filter_files(src, pattern, excludes, ignore_case, + files_to_copy, files_symlinked_to_folders = _filter_files(src, patterns, excludes, ignore_case, excluded_folder) copied_files = _copy_files(files_to_copy, src, dst, keep_path, overwrite_equal) @@ -55,14 +61,15 @@ def copy(conanfile, pattern, src, dst, keep_path=True, excludes=None, return copied_files -def _filter_files(src, pattern, excludes, ignore_case, excluded_folder): - """ return a list of the files matching the patterns - The list will be relative path names wrt to the root src folder +def _filter_files(src, patterns, excludes, ignore_case, excluded_folder): + """Walk src once and return files matched by any of the patterns (minus excludes). + The returned paths are relative to src. """ filenames = [] files_symlinked_to_folders = [] - pattern = pattern.lower() if ignore_case else pattern + if ignore_case: + patterns = [p.lower() for p in patterns] if excludes: if not isinstance(excludes, (tuple, list)): excludes = (excludes, ) @@ -81,7 +88,8 @@ def _filter_files(src, pattern, excludes, ignore_case, excluded_folder): if os.path.islink(os.path.join(root, subfolder)): relative_path = os.path.relpath(os.path.join(root, subfolder), src) compare_relative_path = relative_path.lower() if ignore_case else relative_path - if fnmatch.fnmatch(os.path.normpath(compare_relative_path), pattern): + if any(fnmatch.fnmatch(os.path.normpath(compare_relative_path), p) + for p in patterns): files_symlinked_to_folders.append(relative_path) relative_path = os.path.relpath(root, src) @@ -98,11 +106,13 @@ def _filter_files(src, pattern, excludes, ignore_case, excluded_folder): filenames.append(relative_name) if ignore_case: - files_to_copy = [n for n in filenames if fnmatch.fnmatch(os.path.normpath(n.lower()), - pattern)] + files_to_copy = [n for n in filenames + if any(fnmatch.fnmatch(os.path.normpath(n.lower()), p) + for p in patterns)] else: - files_to_copy = [n for n in filenames if fnmatch.fnmatchcase(os.path.normpath(n), - pattern)] + files_to_copy = [n for n in filenames + if any(fnmatch.fnmatchcase(os.path.normpath(n), p) + for p in patterns)] for exclude in excludes: if ignore_case: diff --git a/test/integration/command/export/export_sources_test.py b/test/integration/command/export/export_sources_test.py index 9d4a94056c2..37c51c1d76d 100644 --- a/test/integration/command/export/export_sources_test.py +++ b/test/integration/command/export/export_sources_test.py @@ -60,6 +60,44 @@ def assert_files(folder, files): assert_files(ref_layout.export_sources(), ['hello.h']) +def test_exports_sources_multiple_patterns_single_scan(): + """Multiple include patterns and excludes must yield the union of matches minus excludes, + and must not require re-walking the tree per pattern (see #18981). + """ + conanfile = textwrap.dedent(""" + from conan import ConanFile + + class HelloConan(ConanFile): + name = "hello" + version = "0.1" + exports_sources = "*.h", "src/*.cpp", "docs/*.md", "!docs/private.md" + """) + c = TestClient(light=True) + c.save({"conanfile.py": conanfile, + "hello.h": "hello", + "other.h": "other", + "src/lib.cpp": "lib", + "src/util.cpp": "util", + "docs/readme.md": "readme", + "docs/private.md": "secret", + "unmatched.txt": "nope"}) + c.run("create .") + ref = RecipeReference.loads("hello/0.1") + ref_layout = c.get_latest_ref_layout(ref) + + exported = [] + for root, _, files in os.walk(ref_layout.export_sources()): + for f in files: + rel = os.path.relpath(os.path.join(root, f), ref_layout.export_sources()) + exported.append(rel.replace(os.sep, "/")) + + assert sorted(exported) == sorted([ + "hello.h", "other.h", + "src/lib.cpp", "src/util.cpp", + "docs/readme.md", + ]) + + def test_test_package_copied(): """The exclusion of the test_package folder have been removed so now we test that indeed is exported""" diff --git a/test/unittests/tools/files/test_tool_copy.py b/test/unittests/tools/files/test_tool_copy.py index 7c93359015a..adcfc9bc067 100644 --- a/test/unittests/tools/files/test_tool_copy.py +++ b/test/unittests/tools/files/test_tool_copy.py @@ -260,6 +260,59 @@ def test_multifolder(self): copy(None, "*", src_folder2, dst_folder) assert ['file1.txt', 'file2.txt'] == sorted(os.listdir(dst_folder)) + def test_multiple_patterns(self): + src_folder = temp_folder() + save(os.path.join(src_folder, "hello.h"), "h") + save(os.path.join(src_folder, "src/lib.cpp"), "cpp") + save(os.path.join(src_folder, "src/util.cpp"), "cpp") + save(os.path.join(src_folder, "docs/readme.md"), "md") + save(os.path.join(src_folder, "docs/private.md"), "secret") + save(os.path.join(src_folder, "unmatched.txt"), "nope") + + dst_folder = temp_folder() + copied = copy(None, ["*.h", "src/*.cpp", "docs/*.md"], src_folder, dst_folder, + excludes=["*/private.md"]) + rels = sorted(os.path.relpath(f, dst_folder).replace(os.sep, "/") for f in copied) + assert rels == ["docs/readme.md", "hello.h", "src/lib.cpp", "src/util.cpp"] + + @mock.patch('shutil.copy2') + def test_multiple_patterns_dedup(self, copy2_mock): + # Files matched by more than one pattern must only be copied once + src_folder = temp_folder() + save(os.path.join(src_folder, "a.h"), "x") + save(os.path.join(src_folder, "b.h"), "x") + dst_folder = temp_folder() + + copy(None, ["*.h", "a.*"], src_folder, dst_folder) + assert copy2_mock.call_count == 2 # a.h counted once, b.h counted once + + def test_multiple_patterns_single_scan(self): + # A list of patterns must walk the src tree exactly once, regardless of pattern count. + # This is the performance guarantee behind #18981. + src_folder = temp_folder() + for i in range(5): + save(os.path.join(src_folder, f"dir{i}/file.h"), "h") + save(os.path.join(src_folder, f"dir{i}/file.cpp"), "cpp") + dst_folder = temp_folder() + + patterns = [f"dir{i}/*.h" for i in range(5)] + [f"dir{i}/*.cpp" for i in range(5)] + + with mock.patch("conan.tools.files.copy_pattern.os.walk", + wraps=os.walk) as walk_mock: + copy(None, patterns, src_folder, dst_folder) + single_scan_calls = walk_mock.call_count + + # Baseline: calling copy() once per pattern would walk once per pattern + dst_folder2 = temp_folder() + with mock.patch("conan.tools.files.copy_pattern.os.walk", + wraps=os.walk) as walk_mock: + for p in patterns: + copy(None, p, src_folder, dst_folder2) + loop_calls = walk_mock.call_count + + assert single_scan_calls == 1 + assert loop_calls == len(patterns) + @mock.patch('shutil.copy2') def test_avoid_repeat_copies(self, copy2_mock): src_folders = [temp_folder() for _ in range(2)]