diff --git a/java/com/google/copybara/git/AddExcludedFilesToIndex.java b/java/com/google/copybara/git/AddExcludedFilesToIndex.java index 5d751d96c..63303ba2f 100644 --- a/java/com/google/copybara/git/AddExcludedFilesToIndex.java +++ b/java/com/google/copybara/git/AddExcludedFilesToIndex.java @@ -25,11 +25,13 @@ import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.PathMatcher; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.List; +import java.util.Iterator; /** * A walker which adds all files not matching a glob to the index of a Git repo using {@code git @@ -68,13 +70,18 @@ void findSubmodules(Console console) throws RepoException { /** * Adds all the excluded files and submodules. */ - void add() throws RepoException, IOException { + void add(Console console) throws RepoException, IOException { + console.progress("Git Destination: Walking Tree for Exclusions"); ExcludesFinder visitor = new ExcludesFinder(repo.getGitDir(), pathMatcher); Files.walkFileTree(repo.getWorkTree(), visitor); + console.progress("Git Destination: Compressing Tree"); + visitor.excludedTree.Compress(); + + console.progress("Git Destination: Adding Excluded Files"); int size = 0; List current = new ArrayList<>(); - for (String path : visitor.excluded) { + for (String path : visitor.excludedTree.Excluded()) { current.add(path); size += path.length(); // Split the executions in chunks of 6K. 8K triggers arg max in some systems, so @@ -89,17 +96,160 @@ void add() throws RepoException, IOException { repo.add().force().files(current).run(); } + console.progress("Git Destination: Adding submodules"); for (String addBackSubmodule : addBackSubmodules) { repo.simpleCommand("reset", "--", "--quiet", addBackSubmodule); repo.add().force().files(ImmutableList.of(addBackSubmodule)).run(); } } + /** + * A tree representation of the set of paths in the repo. + * + * Used to track which paths we're excluding, or equivalently the set of paths we're + * going to `git add` above. + * + * Each interior node is a directory, and each leaf is a path (either a directory or + * a file), with the leaves marked as included or not + */ + private static final class PathTree { + /** + * Internal representation of each leaf of the tree - the last component of its + * path, and whether it's included or not + */ + private static final class PathTreeLeaf { + private final String filename; + private boolean included; + + private PathTreeLeaf(String filename, boolean included) { + this.filename = filename; + this.included = included; + } + } + + private String dirname; + private final ArrayList kids; + private final ArrayList leaves; + + public PathTree(String dirname) { + this.dirname = dirname; + this.kids = new ArrayList<>(); + this.leaves = new ArrayList<>(); + } + + /** + * Add a new path to the tree. + * + * The new path will be added as a leaf node, and marked as included/excluded. + */ + public void AddPath(Path path, boolean included) { + // Set the root dirname lazily when we get the first path + if (dirname == null) { + dirname = path.getRoot().toString(); + } + + // Walk down the tree from the root of the tree (and the root of the path) + PathTree currTree = this; + + for (Path component : path.getParent()) { + // Do we already have an entry for this subdirectory? + boolean foundKid = false; + for (PathTree kid : currTree.kids) { + if (kid.dirname.equals(component.toString())) { + currTree = kid; + foundKid = true; + break; + } + } + + // We don't, so create it + if (!foundKid) { + PathTree newTree = new PathTree(component.toString()); + currTree.kids.add(newTree); + currTree = newTree; + } + } + + // Now add the filename to the bottom subdirectory + currTree.leaves.add(new PathTreeLeaf(path.getFileName().toString(), included)); + } + + /** + * Compresses the tree. + * + * This takes subtrees where all files are excluded and replaces them with a single + * entry for the root of the subtree. This means when we go to call `git add`, we + * can just pass the root of the subdirectory, instead of every file inside it. + * + * This is done bottom-up, recursively. Each directory that has no complex + * subdirectories under it, and where all files are excluded, can be deleted from + * its parent's `kids` list and put instead in its `leaves` list as a single entry + * for the directory. We go bottom-up, so that entire trees can be replaced this + * way. + * + * Returns `true` if the PathTree can be compressed to a single entry + */ + public boolean Compress() { + Iterator itr = kids.iterator(); + while (itr.hasNext()) { + PathTree subTree = itr.next(); + boolean compressed = subTree.Compress(); + if (compressed) { + leaves.add(new PathTreeLeaf(subTree.dirname, false)); + itr.remove(); + } + } + + if (!kids.isEmpty()) { + return false; + } + + for (PathTreeLeaf leaf : leaves) { + if (leaf.included) { + return false; + } + } + + return true; + } + + /** + * Helper to recursively add the excluded paths in this tree to the list `results`. + * + * Each path is prepended with the string `prefix`, i.e. the path from the root to + * this PathTree + */ + private void AddExcluded(ArrayList results, String prefix) { + for (PathTree kid : kids) { + String childPath = prefix == null ? dirname : Paths.get(prefix, dirname).toString(); + kid.AddExcluded(results, childPath); + } + + for (PathTreeLeaf leaf : leaves) { + if (!leaf.included) { + String childPath = prefix == null + ? Paths.get(dirname, leaf.filename).toString() + : Paths.get(prefix, dirname, leaf.filename).toString(); + results.add(childPath); + } + } + } + + /** + * Return the list of excluded paths in this tree. + */ + public ArrayList Excluded() { + ArrayList result = new ArrayList<>(); + AddExcluded(result, dirname); + return result; + } + } + private static final class ExcludesFinder extends SimpleFileVisitor { private final Path gitDir; private final PathMatcher destinationFiles; - private final List excluded = new ArrayList<>(); + private final PathTree excludedTree = new PathTree(null); private ExcludesFinder(Path gitDir, PathMatcher destinationFiles) { this.gitDir = gitDir; @@ -117,11 +267,8 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - if (!destinationFiles.matches(file)) { - excluded.add(file.toString()); - } + excludedTree.AddPath(file, destinationFiles.matches(file)); return FileVisitResult.CONTINUE; } - } } diff --git a/java/com/google/copybara/git/GitDestination.java b/java/com/google/copybara/git/GitDestination.java index 7f7d203fe..b02b41c13 100644 --- a/java/com/google/copybara/git/GitDestination.java +++ b/java/com/google/copybara/git/GitDestination.java @@ -539,7 +539,7 @@ public ImmutableList write(TransformResult transformResult, console.progress("Git Destination: Excluding files"); try (ProfilerTask ignored = generalOptions.profiler().start("exclude_files")) { - excludedAdder.add(); + excludedAdder.add(console); } console.progress("Git Destination: Creating a local commit");