From 0e3d9854ddeb1155302747bf980bb061748c5e72 Mon Sep 17 00:00:00 2001 From: piameier Date: Thu, 24 Jul 2025 11:24:38 +0200 Subject: [PATCH 001/126] fixed: missing toString() in VariationLabel --- .../variantsync/diffdetective/variation/VariationLabel.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/variantsync/diffdetective/variation/VariationLabel.java b/src/main/java/org/variantsync/diffdetective/variation/VariationLabel.java index 7373a04d2..1709dfa95 100644 --- a/src/main/java/org/variantsync/diffdetective/variation/VariationLabel.java +++ b/src/main/java/org/variantsync/diffdetective/variation/VariationLabel.java @@ -49,4 +49,9 @@ public NodeType getNodeType() { public VariationLabel clone() { return new VariationLabel(type, Cast.unchecked(innerLabel.clone())); } + + @Override + public String toString() { + return innerLabel.toString(); + } } From afd99b47635003d52b764a073e828942c2229aa8 Mon Sep 17 00:00:00 2001 From: eshul Date: Sat, 26 Oct 2024 16:49:01 +0200 Subject: [PATCH 002/126] feat: prepare `DiffNode` for storing `endif` lines --- .../diffdetective/variation/diff/DiffNode.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/main/java/org/variantsync/diffdetective/variation/diff/DiffNode.java b/src/main/java/org/variantsync/diffdetective/variation/diff/DiffNode.java index 4244fadfe..5dd4010a9 100644 --- a/src/main/java/org/variantsync/diffdetective/variation/diff/DiffNode.java +++ b/src/main/java/org/variantsync/diffdetective/variation/diff/DiffNode.java @@ -51,6 +51,8 @@ public class DiffNode implements HasNodeType { private Node featureMapping; + private String endIf = null; + /** * The parents {@link DiffNode} before and after the edit. * This array has to be indexed by {@code Time.ordinal()} @@ -146,6 +148,22 @@ public void setLabel(L newLabel) { label.setInnerLabel(newLabel); } + /** + * Returns the line with the endif of the corresponding if, if the node is an if node, otherwise null + * @return String, the Line with endif + */ + public String getEndIf() { + return endIf; + } + + /** + * Sets the line with the endif of the corresponding if, if the node is an if node + * @param endIf String, the Line with endif + */ + public void setEndIf(String endIf) { + this.endIf = endIf; + } + /** * Gets the first {@code if} node in the path from the root to this node at the time * {@code time}. From 4bed4b555cfb21f6ba42ac16eddef04146de2dfd Mon Sep 17 00:00:00 2001 From: eshul Date: Sat, 26 Oct 2024 20:04:29 +0200 Subject: [PATCH 003/126] feat: prepare `VariationTreeNode` for storing `endif` lines --- .../variation/tree/VariationTreeNode.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/main/java/org/variantsync/diffdetective/variation/tree/VariationTreeNode.java b/src/main/java/org/variantsync/diffdetective/variation/tree/VariationTreeNode.java index 0609a8fd1..6ecbac595 100644 --- a/src/main/java/org/variantsync/diffdetective/variation/tree/VariationTreeNode.java +++ b/src/main/java/org/variantsync/diffdetective/variation/tree/VariationTreeNode.java @@ -82,6 +82,11 @@ public class VariationTreeNode extends VariationNode> childOrder; + /** + * The line with the endif of the corresponding if, if the node is an if node, otherwise null + */ + private String endIf = null; + /** * Creates a new node of a variation tree. * @@ -226,6 +231,22 @@ public void removeAllChildren() { childOrder.clear(); } + /** + * Returns the line with the endif of the corresponding if, if the node is an if node, otherwise null + * @return String, the Line with endif + */ + public String getEndIf() { + return endIf; + } + + /** + * Sets the line with the endif of the corresponding if, if the node is an if node + * @param endIf String, the Line with endif + */ + public void setEndIf(String endIf) { + this.endIf = endIf; + } + @Override public Node getFormula() { return featureMapping; From 22d5dea06fbdbe310c41295c3943e1603e38e520 Mon Sep 17 00:00:00 2001 From: eshul Date: Sat, 26 Oct 2024 20:21:07 +0200 Subject: [PATCH 004/126] feat: add a getter for the `endif` line in `VariationNode` --- .../diffdetective/variation/diff/Projection.java | 5 +++++ .../diffdetective/variation/tree/VariationNode.java | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/src/main/java/org/variantsync/diffdetective/variation/diff/Projection.java b/src/main/java/org/variantsync/diffdetective/variation/diff/Projection.java index 20e476f7e..619bafb38 100644 --- a/src/main/java/org/variantsync/diffdetective/variation/diff/Projection.java +++ b/src/main/java/org/variantsync/diffdetective/variation/diff/Projection.java @@ -121,4 +121,9 @@ public Node getFormula() { public int getID() { return getBackingNode().getID(); } + + @Override + public String getEndIf() { + return getBackingNode().getEndIf(); + } }; diff --git a/src/main/java/org/variantsync/diffdetective/variation/tree/VariationNode.java b/src/main/java/org/variantsync/diffdetective/variation/tree/VariationNode.java index d1622423f..fccbccf55 100644 --- a/src/main/java/org/variantsync/diffdetective/variation/tree/VariationNode.java +++ b/src/main/java/org/variantsync/diffdetective/variation/tree/VariationNode.java @@ -113,6 +113,12 @@ public VariationNode downCast() { */ public abstract List getChildren(); + /** + * Returns the line with the endif of the corresponding if, if the node is an if node, otherwise null + * @return String, the Line with endif + */ + public abstract String getEndIf(); + /** * Returns {@code true} iff this node has no parent. * From b694ce56d78f4fecb88e06eed44c79e7c2621f88 Mon Sep 17 00:00:00 2001 From: eshul Date: Sat, 26 Oct 2024 20:24:55 +0200 Subject: [PATCH 005/126] fix: handle `endif` lines in deep copies of variation trees --- .../diffdetective/variation/tree/VariationNode.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/variantsync/diffdetective/variation/tree/VariationNode.java b/src/main/java/org/variantsync/diffdetective/variation/tree/VariationNode.java index fccbccf55..d067c2201 100644 --- a/src/main/java/org/variantsync/diffdetective/variation/tree/VariationNode.java +++ b/src/main/java/org/variantsync/diffdetective/variation/tree/VariationNode.java @@ -483,6 +483,9 @@ public VariationTreeNode toVariationTree(final Map Date: Mon, 28 Oct 2024 17:37:40 +0100 Subject: [PATCH 006/126] feat: create an unparser for variation trees --- .../variation/VariationUnparser.java | 33 ++++++++ src/test/java/VariationUnparserTest.java | 78 +++++++++++++++++++ src/test/resources/unparser/test1.txt | 0 src/test/resources/unparser/test2.txt | 3 + 4 files changed, 114 insertions(+) create mode 100644 src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java create mode 100644 src/test/java/VariationUnparserTest.java create mode 100644 src/test/resources/unparser/test1.txt create mode 100644 src/test/resources/unparser/test2.txt diff --git a/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java b/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java new file mode 100644 index 000000000..546560622 --- /dev/null +++ b/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java @@ -0,0 +1,33 @@ +package org.variantsync.diffdetective.variation; + +import java.util.ArrayList; +import java.util.Stack; +import org.variantsync.diffdetective.variation.tree.VariationTree; +import org.variantsync.diffdetective.variation.tree.VariationTreeNode; + +public class VariationUnparser { + public static String variationTreeUnparser(VariationTree tree) { + StringBuilder result = new StringBuilder(); + Stack> stack = new Stack<>(); + for (int i = tree.root().getChildren().size() - 1; i >= 0; i--) { + stack.push(tree.root().getChildren().get(i)); + } + while (!stack.empty()) { + VariationTreeNode node = stack.pop(); + for (String line : node.getLabel().getLines()) { + result.append(line); + result.append("\n"); + } + if (node.isIf()) { + ArrayList list = new ArrayList<>(); + list.add(node.getEndIf()); + stack.push(new VariationTreeNode<>(NodeType.ARTIFACT, null, null, + DiffLinesLabel.withInvalidLineNumbers(list))); + } + for (int i = node.getChildren().size() - 1; i >= 0; i--) { + stack.push(node.getChildren().get(i)); + } + } + return result.substring(0, result.length() - 1); + } +} diff --git a/src/test/java/VariationUnparserTest.java b/src/test/java/VariationUnparserTest.java new file mode 100644 index 000000000..914627841 --- /dev/null +++ b/src/test/java/VariationUnparserTest.java @@ -0,0 +1,78 @@ +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.variantsync.diffdetective.variation.DiffLinesLabel; +import org.variantsync.diffdetective.variation.tree.VariationTree; +import org.variantsync.diffdetective.variation.VariationUnparser; + +public class VariationUnparserTest { + private final static Path testDir = Constants.RESOURCE_DIR.resolve("unparser"); + + private final static String testCaseSuffix = ".txt"; + + protected static Stream findTestCases(Path dir) { + try { + return Files + .list(dir) + .filter(filename -> filename.getFileName().toString().endsWith(testCaseSuffix)); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + @Disabled + @Test + public void tests() { + findTestCases(testDir).forEach(this::test); + } + + @Disabled + @Test + public void teststest() { + Path path = testDir.resolve("test2.txt"); + String temp = "b"; + VariationTree tree = null; + try { + tree = VariationTree.fromFile(path); + temp = VariationUnparser.variationTreeUnparser(tree); + } catch (Exception e) { + e.printStackTrace(); + } + assertNotNull(tree.root().getChildren().get(0).getEndIf()); + } + + public void test(Path basename) { + testCase(basename); + } + + public static void testCase(Path testCasePath) { + String temp = ""; + try { + temp = Files.readString(testCasePath); + } catch (Exception e) { + e.printStackTrace(); + } + temp = temp.replaceAll("\\r\\n", "\n"); + String unparse = parseUnparse(testCasePath); + assertEquals(temp, unparse); + } + + public static String parseUnparse(Path path) { + String temp = "b"; + try { + VariationTree tree = VariationTree.fromFile(path); + temp = VariationUnparser.variationTreeUnparser(tree); + } catch (Exception e) { + e.printStackTrace(); + } + return temp; + } + +} diff --git a/src/test/resources/unparser/test1.txt b/src/test/resources/unparser/test1.txt new file mode 100644 index 000000000..e69de29bb diff --git a/src/test/resources/unparser/test2.txt b/src/test/resources/unparser/test2.txt new file mode 100644 index 000000000..4200a6eb6 --- /dev/null +++ b/src/test/resources/unparser/test2.txt @@ -0,0 +1,3 @@ +#ifdef C + boob() +#endif \ No newline at end of file From e9fb7c58ce5f1c01d8be8625248539c31b14a04e Mon Sep 17 00:00:00 2001 From: eshul Date: Wed, 30 Oct 2024 14:20:24 +0100 Subject: [PATCH 007/126] feat: store `endif` lines when parsing variation diffs --- .../variation/VariationUnparser.java | 5 +---- .../diffdetective/variation/diff/DiffNode.java | 6 +++--- .../diffdetective/variation/diff/Projection.java | 2 +- .../variation/diff/parse/VariationDiffParser.java | 15 +++++++++++++-- .../variation/tree/VariationNode.java | 2 +- .../variation/tree/VariationTreeNode.java | 6 +++--- src/test/java/VariationUnparserTest.java | 1 - 7 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java b/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java index 546560622..cc33760cc 100644 --- a/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java +++ b/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java @@ -1,6 +1,5 @@ package org.variantsync.diffdetective.variation; -import java.util.ArrayList; import java.util.Stack; import org.variantsync.diffdetective.variation.tree.VariationTree; import org.variantsync.diffdetective.variation.tree.VariationTreeNode; @@ -19,10 +18,8 @@ public static String variationTreeUnparser(VariationTree tree) result.append("\n"); } if (node.isIf()) { - ArrayList list = new ArrayList<>(); - list.add(node.getEndIf()); stack.push(new VariationTreeNode<>(NodeType.ARTIFACT, null, null, - DiffLinesLabel.withInvalidLineNumbers(list))); + DiffLinesLabel.withInvalidLineNumbers(node.getEndIf()))); } for (int i = node.getChildren().size() - 1; i >= 0; i--) { stack.push(node.getChildren().get(i)); diff --git a/src/main/java/org/variantsync/diffdetective/variation/diff/DiffNode.java b/src/main/java/org/variantsync/diffdetective/variation/diff/DiffNode.java index 5dd4010a9..a8ff8e0d0 100644 --- a/src/main/java/org/variantsync/diffdetective/variation/diff/DiffNode.java +++ b/src/main/java/org/variantsync/diffdetective/variation/diff/DiffNode.java @@ -51,7 +51,7 @@ public class DiffNode implements HasNodeType { private Node featureMapping; - private String endIf = null; + private List endIf = null; /** * The parents {@link DiffNode} before and after the edit. @@ -152,7 +152,7 @@ public void setLabel(L newLabel) { * Returns the line with the endif of the corresponding if, if the node is an if node, otherwise null * @return String, the Line with endif */ - public String getEndIf() { + public List getEndIf() { return endIf; } @@ -160,7 +160,7 @@ public String getEndIf() { * Sets the line with the endif of the corresponding if, if the node is an if node * @param endIf String, the Line with endif */ - public void setEndIf(String endIf) { + public void setEndIf(List endIf) { this.endIf = endIf; } diff --git a/src/main/java/org/variantsync/diffdetective/variation/diff/Projection.java b/src/main/java/org/variantsync/diffdetective/variation/diff/Projection.java index 619bafb38..f9ac7a02d 100644 --- a/src/main/java/org/variantsync/diffdetective/variation/diff/Projection.java +++ b/src/main/java/org/variantsync/diffdetective/variation/diff/Projection.java @@ -123,7 +123,7 @@ public int getID() { } @Override - public String getEndIf() { + public List getEndIf() { return getBackingNode().getEndIf(); } }; diff --git a/src/main/java/org/variantsync/diffdetective/variation/diff/parse/VariationDiffParser.java b/src/main/java/org/variantsync/diffdetective/variation/diff/parse/VariationDiffParser.java index 2df9c3bf3..5154bdb08 100644 --- a/src/main/java/org/variantsync/diffdetective/variation/diff/parse/VariationDiffParser.java +++ b/src/main/java/org/variantsync/diffdetective/variation/diff/parse/VariationDiffParser.java @@ -1,5 +1,7 @@ package org.variantsync.diffdetective.variation.diff.parse; +import java.util.ArrayList; +import java.util.List; import org.apache.commons.lang3.function.FailableSupplier; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.lib.ObjectId; @@ -327,7 +329,7 @@ private void parseLine( // Do not create a node for ENDIF, but update the line numbers of the closed if-chain // and remove that if-chain from the relevant stacks. diffType.forAllTimesOfExistence(beforeStack, afterStack, stack -> - popIfChain(stack, fromLine) + popIfChain(stack, fromLine, line) ); } else if (options.collapseMultipleCodeLines() && annotation.type() == AnnotationType.None @@ -361,15 +363,24 @@ private void parseLine( * * @param stack the stack which should be popped * @param elseLineNumber the first line of the else which causes this IF to be popped + * @param line the line containing the endif * @throws DiffParseException if {@code stack} doesn't contain an IF node */ private void popIfChain( Stack> stack, - DiffLineNumber elseLineNumber + DiffLineNumber elseLineNumber, + LogicalLine line ) throws DiffParseException { DiffLineNumber previousLineNumber = elseLineNumber; do { DiffNode annotation = stack.peek(); + if (annotation.isIf()) { + List list = new ArrayList<>(); + for (int i = 0; i < line.getLines().size(); i++) { + list.add(line.getLines().get(i).content()); + } + annotation.setEndIf(list); + } // Set the line number of now closed annotations to the beginning of the // following annotation. diff --git a/src/main/java/org/variantsync/diffdetective/variation/tree/VariationNode.java b/src/main/java/org/variantsync/diffdetective/variation/tree/VariationNode.java index d067c2201..0b9ceed1e 100644 --- a/src/main/java/org/variantsync/diffdetective/variation/tree/VariationNode.java +++ b/src/main/java/org/variantsync/diffdetective/variation/tree/VariationNode.java @@ -117,7 +117,7 @@ public VariationNode downCast() { * Returns the line with the endif of the corresponding if, if the node is an if node, otherwise null * @return String, the Line with endif */ - public abstract String getEndIf(); + public abstract List getEndIf(); /** * Returns {@code true} iff this node has no parent. diff --git a/src/main/java/org/variantsync/diffdetective/variation/tree/VariationTreeNode.java b/src/main/java/org/variantsync/diffdetective/variation/tree/VariationTreeNode.java index 6ecbac595..d26bf3b68 100644 --- a/src/main/java/org/variantsync/diffdetective/variation/tree/VariationTreeNode.java +++ b/src/main/java/org/variantsync/diffdetective/variation/tree/VariationTreeNode.java @@ -85,7 +85,7 @@ public class VariationTreeNode extends VariationNode endIf = null; /** * Creates a new node of a variation tree. @@ -235,7 +235,7 @@ public void removeAllChildren() { * Returns the line with the endif of the corresponding if, if the node is an if node, otherwise null * @return String, the Line with endif */ - public String getEndIf() { + public List getEndIf() { return endIf; } @@ -243,7 +243,7 @@ public String getEndIf() { * Sets the line with the endif of the corresponding if, if the node is an if node * @param endIf String, the Line with endif */ - public void setEndIf(String endIf) { + public void setEndIf(List endIf) { this.endIf = endIf; } diff --git a/src/test/java/VariationUnparserTest.java b/src/test/java/VariationUnparserTest.java index 914627841..7f62dd495 100644 --- a/src/test/java/VariationUnparserTest.java +++ b/src/test/java/VariationUnparserTest.java @@ -33,7 +33,6 @@ public void tests() { findTestCases(testDir).forEach(this::test); } - @Disabled @Test public void teststest() { Path path = testDir.resolve("test2.txt"); From 92b92473348b6efc5d6969c73c53ce5dd92fb6a0 Mon Sep 17 00:00:00 2001 From: eshul Date: Fri, 1 Nov 2024 20:15:30 +0100 Subject: [PATCH 008/126] feat: create an unparser for variation diffs --- .../variation/VariationUnparser.java | 91 +++++++++++++++---- 1 file changed, 73 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java b/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java index cc33760cc..48958fe47 100644 --- a/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java +++ b/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java @@ -1,30 +1,85 @@ package org.variantsync.diffdetective.variation; +import java.io.IOException; +import java.util.List; import java.util.Stack; +import java.util.function.Function; +import org.eclipse.jgit.diff.DiffAlgorithm.SupportedAlgorithm; +import org.variantsync.diffdetective.variation.diff.Time; +import org.variantsync.diffdetective.variation.diff.VariationDiff; +import org.variantsync.diffdetective.variation.diff.construction.JGitDiff; import org.variantsync.diffdetective.variation.tree.VariationTree; import org.variantsync.diffdetective.variation.tree.VariationTreeNode; public class VariationUnparser { - public static String variationTreeUnparser(VariationTree tree) { - StringBuilder result = new StringBuilder(); - Stack> stack = new Stack<>(); - for (int i = tree.root().getChildren().size() - 1; i >= 0; i--) { - stack.push(tree.root().getChildren().get(i)); - } - while (!stack.empty()) { - VariationTreeNode node = stack.pop(); - for (String line : node.getLabel().getLines()) { - result.append(line); - result.append("\n"); - } - if (node.isIf()) { - stack.push(new VariationTreeNode<>(NodeType.ARTIFACT, null, null, - DiffLinesLabel.withInvalidLineNumbers(node.getEndIf()))); + /** + * Unparse VariationTrees to Text/String + * + * @param tree VariationTree, that be unparsed + * @param linesToLabel Function, that return list of String and has a Class T + * @return String, the result of unparsing + * @param that implements Label + */ + public static String variationTreeUnparser(VariationTree tree, Function, T> linesToLabel) { + if (!tree.root().getChildren().isEmpty()) { + StringBuilder result = new StringBuilder(); + Stack> stack = new Stack<>(); + for (int i = tree.root().getChildren().size() - 1; i >= 0; i--) { + stack.push(tree.root().getChildren().get(i)); } - for (int i = node.getChildren().size() - 1; i >= 0; i--) { - stack.push(node.getChildren().get(i)); + while (!stack.empty()) { + VariationTreeNode node = stack.pop(); + for (String line : node.getLabel().getLines()) { + result.append(line); + result.append("\n"); + } + if (node.isIf()) { + stack.push(new VariationTreeNode<>(NodeType.ARTIFACT, null, null, + linesToLabel.apply(node.getEndIf()))); + } + for (int i = node.getChildren().size() - 1; i >= 0; i--) { + stack.push(node.getChildren().get(i)); + } } + return result.substring(0, result.length() - 1); + } else { + return ""; } - return result.substring(0, result.length() - 1); + } + + /** + * Unparse VariationTrees to Text/String + * + * @param tree VariationTree, that be unparsed + * @return String, the result of unparsing + */ + public static String variationTreeUnparser(VariationTree tree) { + return variationTreeUnparser(tree, DiffLinesLabel::withInvalidLineNumbers); + } + + /** + * Unparse VariationDiffs to Text/String + * + * @param diff VariationDiff, that be unparsed + * @param linesToLabel Function, that return list of String and has a Class T + * @return String, the result of unparsing + * @param that implements Label + * @throws IOException + */ + public static String variationDiffUnparser(VariationDiff diff, Function, T> linesToLabel) throws IOException { + String tree1 = variationTreeUnparser(diff.project(Time.BEFORE), linesToLabel); + String tree2 = variationTreeUnparser(diff.project(Time.AFTER), linesToLabel); + return JGitDiff.textDiff(tree1, tree2, SupportedAlgorithm.MYERS); + } + + /** + * Unparse VariationDiffs to Text/String + * + * @param diff VariationDiff, that be unparsed + * @return String, the result of unparsing + * @throws IOException + */ + public static String variationDiffUnparser(VariationDiff diff) throws IOException { + return variationDiffUnparser(diff, DiffLinesLabel::withInvalidLineNumbers); } } From e2934fee7fdb9baa48b538ef122560a38fa0c2d1 Mon Sep 17 00:00:00 2001 From: eshul Date: Sat, 2 Nov 2024 18:50:13 +0100 Subject: [PATCH 009/126] feat: create a method that projects text diffs --- .../variation/VariationUnparser.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java b/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java index 48958fe47..090e64496 100644 --- a/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java +++ b/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java @@ -82,4 +82,26 @@ public static String variationDiffUnparser(VariationDiff di public static String variationDiffUnparser(VariationDiff diff) throws IOException { return variationDiffUnparser(diff, DiffLinesLabel::withInvalidLineNumbers); } + + public static String undiff(String text, Time time) { + StringBuilder result = new StringBuilder(); + String[] textSplit = text.split("\n"); + if (Time.AFTER == time) { + for (String line : textSplit) { + if (line.charAt(0) != '-') { + result.append(line.substring(1)); + result.append("\n"); + } + } + return result.substring(0, result.length() - 1); + } else { + for (String line : textSplit) { + if (line.charAt(0) != '+') { + result.append(line.substring(1)); + result.append("\n"); + } + } + return result.substring(0, result.length() - 1); + } + } } From 2c1827901c5381f0af2b29ce07a656a4cfefc83b Mon Sep 17 00:00:00 2001 From: eshul Date: Mon, 4 Nov 2024 14:37:14 +0100 Subject: [PATCH 010/126] refactor: move some code for readability --- .../diffdetective/variation/VariationUnparser.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java b/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java index 090e64496..3919328cb 100644 --- a/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java +++ b/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java @@ -29,14 +29,14 @@ public static String variationTreeUnparser(VariationTree tr } while (!stack.empty()) { VariationTreeNode node = stack.pop(); - for (String line : node.getLabel().getLines()) { - result.append(line); - result.append("\n"); - } if (node.isIf()) { stack.push(new VariationTreeNode<>(NodeType.ARTIFACT, null, null, linesToLabel.apply(node.getEndIf()))); } + for (String line : node.getLabel().getLines()) { + result.append(line); + result.append("\n"); + } for (int i = node.getChildren().size() - 1; i >= 0; i--) { stack.push(node.getChildren().get(i)); } From a8f530615ed8fe05583ff7954f34b323d51dc35e Mon Sep 17 00:00:00 2001 From: eshul Date: Thu, 7 Nov 2024 16:46:35 +0100 Subject: [PATCH 011/126] test: create tests for VariationUnparser --- src/test/java/VariationUnparserTest.java | 131 +++++++++++++++++------ src/test/resources/unparser/test3.txt | 11 ++ src/test/resources/unparser/test4.txt | 8 ++ src/test/resources/unparser/test5.txt | 5 + src/test/resources/unparser/test6.txt | 5 + src/test/resources/unparser/test7.txt | 9 ++ src/test/resources/unparser/test8.txt | 11 ++ 7 files changed, 150 insertions(+), 30 deletions(-) create mode 100644 src/test/resources/unparser/test3.txt create mode 100644 src/test/resources/unparser/test4.txt create mode 100644 src/test/resources/unparser/test5.txt create mode 100644 src/test/resources/unparser/test6.txt create mode 100644 src/test/resources/unparser/test7.txt create mode 100644 src/test/resources/unparser/test8.txt diff --git a/src/test/java/VariationUnparserTest.java b/src/test/java/VariationUnparserTest.java index 7f62dd495..e0dd2c40a 100644 --- a/src/test/java/VariationUnparserTest.java +++ b/src/test/java/VariationUnparserTest.java @@ -1,22 +1,26 @@ -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.stream.Stream; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.variantsync.diffdetective.diff.result.DiffParseException; import org.variantsync.diffdetective.variation.DiffLinesLabel; +import org.variantsync.diffdetective.variation.diff.Time; +import org.variantsync.diffdetective.variation.diff.VariationDiff; +import org.variantsync.diffdetective.variation.diff.parse.VariationDiffParseOptions; import org.variantsync.diffdetective.variation.tree.VariationTree; import org.variantsync.diffdetective.variation.VariationUnparser; public class VariationUnparserTest { - private final static Path testDir = Constants.RESOURCE_DIR.resolve("unparser"); + private final static Path testDirTree = Constants.RESOURCE_DIR.resolve("unparser"); - private final static String testCaseSuffix = ".txt"; + private final static Path testDirDiff = Constants.RESOURCE_DIR.resolve("diffs").resolve("parser"); + private final static String testCaseSuffixTree = ".txt"; - protected static Stream findTestCases(Path dir) { + private final static String testCaseSuffixDiff = ".diff"; + + protected static Stream findTestCases(Path dir, String testCaseSuffix) { try { return Files .list(dir) @@ -27,46 +31,99 @@ protected static Stream findTestCases(Path dir) { return null; } - @Disabled - @Test - public void tests() { - findTestCases(testDir).forEach(this::test); + public static Stream testsTree() throws IOException { + return findTestCases(testDirTree, testCaseSuffixTree); } - @Test - public void teststest() { - Path path = testDir.resolve("test2.txt"); - String temp = "b"; - VariationTree tree = null; + public static Stream testsDiff() throws IOException { + return findTestCases(testDirDiff, testCaseSuffixDiff); + } + + @ParameterizedTest + @MethodSource("testsTree") + public void testTree(Path basename) throws IOException, DiffParseException { + testCaseTree(basename); + } + + @ParameterizedTest + @MethodSource("testsDiff") + public void testDiff(Path basename) throws IOException, DiffParseException { + testCaseDiff(basename); + } + + public static void testCaseTree(Path testCasePath) { + String temp = ""; try { - tree = VariationTree.fromFile(path); - temp = VariationUnparser.variationTreeUnparser(tree); + temp = Files.readString(testCasePath); } catch (Exception e) { e.printStackTrace(); } - assertNotNull(tree.root().getChildren().get(0).getEndIf()); - } - - public void test(Path basename) { - testCase(basename); + System.out.println(testCasePath); + temp = temp.replaceAll("\\r\\n", "\n"); + String unparse1 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(false, false)); + String unparse2 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(false, true)); + String unparse3 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(true, false)); + String unparse4 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(true, true)); + System.out.println(temp.equals(unparse1) + " " + temp.equals(unparse2) + " " + temp.equals(unparse3) + " " + + temp.equals(unparse4)); + temp = removeWhitespace(temp); + unparse1 = removeWhitespace(unparse1); + unparse2 = removeWhitespace(unparse2); + unparse3 = removeWhitespace(unparse3); + unparse4 = removeWhitespace(unparse4); + System.out.println(temp.equals(unparse1) + " " + temp.equals(unparse2) + " " + temp.equals(unparse3) + " " + + temp.equals(unparse4)); } - public static void testCase(Path testCasePath) { + public static void testCaseDiff(Path testCasePath) { String temp = ""; try { temp = Files.readString(testCasePath); } catch (Exception e) { e.printStackTrace(); } + System.out.println(testCasePath); temp = temp.replaceAll("\\r\\n", "\n"); - String unparse = parseUnparse(testCasePath); - assertEquals(temp, unparse); + String unparse1 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(false, false)); + String unparse2 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(false, true)); + String unparse3 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(true, false)); + String unparse4 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(true, true)); + System.out.println(temp.equals(unparse1) + " " + temp.equals(unparse2) + " " + temp.equals(unparse3) + " " + + temp.equals(unparse4)); + String temp1 = VariationUnparser.undiff(temp, Time.BEFORE); + String temp2 = VariationUnparser.undiff(temp, Time.AFTER); + String unparse11 = VariationUnparser.undiff(unparse1, Time.BEFORE); + String unparse12 = VariationUnparser.undiff(unparse1, Time.AFTER); + String unparse21 = VariationUnparser.undiff(unparse2, Time.BEFORE); + String unparse22 = VariationUnparser.undiff(unparse2, Time.AFTER); + String unparse31 = VariationUnparser.undiff(unparse3, Time.BEFORE); + String unparse32 = VariationUnparser.undiff(unparse3, Time.AFTER); + String unparse41 = VariationUnparser.undiff(unparse1, Time.BEFORE); + String unparse42 = VariationUnparser.undiff(unparse1, Time.AFTER); + System.out.println(temp1.equals(unparse11) + " " + temp2.equals(unparse12) + " " + temp1.equals(unparse21) + " " + + temp2.equals(unparse22) + " " + temp1.equals(unparse31) + " " + temp2.equals(unparse32) + " " + + temp1.equals(unparse41) + " " + temp2.equals(unparse42)); + System.out.println(removeWhitespace(temp1).equals(removeWhitespace(unparse11)) + " " + + removeWhitespace(temp2).equals(removeWhitespace(unparse12)) + " " + + removeWhitespace(temp1).equals(removeWhitespace(unparse21)) + " " + + removeWhitespace(temp2).equals(removeWhitespace(unparse22)) + " " + + removeWhitespace(temp1).equals(removeWhitespace(unparse31)) + " " + + removeWhitespace(temp2).equals(removeWhitespace(unparse32)) + " " + + removeWhitespace(temp1).equals(removeWhitespace(unparse41)) + " " + + removeWhitespace(temp2).equals(removeWhitespace(unparse42))); + temp = removeWhitespace(temp); + unparse1 = removeWhitespace(unparse1); + unparse2 = removeWhitespace(unparse2); + unparse3 = removeWhitespace(unparse3); + unparse4 = removeWhitespace(unparse4); + System.out.println(temp.equals(unparse1) + " " + temp.equals(unparse2) + " " + temp.equals(unparse3) + " " + + temp.equals(unparse4)); } - public static String parseUnparse(Path path) { + public static String parseUnparseTree(Path path, VariationDiffParseOptions option) { String temp = "b"; try { - VariationTree tree = VariationTree.fromFile(path); + VariationTree tree = VariationTree.fromFile(path, option); temp = VariationUnparser.variationTreeUnparser(tree); } catch (Exception e) { e.printStackTrace(); @@ -74,4 +131,18 @@ public static String parseUnparse(Path path) { return temp; } + public static String parseUnparseDiff(Path path, VariationDiffParseOptions option) { + String temp = "b"; + try { + VariationDiff diff = VariationDiff.fromFile(path, option); + temp = VariationUnparser.variationDiffUnparser(diff); + } catch (Exception e) { + e.printStackTrace(); + } + return temp; + } + + public static String removeWhitespace(String string) { + return string.replaceAll("\\s+", ""); + } } diff --git a/src/test/resources/unparser/test3.txt b/src/test/resources/unparser/test3.txt new file mode 100644 index 000000000..1e1a94da3 --- /dev/null +++ b/src/test/resources/unparser/test3.txt @@ -0,0 +1,11 @@ +#ifdef A + foo(); + bar(); +#else + #ifdef B + baz(); + #endif +#endif +#ifdef C + boob() +#endif \ No newline at end of file diff --git a/src/test/resources/unparser/test4.txt b/src/test/resources/unparser/test4.txt new file mode 100644 index 000000000..21cb52f21 --- /dev/null +++ b/src/test/resources/unparser/test4.txt @@ -0,0 +1,8 @@ +#ifdef A + foo(); + bar(); +#endif + +#if B && (!A || C) + baz(); +#endif \ No newline at end of file diff --git a/src/test/resources/unparser/test5.txt b/src/test/resources/unparser/test5.txt new file mode 100644 index 000000000..3137d7b23 --- /dev/null +++ b/src/test/resources/unparser/test5.txt @@ -0,0 +1,5 @@ +#ifdef A + baz(); + foo(); +#endif +foo(); \ No newline at end of file diff --git a/src/test/resources/unparser/test6.txt b/src/test/resources/unparser/test6.txt new file mode 100644 index 000000000..b0f909a06 --- /dev/null +++ b/src/test/resources/unparser/test6.txt @@ -0,0 +1,5 @@ +#ifdef A + baz(); + foo(); +#endif + diff --git a/src/test/resources/unparser/test7.txt b/src/test/resources/unparser/test7.txt new file mode 100644 index 000000000..2a248d683 --- /dev/null +++ b/src/test/resources/unparser/test7.txt @@ -0,0 +1,9 @@ +#ifdef FEAT_GUI + if (gui.in_use) + gui_mch_set_foreground(); +#else +# ifdef MSWIN + win32_set_foreground(); +# endif +#endif +} diff --git a/src/test/resources/unparser/test8.txt b/src/test/resources/unparser/test8.txt new file mode 100644 index 000000000..2ba7d7d28 --- /dev/null +++ b/src/test/resources/unparser/test8.txt @@ -0,0 +1,11 @@ +#ifdef FEAT_GUI + if (gui.in_use) + { + gui_mch_set_foreground(); + return; + } +#endif +#if defined(MSWIN) && (!defined(FEAT_GUI) || defined(VIMDLL)) + win32_set_foreground(); +#endif +} From af4eb0fe813344d419a92cc44a5bfa91b6691171 Mon Sep 17 00:00:00 2001 From: eshul Date: Wed, 27 Nov 2024 18:15:22 +0100 Subject: [PATCH 012/126] feat: create a dataset for my bachelor thesis --- docs/datasets/eugen-bachelor-thesis.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 docs/datasets/eugen-bachelor-thesis.md diff --git a/docs/datasets/eugen-bachelor-thesis.md b/docs/datasets/eugen-bachelor-thesis.md new file mode 100644 index 000000000..ee3018f1d --- /dev/null +++ b/docs/datasets/eugen-bachelor-thesis.md @@ -0,0 +1,3 @@ +Project name | Domain | Source code available (\*\*y\*\*es/\*\*n\*\*o)? | Is it a git repository (\*\*y\*\*es/\*\*n\*\*o)? | Repository URL | Clone URL | Estimated number of commits +-------------------|-------------------------|-------------------------------------------------|--------------------------------------------------|--------------------------------------------------------------|----------------------------------------------------|--------------------------------- +sylpheed | e-mail client | y | y | https://github.com/jan0sch/sylpheed | https://github.com/jan0sch/sylpheed.git | 2,682 From 5408c720445952da436cf73b9e71e613d94d2140 Mon Sep 17 00:00:00 2001 From: eshul Date: Wed, 27 Nov 2024 18:16:34 +0100 Subject: [PATCH 013/126] feat: create an experiment for testing the unparser --- replication/thesis-es/Dockerfile | 57 +++++ replication/thesis-es/README.md | 49 +++++ replication/thesis-es/build.bat | 19 ++ replication/thesis-es/build.sh | 11 + replication/thesis-es/docker/DOCKER.md | 6 + replication/thesis-es/docker/execute.sh | 11 + replication/thesis-es/execute.bat | 15 ++ replication/thesis-es/execute.sh | 8 + replication/thesis-es/results/.gitignore | 4 + replication/thesis-es/stop-execution.bat | 3 + replication/thesis-es/stop-execution.sh | 4 + .../experiments/thesis_es/Main.java | 52 +++++ .../thesis_es/UnparseAnalysis.java | 198 ++++++++++++++++++ .../thesis_es/UnparseEvaluation.java | 130 ++++++++++++ 14 files changed, 567 insertions(+) create mode 100644 replication/thesis-es/Dockerfile create mode 100644 replication/thesis-es/README.md create mode 100644 replication/thesis-es/build.bat create mode 100644 replication/thesis-es/build.sh create mode 100644 replication/thesis-es/docker/DOCKER.md create mode 100644 replication/thesis-es/docker/execute.sh create mode 100644 replication/thesis-es/execute.bat create mode 100644 replication/thesis-es/execute.sh create mode 100644 replication/thesis-es/results/.gitignore create mode 100644 replication/thesis-es/stop-execution.bat create mode 100644 replication/thesis-es/stop-execution.sh create mode 100644 src/main/java/org/variantsync/diffdetective/experiments/thesis_es/Main.java create mode 100644 src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java create mode 100644 src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseEvaluation.java diff --git a/replication/thesis-es/Dockerfile b/replication/thesis-es/Dockerfile new file mode 100644 index 000000000..4520b8042 --- /dev/null +++ b/replication/thesis-es/Dockerfile @@ -0,0 +1,57 @@ +# syntax=docker/dockerfile:1 + +FROM alpine:3.15 +# PACKAGE STAGE + +# Prepare the compile environment. JDK is automatically installed +RUN apk add maven + +# Create and navigate to a working directory +WORKDIR /home/user + +COPY local-maven-repo ./local-maven-repo + +# Copy the source code +COPY src ./src +# Copy the pom.xml if Maven is used +COPY pom.xml . +# Execute the maven package process +RUN mvn package || exit + +FROM alpine:3.15 + +# Create a user +RUN adduser --disabled-password --home /home/sherlock --gecos '' sherlock + +RUN apk add --no-cache --upgrade bash +RUN apk add --update openjdk17 + +# Change into the home directory +WORKDIR /home/sherlock + +# Copy the compiled JAR file from the first stage into the second stage +# Syntax: COPY --from=STAGE_ID SOURCE_PATH TARGET_PATH +WORKDIR /home/sherlock/holmes +COPY --from=0 /home/user/target/diffdetective-*-jar-with-dependencies.jar ./DiffDetective.jar +WORKDIR /home/sherlock +RUN mkdir results + +# Copy the setup +COPY docs holmes/docs + +# Copy the docker resources +COPY docker/* ./ +COPY replication/thesis-es/docker/* ./ +RUN mkdir DiffDetectiveMining + +# Adjust permissions +RUN chown sherlock:sherlock /home/sherlock -R +RUN chmod +x execute.sh +RUN chmod +x entrypoint.sh +RUN chmod +x fix-perms.sh + +# Set the entrypoint +ENTRYPOINT ["./entrypoint.sh", "./execute.sh"] + +# Set the user +USER sherlock diff --git a/replication/thesis-es/README.md b/replication/thesis-es/README.md new file mode 100644 index 000000000..dbac2923f --- /dev/null +++ b/replication/thesis-es/README.md @@ -0,0 +1,49 @@ +![Maven](https://github.com/VariantSync/DiffDetective/actions/workflows/maven.yml/badge.svg) +[![Documentation](https://img.shields.io/badge/Documentation-Read-purple)][documentation] +[![Install](https://img.shields.io/badge/Install-Instructions-blue)](INSTALL.md) +[![GitHubPages](https://img.shields.io/badge/GitHub%20Pages-online-blue.svg?style=flat)][website] +[![License](https://img.shields.io/badge/License-GNU%20LGPLv3-blue)](../../LICENSE.LGPL3) + +# Unparsing Experiment +This is an experiment for the bachelor thesis by Eugen Shulimov which tests the unparser for variation trees and diffs. + +### Prerequisite +All following commands assume that working directory of your terminal is the `thesis-es` directory. Please switch directories, if this is not the case: +```shell +cd DiffDetective/replication/thesis-es +``` + +### Build the Docker container +Start the docker deamon. +Clone this repository. +Open a terminal and navigate to the root directory of this repository. +To build the Docker container you can run the `build` script corresponding to your operating system. +#### Windows: +`.\build.bat` +#### Linux/Mac (bash): +`./build.sh` + +### Start the experiment +To execute the experiment you can run the `execute`script corresponding to your operating system. + +#### Windows: +`.\execute.bat +#### Linux/Mac (bash): +`./execute.sh + +> If you want to stop the execution, you can call the provided script for stopping the container in a separate terminal. +> When restarted, the execution will continue processing by restarting at the last unfinished repository. +> #### Windows: +> `.\stop-execution.bat` +> #### Linux/Mac (bash): +> `./stop-execution.sh` + +You might see warnings or errors reported from SLF4J like `Failed to load class "org.slf4j.impl.StaticLoggerBinder"` which you can safely ignore. + +### View the results in the [results][resultsdir] directory +All raw results are stored in the [results][resultsdir] directory. + +[documentation]: https://variantsync.github.io/DiffDetective/docs/javadoc/ +[website]: https://variantsync.github.io/DiffDetective/ + +[resultsdir]: results diff --git a/replication/thesis-es/build.bat b/replication/thesis-es/build.bat new file mode 100644 index 000000000..73bc8d9ab --- /dev/null +++ b/replication/thesis-es/build.bat @@ -0,0 +1,19 @@ +@echo off +setlocal + +set "targetSubPath=thesis-es" + +rem Get the current directory +for %%A in ("%CD%") do set "currentDir=%%~nxA" + +rem Check if the current directory ends with the target sub-path + +if "%currentDir:~-9%"=="%targetSubPath%" ( + cd ..\.. + docker build -t diff-detective-unparse -f replication\thesis-es\Dockerfile . + @pause +) else ( + echo error: the script must be run from inside the thesis-es directory, i.e., DiffDetective\replication\%targetSubPath% +) +endlocal + diff --git a/replication/thesis-es/build.sh b/replication/thesis-es/build.sh new file mode 100644 index 000000000..19bb7b741 --- /dev/null +++ b/replication/thesis-es/build.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +# We have to switch to the root directory of the project and build the Docker image from there, +# because Docker only allows access to the files in the current file system subtree (i.e., no access to ancestors). +# We have to do this to get access to 'src', 'docker', 'local-maven-repo', etc. +# For resiliency against different working directories during execution of this +# script we calculate the correct path using the special bash variable +# BASH_SOURCE. +cd "$(dirname "${BASH_SOURCE[0]}")/../.." || exit + +docker build -t diff-detective-unparse -f replication/thesis-es/Dockerfile . diff --git a/replication/thesis-es/docker/DOCKER.md b/replication/thesis-es/docker/DOCKER.md new file mode 100644 index 000000000..966ebc0d5 --- /dev/null +++ b/replication/thesis-es/docker/DOCKER.md @@ -0,0 +1,6 @@ +# Docker Files + +This directory contains the files that are required to run the Docker container. + +## Execution +The [`execute.sh`](execute.sh) script can be adjusted to run the program that should be executed by the Docker container. \ No newline at end of file diff --git a/replication/thesis-es/docker/execute.sh b/replication/thesis-es/docker/execute.sh new file mode 100644 index 000000000..a4e2ce4dc --- /dev/null +++ b/replication/thesis-es/docker/execute.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +cd /home/sherlock/holmes || exit + +echo "Running the experiment." +java -cp DiffDetective.jar org.variantsync.diffdetective.experiments.thesis-es.Main + +echo "Collecting results." +cp -r results/* ../results/ +echo "The results are located in the 'results' directory." + diff --git a/replication/thesis-es/execute.bat b/replication/thesis-es/execute.bat new file mode 100644 index 000000000..2467d7379 --- /dev/null +++ b/replication/thesis-es/execute.bat @@ -0,0 +1,15 @@ +@echo off +setlocal + +set "targetSubPath=thesis-es" + +rem Get the current directory +for %%A in ("%CD%") do set "currentDir=%%~nxA" + +rem Check if the current directory ends with the target sub-path +if "%currentDir:~-9%"=="%targetSubPath%" ( +docker run --rm -v "%cd%\results":"/home/sherlock/results" diff-detective-unparse %* +) else ( + echo error: the script must be run from inside the thesis-es directory, i.e., DiffDetective\replication\%targetSubPath% +) +endlocal diff --git a/replication/thesis-es/execute.sh b/replication/thesis-es/execute.sh new file mode 100644 index 000000000..c0b2fba9b --- /dev/null +++ b/replication/thesis-es/execute.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Assure that the script is only called from the folder cotaining this script +cd "$(dirname "${BASH_SOURCE[0]}")" || exit + +if [[ $# -gt 0 ]]; then +echo "Executing $1" +fi +docker run --rm -v "$(pwd)/results":"/home/sherlock/results" diff-detective-unparse "$@" diff --git a/replication/thesis-es/results/.gitignore b/replication/thesis-es/results/.gitignore new file mode 100644 index 000000000..86d0cb272 --- /dev/null +++ b/replication/thesis-es/results/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore \ No newline at end of file diff --git a/replication/thesis-es/stop-execution.bat b/replication/thesis-es/stop-execution.bat new file mode 100644 index 000000000..009494d8f --- /dev/null +++ b/replication/thesis-es/stop-execution.bat @@ -0,0 +1,3 @@ +@echo "Stopping all running simulations. This will take a moment..." +@FOR /f "tokens=*" %%i IN ('docker ps -a -q --filter "ancestor=diff-detective-unparse"') DO docker stop %%i +@echo "...done." \ No newline at end of file diff --git a/replication/thesis-es/stop-execution.sh b/replication/thesis-es/stop-execution.sh new file mode 100644 index 000000000..0ebe6a83a --- /dev/null +++ b/replication/thesis-es/stop-execution.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +echo "Stopping Docker container. This will take a moment..." +docker stop "$(docker ps -a -q --filter "ancestor=diff-detective-unparse")" +echo "...done." diff --git a/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/Main.java b/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/Main.java new file mode 100644 index 000000000..9d42b4f2c --- /dev/null +++ b/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/Main.java @@ -0,0 +1,52 @@ +package org.variantsync.diffdetective.experiments.thesis_es; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import org.variantsync.diffdetective.AnalysisRunner; +import org.variantsync.diffdetective.analysis.Analysis; +import org.variantsync.diffdetective.datasets.PatchDiffParseOptions; +import org.variantsync.diffdetective.datasets.Repository; +import org.variantsync.diffdetective.diff.git.DiffFilter; +import org.variantsync.diffdetective.variation.diff.parse.VariationDiffParseOptions; + +public class Main { + + public static String dataSetPath = ""; + + public static void main(String[] args) throws IOException { + startAnalysis(); + + } + + private static void startAnalysis() throws IOException { + final AnalysisRunner.Options analysisOptions = new AnalysisRunner.Options( + Paths.get("..", "DiffDetectiveReplicationDatasets"), + Paths.get("results", "thesis_es"), + Paths.get("docs", "datasets", "eugen-bachelor-thesis.md"), + repo -> new PatchDiffParseOptions( + PatchDiffParseOptions.DiffStoragePolicy.DO_NOT_REMEMBER, + VariationDiffParseOptions.Default), + repo -> new DiffFilter.Builder().allowMerge(true) + .allowedFileExtensions("c", "cpp").build(), + true, + false); + + AnalysisRunner.run(analysisOptions, extractionRunner()); + } + + protected static BiConsumer extractionRunner() { + return (repo, repoOutputDir) -> { + + final BiFunction AnalysisFactory = (r, out) -> new Analysis("Thesis Eugen Shulimov", + List.of(new UnparseAnalysis()), r, out); + + Analysis.forEachCommit(() -> AnalysisFactory.apply(repo, repoOutputDir)); + + }; + } + +} diff --git a/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java b/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java new file mode 100644 index 000000000..2cfa8144b --- /dev/null +++ b/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java @@ -0,0 +1,198 @@ +package org.variantsync.diffdetective.experiments.thesis_es; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import org.eclipse.jgit.diff.DiffAlgorithm.SupportedAlgorithm; +import org.variantsync.diffdetective.analysis.Analysis; +import org.variantsync.diffdetective.diff.git.PatchDiff; +import org.variantsync.diffdetective.util.CSV; +import org.variantsync.diffdetective.util.FileUtils; +import org.variantsync.diffdetective.util.IO; +import org.variantsync.diffdetective.util.StringUtils; +import org.variantsync.diffdetective.variation.DiffLinesLabel; +import org.variantsync.diffdetective.variation.VariationUnparser; +import org.variantsync.diffdetective.variation.diff.Time; +import org.variantsync.diffdetective.variation.diff.VariationDiff; +import org.variantsync.diffdetective.variation.diff.construction.JGitDiff; +import org.variantsync.diffdetective.variation.diff.parse.VariationDiffParseOptions; +import org.variantsync.diffdetective.variation.tree.VariationTree; + +public class UnparseAnalysis implements Analysis.Hooks { + + public static final String CSV_EXTENSION = ".thesis_es.csv"; + + private StringBuilder csv; + + @Override + public void initializeResults(Analysis analysis) { + Analysis.Hooks.super.initializeResults(analysis); + + csv = new StringBuilder(); + csv.append(UnparseEvaluation.makeHeader(CSV.DEFAULT_CSV_DELIMITER)).append(StringUtils.LINEBREAK); + } + + @Override + public boolean analyzeVariationDiff(Analysis analysis) throws Exception { + PatchDiff patch = analysis.getCurrentPatch(); + String textDiff = patch.getDiff(); + String codeBefore = ""; + String codeAfter = ""; + codeBefore = Files.readString(Path.of(patch.getFileName(Time.BEFORE))); + codeAfter = Files.readString(Path.of(patch.getFileName(Time.AFTER))); + codeBefore = codeBefore.replaceAll("\\r\\n", "\n"); + codeAfter = codeAfter.replaceAll("\\r\\n", "\n"); + boolean[][] diffTestAll = runTestsDiff(textDiff); + boolean[] treeBeforeTest = runTestsTree(codeBefore, Path.of(patch.getFileName(Time.BEFORE))); + boolean[] treeAfterTest = runTestsTree(codeAfter, Path.of(patch.getFileName(Time.AFTER))); + boolean[] dataTests = runDataTest(textDiff, codeBefore, codeAfter); + int error = 1; + String[] errorSave = new String[] { null, null, null }; + if (!(boolOr(diffTestAll[0]) || (boolOr(diffTestAll[1]) && boolOr(diffTestAll[2])))) { + error = error * 2; + errorSave[0] = textDiff; + } + if (!boolOr(treeBeforeTest)) { + error = error * 3; + errorSave[1] = codeBefore; + } + if (!boolOr(treeAfterTest)) { + error = error * 5; + errorSave[2] = codeAfter; + } + if (!boolOr(dataTests)) { + error = error * 7; + } + + final UnparseEvaluation ue = new UnparseEvaluation( + boolToInt(dataTests), + boolToInt(diffTestAll[0]), + boolToInt(diffTestAll[1]), + boolToInt(diffTestAll[2]), + boolToInt(treeBeforeTest), + boolToInt(treeAfterTest), + error, + errorSave); + csv.append(ue.toCSV()).append(StringUtils.LINEBREAK); + return Analysis.Hooks.super.analyzeVariationDiff(analysis); + } + + @Override + public void endBatch(Analysis analysis) throws IOException { + IO.write( + FileUtils.addExtension(analysis.getOutputFile(), CSV_EXTENSION), + csv.toString()); + } + + public static String removeWhitespace(String string) { + return string.replaceAll("\\s+", ""); + } + + public static String parseUnparseTree(Path path, VariationDiffParseOptions option) { + String temp = "b"; + try { + VariationTree tree = VariationTree.fromFile(path, option); + temp = VariationUnparser.variationTreeUnparser(tree); + } catch (Exception e) { + e.printStackTrace(); + } + return temp; + } + + public static String parseUnparseDiff(String textDiff, VariationDiffParseOptions option) { + String temp = "b"; + try { + VariationDiff diff = VariationDiff.fromDiff(textDiff, option); + temp = VariationUnparser.variationDiffUnparser(diff); + } catch (Exception e) { + e.printStackTrace(); + } + return temp; + } + + public static VariationDiffParseOptions optionsSetter(int i) { + if (i == 0) { + return new VariationDiffParseOptions(false, false); + } else if (i == 1) { + return new VariationDiffParseOptions(true, false); + } else if (i == 2) { + return new VariationDiffParseOptions(false, true); + } else { + return new VariationDiffParseOptions(true, true); + } + } + + public static boolean equalsText(String text1, String text2, boolean whitespace) { + if (whitespace) { + return text1.equals(text2); + } else { + return removeWhitespace(text1).equals(removeWhitespace(text2)); + } + } + + public static boolean[][] runTestsDiff(String text) { + boolean[][] array = new boolean[3][8]; + for (int i = 0; i < 4; i++) { + String diff = parseUnparseDiff(text, optionsSetter(i)); + array[0][i] = equalsText(text, diff, true); + array[0][i + 4] = equalsText(text, diff, false); + array[1][i] = equalsText(VariationUnparser.undiff(text, Time.BEFORE), + VariationUnparser.undiff(diff, Time.BEFORE), true); + array[1][i + 4] = equalsText(VariationUnparser.undiff(text, Time.BEFORE), + VariationUnparser.undiff(diff, Time.BEFORE), false); + array[2][i] = equalsText(VariationUnparser.undiff(text, Time.AFTER), + VariationUnparser.undiff(diff, Time.AFTER), true); + array[2][i + 4] = equalsText(VariationUnparser.undiff(text, Time.AFTER), + VariationUnparser.undiff(diff, Time.AFTER), false); + } + return array; + } + + public static boolean[] runTestsTree(String text, Path path) { + boolean[] array = new boolean[8]; + for (int i = 0; i < 4; i++) { + String temp = parseUnparseTree(path, optionsSetter(i)); + array[i] = equalsText(text, temp, true); + array[i + 4] = equalsText(text, temp, false); + } + return array; + } + + public static boolean[] runDataTest(String textDiff, String treeBefore, String treeAfter) throws IOException { + boolean[] array = new boolean[8]; + array[0] = JGitDiff.textDiff(treeBefore, treeAfter, SupportedAlgorithm.MYERS).equals(textDiff); + array[1] = removeWhitespace(JGitDiff.textDiff(treeBefore, treeAfter, SupportedAlgorithm.MYERS)) + .equals(removeWhitespace(textDiff)); + array[2] = JGitDiff.textDiff(treeBefore, treeAfter, SupportedAlgorithm.HISTOGRAM).equals(textDiff); + array[3] = removeWhitespace(JGitDiff.textDiff(treeBefore, treeAfter, SupportedAlgorithm.HISTOGRAM)) + .equals(removeWhitespace(textDiff)); + array[4] = treeBefore.equals(VariationUnparser.undiff(textDiff, Time.BEFORE)); + array[5] = removeWhitespace(treeBefore) + .equals(removeWhitespace(VariationUnparser.undiff(textDiff, Time.BEFORE))); + array[6] = treeAfter.equals(VariationUnparser.undiff(textDiff, Time.AFTER)); + array[7] = removeWhitespace(treeAfter).equals(removeWhitespace(VariationUnparser.undiff(textDiff, Time.AFTER))); + return array; + } + + public static int[] boolToInt(boolean[] array) { + int[] temp = new int[array.length]; + for (int i = 0; i < array.length; i++) { + if (array[i]) { + temp[i] = 1; + } else { + temp[i] = 0; + } + } + return temp; + } + + public static boolean boolOr(boolean[] array) { + for (boolean temp : array) { + if (temp) { + return true; + } + } + return false; + } + +} diff --git a/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseEvaluation.java b/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseEvaluation.java new file mode 100644 index 000000000..9b7938f1f --- /dev/null +++ b/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseEvaluation.java @@ -0,0 +1,130 @@ +package org.variantsync.diffdetective.experiments.thesis_es; + +import static org.variantsync.functjonal.Functjonal.intercalate; + +import org.variantsync.diffdetective.util.CSV; + +public record UnparseEvaluation( + int[] dataTest, + int[] diffTest, + int[] diffBeforeTreeTest, + int[] diffAfterTreeTest, + int[] treeBeforeTest, + int[] treeAfterTest, + int error, + String[] errorSave) implements CSV { + + @Override + public String toCSV(String delimiter) { + return intercalate(delimiter, + dataTest[0], + dataTest[1], + dataTest[2], + dataTest[3], + dataTest[4], + dataTest[5], + dataTest[6], + dataTest[7], + diffTest[0], + diffTest[1], + diffTest[2], + diffTest[3], + diffTest[4], + diffTest[5], + diffTest[6], + diffTest[7], + diffBeforeTreeTest[0], + diffBeforeTreeTest[1], + diffBeforeTreeTest[2], + diffBeforeTreeTest[3], + diffBeforeTreeTest[4], + diffBeforeTreeTest[5], + diffBeforeTreeTest[6], + diffBeforeTreeTest[7], + diffAfterTreeTest[0], + diffAfterTreeTest[1], + diffAfterTreeTest[2], + diffAfterTreeTest[3], + diffAfterTreeTest[4], + diffAfterTreeTest[5], + diffAfterTreeTest[6], + diffAfterTreeTest[7], + treeBeforeTest[0], + treeBeforeTest[1], + treeBeforeTest[2], + treeBeforeTest[3], + treeBeforeTest[4], + treeBeforeTest[5], + treeBeforeTest[6], + treeBeforeTest[7], + treeAfterTest[0], + treeAfterTest[1], + treeAfterTest[2], + treeAfterTest[3], + treeAfterTest[4], + treeAfterTest[5], + treeAfterTest[6], + treeAfterTest[7], + error, + errorSave[0], + errorSave[1], + errorSave[2]); + } + + public static String makeHeader(String delimiter) { + return intercalate(delimiter, + "diffFromTreesEqTextDiffMye", + "diffFromTreesEqTextDiffMyeWhite", + "diffFromTreesEqTextDiffHis", + "diffFromTreesEqTextDiffHisWhite", + "treeBefEqTreeBefDiff", + "treeBefEqTreeBefDiffWhite", + "treeAftEqTreeAftDiff", + "treeAftEqTreeAftDiffWhite", + "diffEqTestMultiL0EmptyL0", + "diffEqTestMultiL1EmptyL0", + "diffEqTestMultiL0EmptyL1", + "diffEqTestMultiL1EmptyL1", + "diffEqTestMultiL0EmptyL0White", + "diffEqTestMultiL1EmptyL0White", + "diffEqTestMultiL0EmptyL1White", + "diffEqTestMultiL1EmptyL1White", + "diffEqTestBefTreeMultiL0EmptyL0", + "diffEqTestBefTreeMultiL1EmptyL0", + "diffEqTestBefTreeMultiL0EmptyL1", + "diffEqTestBefTreeMultiL1EmptyL1", + "diffEqTestBefTreeMultiL0EmptyL0White", + "diffEqTestBefTreeMultiL1EmptyL0White", + "diffEqTestBefTreeMultiL0EmptyL1White", + "diffEqTestBefTreeMultiL1EmptyL1White", + "diffEqTestAftTreeMultiL0EmptyL0", + "diffEqTestAftTreeMultiL1EmptyL0", + "diffEqTestAftTreeMultiL0EmptyL1", + "diffEqTestAftTreeMultiL1EmptyL1", + "diffEqTestAftTreeMultiL0EmptyL0White", + "diffEqTestAftTreeMultiL1EmptyL0White", + "diffEqTestAftTreeMultiL0EmptyL1White", + "diffEqTestAftTreeMultiL1EmptyL1White", + "treeBeforeEqTestMultiL0EmptyL0", + "treeBeforeEqTestMultiL1EmptyL0", + "treeBeforeEqTestMultiL0EmptyL1", + "treeBeforeEqTestMultiL1EmptyL1", + "treeBeforeEqTestMultiL0EmptyL0White", + "treeBeforeEqTestMultiL1EmptyL0White", + "treeBeforeEqTestMultiL0EmptyL1White", + "treeBeforeEqTestMultiL1EmptyL1White", + "treeAfterEqTestMultiL0EmptyL0", + "treeAfterEqTestMultiL1EmptyL0", + "treeAfterEqTestMultiL0EmptyL1", + "treeAfterEqTestMultiL1EmptyL1", + "treeAfterEqTestMultiL0EmptyL0White", + "treeAfterEqTestMultiL1EmptyL0White", + "treeAfterEqTestMultiL0EmptyL1White", + "treeAfterEqTestMultiL1EmptyL1White", + "errorTyp", + "errorData1", + "errorData2", + "errorData3"); + } + +} From b5a2a8cdda6a39430cc89f4b846d61d011b866b7 Mon Sep 17 00:00:00 2001 From: eshul Date: Thu, 28 Nov 2024 11:29:43 +0100 Subject: [PATCH 014/126] fix: fix the analysis in the unparser experiment --- .../diffdetective/experiments/thesis_es/UnparseAnalysis.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java b/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java index 2cfa8144b..ef35c1346 100644 --- a/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java +++ b/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java @@ -38,8 +38,8 @@ public boolean analyzeVariationDiff(Analysis analysis) throws Exception { String textDiff = patch.getDiff(); String codeBefore = ""; String codeAfter = ""; - codeBefore = Files.readString(Path.of(patch.getFileName(Time.BEFORE))); - codeAfter = Files.readString(Path.of(patch.getFileName(Time.AFTER))); + codeBefore = VariationUnparser.undiff(patch.getDiff(), Time.BEFORE); + codeAfter = VariationUnparser.undiff(patch.getDiff(), Time.AFTER); codeBefore = codeBefore.replaceAll("\\r\\n", "\n"); codeAfter = codeAfter.replaceAll("\\r\\n", "\n"); boolean[][] diffTestAll = runTestsDiff(textDiff); From 870b76a41de6d4ef932b9319f3f739307ae90c7d Mon Sep 17 00:00:00 2001 From: eshul Date: Tue, 3 Dec 2024 16:41:47 +0100 Subject: [PATCH 015/126] feat: change the dataset of the unparse experiment --- docs/datasets/eugen-bachelor-thesis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/datasets/eugen-bachelor-thesis.md b/docs/datasets/eugen-bachelor-thesis.md index ee3018f1d..1baa9ac75 100644 --- a/docs/datasets/eugen-bachelor-thesis.md +++ b/docs/datasets/eugen-bachelor-thesis.md @@ -1,3 +1,3 @@ Project name | Domain | Source code available (\*\*y\*\*es/\*\*n\*\*o)? | Is it a git repository (\*\*y\*\*es/\*\*n\*\*o)? | Repository URL | Clone URL | Estimated number of commits -------------------|-------------------------|-------------------------------------------------|--------------------------------------------------|--------------------------------------------------------------|----------------------------------------------------|--------------------------------- -sylpheed | e-mail client | y | y | https://github.com/jan0sch/sylpheed | https://github.com/jan0sch/sylpheed.git | 2,682 +xfig | vector graphics editor | y | y | https://github.com/hhoeflin/xfig | https://github.com/hhoeflin/xfig.git | 9 From 5bdb8f10028e0d154cefdd7036434268d75c51cb Mon Sep 17 00:00:00 2001 From: eshul Date: Thu, 5 Dec 2024 12:24:43 +0100 Subject: [PATCH 016/126] feat: change some things in the analysis --- .../experiments/thesis_es/Main.java | 105 +++++++++++++++++- .../thesis_es/UnparseAnalysis.java | 87 ++++++++++----- .../thesis_es/UnparseEvaluation.java | 43 ++----- .../variation/VariationUnparser.java | 30 ++--- .../variation/tree/VariationTree.java | 23 ++++ src/test/java/VariationUnparserTest.java | 31 +++++- 6 files changed, 240 insertions(+), 79 deletions(-) diff --git a/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/Main.java b/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/Main.java index 9d42b4f2c..abbc864ee 100644 --- a/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/Main.java +++ b/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/Main.java @@ -1,14 +1,18 @@ package org.variantsync.diffdetective.experiments.thesis_es; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.List; import java.util.function.BiConsumer; import java.util.function.BiFunction; +import java.util.stream.Stream; import org.variantsync.diffdetective.AnalysisRunner; import org.variantsync.diffdetective.analysis.Analysis; import org.variantsync.diffdetective.datasets.PatchDiffParseOptions; +import org.variantsync.diffdetective.datasets.PatchDiffParseOptions.DiffStoragePolicy; import org.variantsync.diffdetective.datasets.Repository; import org.variantsync.diffdetective.diff.git.DiffFilter; import org.variantsync.diffdetective.variation.diff.parse.VariationDiffParseOptions; @@ -19,6 +23,7 @@ public class Main { public static void main(String[] args) throws IOException { startAnalysis(); + // evaluationAnalysis(Path.of("docs","datasets","eugen-bachelor-thesis.md")); } @@ -28,7 +33,7 @@ private static void startAnalysis() throws IOException { Paths.get("results", "thesis_es"), Paths.get("docs", "datasets", "eugen-bachelor-thesis.md"), repo -> new PatchDiffParseOptions( - PatchDiffParseOptions.DiffStoragePolicy.DO_NOT_REMEMBER, + DiffStoragePolicy.REMEMBER_FULL_DIFF, VariationDiffParseOptions.Default), repo -> new DiffFilter.Builder().allowMerge(true) .allowedFileExtensions("c", "cpp").build(), @@ -49,4 +54,102 @@ protected static BiConsumer extractionRunner() { }; } + private static void evaluationAnalysis(Path path) throws IOException { + int count = 0; + int error = 0; + int[] diffTest = { 0, 0, 0, 0, 0, 0, 0, 0 }; + int[] diffSemEqTest = { 0, 0, 0, 0, }; + int[] treeTest = { 0, 0, 0, 0, 0, 0, 0, 0 }; + List errorList = new ArrayList<>(); + String data = Files.readString(path); + String[] splitPathData = data.split("\n"); + for (int i = 2; i < splitPathData.length; i++) { + String name = splitPathData[i].split("|")[0].replaceAll("\\s+", ""); + Stream files = Files + .list(Path.of("results", "thesis_es", name)) + .filter(filename -> filename.getFileName().toString().endsWith(".thesis_es.csv")); + for (Path tempPath : files.toList()) { + String[] splitFileData = Files.readString(tempPath).split("\n"); + for (int j = 1; j < splitFileData.length; j++) { + String[] splitLineData = splitFileData[j].split(";"); + count = count + 1; + diffTest[0] = diffTest[0] + Integer.parseInt(splitLineData[8]); + diffTest[1] = diffTest[1] + Integer.parseInt(splitLineData[9]); + diffTest[2] = diffTest[2] + Integer.parseInt(splitLineData[10]); + diffTest[3] = diffTest[3] + Integer.parseInt(splitLineData[11]); + diffTest[4] = diffTest[4] + Integer.parseInt(splitLineData[12]); + diffTest[5] = diffTest[5] + Integer.parseInt(splitLineData[13]); + diffTest[6] = diffTest[6] + Integer.parseInt(splitLineData[14]); + diffTest[7] = diffTest[7] + Integer.parseInt(splitLineData[15]); + diffSemEqTest[0] = diffSemEqTest[0] + Integer.parseInt(splitLineData[16]); + diffSemEqTest[1] = diffSemEqTest[1] + Integer.parseInt(splitLineData[17]); + diffSemEqTest[2] = diffSemEqTest[2] + Integer.parseInt(splitLineData[18]); + diffSemEqTest[3] = diffSemEqTest[3] + Integer.parseInt(splitLineData[19]); + treeTest[0] = treeTest[0] + Integer.parseInt(splitLineData[20]); + treeTest[1] = treeTest[1] + Integer.parseInt(splitLineData[21]); + treeTest[2] = treeTest[2] + Integer.parseInt(splitLineData[22]); + treeTest[3] = treeTest[3] + Integer.parseInt(splitLineData[23]); + treeTest[4] = treeTest[4] + Integer.parseInt(splitLineData[24]); + treeTest[5] = treeTest[5] + Integer.parseInt(splitLineData[25]); + treeTest[6] = treeTest[6] + Integer.parseInt(splitLineData[26]); + treeTest[7] = treeTest[7] + Integer.parseInt(splitLineData[27]); + treeTest[0] = treeTest[0] + Integer.parseInt(splitLineData[28]); + treeTest[1] = treeTest[1] + Integer.parseInt(splitLineData[29]); + treeTest[2] = treeTest[2] + Integer.parseInt(splitLineData[30]); + treeTest[3] = treeTest[3] + Integer.parseInt(splitLineData[31]); + treeTest[4] = treeTest[4] + Integer.parseInt(splitLineData[32]); + treeTest[5] = treeTest[5] + Integer.parseInt(splitLineData[33]); + treeTest[6] = treeTest[6] + Integer.parseInt(splitLineData[34]); + treeTest[7] = treeTest[7] + Integer.parseInt(splitLineData[35]); + int errorCode = Integer.parseInt(splitLineData[36]); + if (errorCode != 1) { + if (errorCode % 2 == 0) { + errorList.add(splitLineData[37]); + error = error + 1; + } + if (errorCode % 3 == 0) { + errorList.add(splitLineData[38]); + error = error + 1; + } + if (errorCode % 5 == 0) { + errorList.add(splitLineData[39]); + error = error + 1; + } + } + } + } + } + List result = new ArrayList<>(); + result.add("Anzahl geprüfter Diffs : " + count); + result.add("Anzahl syntaktisch korrekter Diffs mit MultiLine0 und EmptyLine0 : " + diffTest[0]); + result.add("Anzahl syntaktisch korrekter Diffs mit MultiLine1 und EmptyLine0 : " + diffTest[1]); + result.add("Anzahl syntaktisch korrekter Diffs mit MultiLine0 und EmptyLine1 : " + diffTest[2]); + result.add("Anzahl syntaktisch korrekter Diffs mit MultiLine1 und EmptyLine1 : " + diffTest[3]); + result.add("Anzahl syntaktisch korrekter Diffs ohne Whitespace mit MultiLine0 und EmptyLine0 : " + diffTest[4]); + result.add("Anzahl syntaktisch korrekter Diffs ohne Whitespace mit MultiLine1 und EmptyLine0 : " + diffTest[5]); + result.add("Anzahl syntaktisch korrekter Diffs ohne Whitespace mit MultiLine0 und EmptyLine1 : " + diffTest[6]); + result.add("Anzahl syntaktisch korrekter Diffs ohne Whitespace mit MultiLine1 und EmptyLine1 : " + diffTest[7]); + result.add("Anzahl semantisch korrekter Diffs mit MultiLine0 und EmptyLine0 : " + diffSemEqTest[0]); + result.add("Anzahl semantisch korrekter Diffs mit MultiLine1 und EmptyLine0 : " + diffSemEqTest[1]); + result.add("Anzahl semantisch korrekter Diffs mit MultiLine0 und EmptyLine1 : " + diffSemEqTest[2]); + result.add("Anzahl semantisch korrekter Diffs mit MultiLine1 und EmptyLine1 : " + diffSemEqTest[3]); + result.add("Anzahl syntaktisch korrekter Trees mit MultiLine0 und EmptyLine0 : " + treeTest[0]); + result.add("Anzahl syntaktisch korrekter Trees mit MultiLine1 und EmptyLine0 : " + treeTest[1]); + result.add("Anzahl syntaktisch korrekter Trees mit MultiLine0 und EmptyLine1 : " + treeTest[2]); + result.add("Anzahl syntaktisch korrekter Trees mit MultiLine1 und EmptyLine1 : " + treeTest[3]); + result.add("Anzahl syntaktisch korrekter Trees ohne Whitespace mit MultiLine0 und EmptyLine0 : " + treeTest[4]); + result.add("Anzahl syntaktisch korrekter Trees ohne Whitespace mit MultiLine1 und EmptyLine0 : " + treeTest[5]); + result.add("Anzahl syntaktisch korrekter Trees ohne Whitespace mit MultiLine0 und EmptyLine1 : " + treeTest[6]); + result.add("Anzahl syntaktisch korrekter Trees ohne Whitespace mit MultiLine1 und EmptyLine1 : " + treeTest[7]); + result.add("Anzahl Fehler : " + error); + + Files.write(Path.of("results", "thesis_es", "resultOfAnalysis.txt"), result); + int c = 1; + for (String line : errorList) { + Files.writeString(Path.of("results", "thesis_es", "errors", c + "error.txt"), line); + c = c + 1; + } + + } + } diff --git a/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java b/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java index ef35c1346..c182812d9 100644 --- a/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java +++ b/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java @@ -1,8 +1,6 @@ package org.variantsync.diffdetective.experiments.thesis_es; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import org.eclipse.jgit.diff.DiffAlgorithm.SupportedAlgorithm; import org.variantsync.diffdetective.analysis.Analysis; import org.variantsync.diffdetective.diff.git.PatchDiff; @@ -17,11 +15,14 @@ import org.variantsync.diffdetective.variation.diff.construction.JGitDiff; import org.variantsync.diffdetective.variation.diff.parse.VariationDiffParseOptions; import org.variantsync.diffdetective.variation.tree.VariationTree; +import org.variantsync.diffdetective.variation.tree.source.VariationTreeSource; public class UnparseAnalysis implements Analysis.Hooks { public static final String CSV_EXTENSION = ".thesis_es.csv"; + public static int count = 0; + private StringBuilder csv; @Override @@ -38,19 +39,31 @@ public boolean analyzeVariationDiff(Analysis analysis) throws Exception { String textDiff = patch.getDiff(); String codeBefore = ""; String codeAfter = ""; + /* + if (count < 10) { + Path pathTreeBefore = Paths.get("results", "thesis_es", "treeBefore" + count + ".txt"); + Path pathTreeAfter = Paths.get("results", "thesis_es", "treeAfter" + count + ".txt"); + Path pathDiff = Paths.get("results", "thesis_es", "diff" + count + ".txt"); + Files.writeString(pathTreeBefore, codeBefore); + Files.writeString(pathTreeAfter, codeAfter); + Files.writeString(pathDiff, textDiff); + count++; + } + */ + codeBefore = VariationUnparser.undiff(patch.getDiff(), Time.BEFORE); codeAfter = VariationUnparser.undiff(patch.getDiff(), Time.AFTER); - codeBefore = codeBefore.replaceAll("\\r\\n", "\n"); - codeAfter = codeAfter.replaceAll("\\r\\n", "\n"); + // codeBefore = codeBefore.replaceAll("\\r\\n", "\n"); + // codeAfter = codeAfter.replaceAll("\\r\\n", "\n"); boolean[][] diffTestAll = runTestsDiff(textDiff); - boolean[] treeBeforeTest = runTestsTree(codeBefore, Path.of(patch.getFileName(Time.BEFORE))); - boolean[] treeAfterTest = runTestsTree(codeAfter, Path.of(patch.getFileName(Time.AFTER))); + boolean[] treeBeforeTest = runTestsTree(codeBefore); + boolean[] treeAfterTest = runTestsTree(codeAfter); boolean[] dataTests = runDataTest(textDiff, codeBefore, codeAfter); int error = 1; - String[] errorSave = new String[] { null, null, null }; - if (!(boolOr(diffTestAll[0]) || (boolOr(diffTestAll[1]) && boolOr(diffTestAll[2])))) { + String[] errorSave = new String[] { "", "", "" }; + if (!(boolOr(diffTestAll[0]) || boolOr(diffTestAll[1]))) { error = error * 2; - errorSave[0] = textDiff; + // errorSave[0] = ; } if (!boolOr(treeBeforeTest)) { error = error * 3; @@ -58,17 +71,18 @@ public boolean analyzeVariationDiff(Analysis analysis) throws Exception { } if (!boolOr(treeAfterTest)) { error = error * 5; - errorSave[2] = codeAfter; + // errorSave[2] = codeAfter; } + /* if (!boolOr(dataTests)) { error = error * 7; } + */ final UnparseEvaluation ue = new UnparseEvaluation( boolToInt(dataTests), boolToInt(diffTestAll[0]), boolToInt(diffTestAll[1]), - boolToInt(diffTestAll[2]), boolToInt(treeBeforeTest), boolToInt(treeAfterTest), error, @@ -84,14 +98,36 @@ public void endBatch(Analysis analysis) throws IOException { csv.toString()); } + public static String removeWhitespace1(String string) { + string = string.replaceAll("\n(\\s*\n)+", "\n"); + string = string.replaceAll("\n\\s+", "\n"); + if (!string.isEmpty()) { + if (string.charAt(string.length() - 1) == '\n') { + return string.substring(0, string.length() - 1); + } + } + return string; + } + public static String removeWhitespace(String string) { - return string.replaceAll("\\s+", ""); + if (string.isEmpty()) { + return ""; + } else { + StringBuilder result = new StringBuilder(); + for (String line : string.split("\n")) { + if (!line.replaceAll("\\s+", "").isEmpty()) { + result.append(line.trim()); + result.append("\n"); + } + } + return result.substring(0, result.length() - 1); + } } - public static String parseUnparseTree(Path path, VariationDiffParseOptions option) { + public static String parseUnparseTree(String text, VariationDiffParseOptions option) { String temp = "b"; try { - VariationTree tree = VariationTree.fromFile(path, option); + VariationTree tree = VariationTree.fromText(text, VariationTreeSource.Unknown, option); temp = VariationUnparser.variationTreeUnparser(tree); } catch (Exception e) { e.printStackTrace(); @@ -131,27 +167,28 @@ public static boolean equalsText(String text1, String text2, boolean whitespace) } public static boolean[][] runTestsDiff(String text) { - boolean[][] array = new boolean[3][8]; + boolean[][] array = new boolean[2][8]; for (int i = 0; i < 4; i++) { String diff = parseUnparseDiff(text, optionsSetter(i)); array[0][i] = equalsText(text, diff, true); array[0][i + 4] = equalsText(text, diff, false); - array[1][i] = equalsText(VariationUnparser.undiff(text, Time.BEFORE), - VariationUnparser.undiff(diff, Time.BEFORE), true); - array[1][i + 4] = equalsText(VariationUnparser.undiff(text, Time.BEFORE), - VariationUnparser.undiff(diff, Time.BEFORE), false); - array[2][i] = equalsText(VariationUnparser.undiff(text, Time.AFTER), - VariationUnparser.undiff(diff, Time.AFTER), true); - array[2][i + 4] = equalsText(VariationUnparser.undiff(text, Time.AFTER), - VariationUnparser.undiff(diff, Time.AFTER), false); + array[1][i] = (equalsText(VariationUnparser.undiff(text, Time.BEFORE), + VariationUnparser.undiff(diff, Time.BEFORE), true) + && equalsText(VariationUnparser.undiff(text, Time.AFTER), + VariationUnparser.undiff(diff, Time.AFTER), true)) + || (equalsText(VariationUnparser.undiff(text, Time.BEFORE), + VariationUnparser.undiff(diff, Time.BEFORE), false) + && equalsText(VariationUnparser.undiff(text, Time.AFTER), + VariationUnparser.undiff(diff, Time.AFTER), false)); + array[1][i + 4] = false; } return array; } - public static boolean[] runTestsTree(String text, Path path) { + public static boolean[] runTestsTree(String text) { boolean[] array = new boolean[8]; for (int i = 0; i < 4; i++) { - String temp = parseUnparseTree(path, optionsSetter(i)); + String temp = parseUnparseTree(text, optionsSetter(i)); array[i] = equalsText(text, temp, true); array[i + 4] = equalsText(text, temp, false); } diff --git a/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseEvaluation.java b/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseEvaluation.java index 9b7938f1f..41c6328e2 100644 --- a/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseEvaluation.java +++ b/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseEvaluation.java @@ -7,8 +7,7 @@ public record UnparseEvaluation( int[] dataTest, int[] diffTest, - int[] diffBeforeTreeTest, - int[] diffAfterTreeTest, + int[] diffSemEqTest, int[] treeBeforeTest, int[] treeAfterTest, int error, @@ -33,22 +32,10 @@ public String toCSV(String delimiter) { diffTest[5], diffTest[6], diffTest[7], - diffBeforeTreeTest[0], - diffBeforeTreeTest[1], - diffBeforeTreeTest[2], - diffBeforeTreeTest[3], - diffBeforeTreeTest[4], - diffBeforeTreeTest[5], - diffBeforeTreeTest[6], - diffBeforeTreeTest[7], - diffAfterTreeTest[0], - diffAfterTreeTest[1], - diffAfterTreeTest[2], - diffAfterTreeTest[3], - diffAfterTreeTest[4], - diffAfterTreeTest[5], - diffAfterTreeTest[6], - diffAfterTreeTest[7], + diffSemEqTest[0], + diffSemEqTest[1], + diffSemEqTest[2], + diffSemEqTest[3], treeBeforeTest[0], treeBeforeTest[1], treeBeforeTest[2], @@ -89,22 +76,10 @@ public static String makeHeader(String delimiter) { "diffEqTestMultiL1EmptyL0White", "diffEqTestMultiL0EmptyL1White", "diffEqTestMultiL1EmptyL1White", - "diffEqTestBefTreeMultiL0EmptyL0", - "diffEqTestBefTreeMultiL1EmptyL0", - "diffEqTestBefTreeMultiL0EmptyL1", - "diffEqTestBefTreeMultiL1EmptyL1", - "diffEqTestBefTreeMultiL0EmptyL0White", - "diffEqTestBefTreeMultiL1EmptyL0White", - "diffEqTestBefTreeMultiL0EmptyL1White", - "diffEqTestBefTreeMultiL1EmptyL1White", - "diffEqTestAftTreeMultiL0EmptyL0", - "diffEqTestAftTreeMultiL1EmptyL0", - "diffEqTestAftTreeMultiL0EmptyL1", - "diffEqTestAftTreeMultiL1EmptyL1", - "diffEqTestAftTreeMultiL0EmptyL0White", - "diffEqTestAftTreeMultiL1EmptyL0White", - "diffEqTestAftTreeMultiL0EmptyL1White", - "diffEqTestAftTreeMultiL1EmptyL1White", + "diffSemEqTestMultiL0EmptyL0", + "diffSemEqTestMultiL1EmptyL0", + "diffSemEqTestMultiL0EmptyL1", + "diffSemEqTestMultiL1EmptyL1", "treeBeforeEqTestMultiL0EmptyL0", "treeBeforeEqTestMultiL1EmptyL0", "treeBeforeEqTestMultiL0EmptyL1", diff --git a/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java b/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java index 3919328cb..9f98d48d0 100644 --- a/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java +++ b/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java @@ -84,24 +84,28 @@ public static String variationDiffUnparser(VariationDiff diff) t } public static String undiff(String text, Time time) { - StringBuilder result = new StringBuilder(); - String[] textSplit = text.split("\n"); - if (Time.AFTER == time) { - for (String line : textSplit) { - if (line.charAt(0) != '-') { - result.append(line.substring(1)); - result.append("\n"); - } - } - return result.substring(0, result.length() - 1); + if (text.isEmpty()) { + return ""; } else { + StringBuilder result = new StringBuilder(); + String[] textSplit = text.split("\n"); + char zeichen; + if (Time.AFTER == time) { + zeichen = '-'; + } else { + zeichen = '+'; + } for (String line : textSplit) { - if (line.charAt(0) != '+') { + if (line.isEmpty()) { + result.append(line); + } else if (line.charAt(0) != zeichen) { result.append(line.substring(1)); - result.append("\n"); } } - return result.substring(0, result.length() - 1); + if (result.isEmpty()) { + return ""; + } + return result.toString(); } } } diff --git a/src/main/java/org/variantsync/diffdetective/variation/tree/VariationTree.java b/src/main/java/org/variantsync/diffdetective/variation/tree/VariationTree.java index 36cdff174..c9d7861ed 100644 --- a/src/main/java/org/variantsync/diffdetective/variation/tree/VariationTree.java +++ b/src/main/java/org/variantsync/diffdetective/variation/tree/VariationTree.java @@ -16,6 +16,8 @@ import java.io.BufferedReader; import java.io.IOException; +import java.io.StringReader; +import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; @@ -103,6 +105,27 @@ public static VariationTree fromFile( return new VariationTree<>(tree, source); } + /** + * Parses a {@code VariationTree} from source code with C preprocessor annotations. + * + * @param input the source code to be parsed + * @param parseOptions {@link PatchDiffParseOptions} for the parsing process. + * @return a new {@code VariationTree} representing {@code input} + * @throws DiffParseException if some preprocessor annotations can't be parsed + */ + public static VariationTree fromText( + final String input, + final VariationTreeSource source, + final VariationDiffParseOptions parseOptions + ) throws DiffParseException { + try { + return fromFile(new BufferedReader(new StringReader(input)), source, parseOptions); + } catch (IOException e) { + // Only thrown on programming errors because StringReader doesn't do IO. + throw new UncheckedIOException(e); + } + } + public static VariationTree fromProjection(final Projection projection, final VariationTreeSource source) { return fromVariationNode(projection, source); } diff --git a/src/test/java/VariationUnparserTest.java b/src/test/java/VariationUnparserTest.java index e0dd2c40a..857c33b26 100644 --- a/src/test/java/VariationUnparserTest.java +++ b/src/test/java/VariationUnparserTest.java @@ -2,6 +2,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.stream.Stream; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.variantsync.diffdetective.diff.result.DiffParseException; @@ -11,6 +12,9 @@ import org.variantsync.diffdetective.variation.diff.parse.VariationDiffParseOptions; import org.variantsync.diffdetective.variation.tree.VariationTree; import org.variantsync.diffdetective.variation.VariationUnparser; +import org.variantsync.diffdetective.variation.tree.source.VariationTreeSource; + +import static org.variantsync.diffdetective.experiments.thesis_es.UnparseAnalysis.removeWhitespace; public class VariationUnparserTest { private final static Path testDirTree = Constants.RESOURCE_DIR.resolve("unparser"); @@ -41,16 +45,35 @@ public static Stream testsDiff() throws IOException { @ParameterizedTest @MethodSource("testsTree") - public void testTree(Path basename) throws IOException, DiffParseException { + public void testTreeDir(Path basename) throws IOException, DiffParseException { testCaseTree(basename); } @ParameterizedTest @MethodSource("testsDiff") - public void testDiff(Path basename) throws IOException, DiffParseException { + public void testDiffDir(Path basename) throws IOException, DiffParseException { testCaseDiff(basename); } + @Test + public void testDiff() { + String source = ""; + String temp = ""; + try { + source = Files.readString(Path.of("src", "test", "resources", "unparser", "test8.txt")); + VariationTree tree = VariationTree.fromText(source, VariationTreeSource.Unknown, + VariationDiffParseOptions.Default); + temp = VariationUnparser.variationTreeUnparser(tree); + System.out.println(removeWhitespace(source).equals(removeWhitespace(temp))); + // System.out.println(removeWhitespace(source)); + // System.out.println("Ende"); + // System.out.println(removeWhitespace(temp)); + // System.out.println("Ende"); + } catch (Exception e) { + e.printStackTrace(); + } + } + public static void testCaseTree(Path testCasePath) { String temp = ""; try { @@ -141,8 +164,4 @@ public static String parseUnparseDiff(Path path, VariationDiffParseOptions optio } return temp; } - - public static String removeWhitespace(String string) { - return string.replaceAll("\\s+", ""); - } } From efb4b2c07004e0ce26d4f1fe7f890cd593c9c702 Mon Sep 17 00:00:00 2001 From: pmbittner Date: Thu, 5 Dec 2024 12:40:25 +0100 Subject: [PATCH 017/126] feat: add error reporting to UnparseAnalysis --- .../experiments/thesis_es/UnparseAnalysis.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java b/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java index c182812d9..315e57139 100644 --- a/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java +++ b/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java @@ -25,12 +25,17 @@ public class UnparseAnalysis implements Analysis.Hooks { private StringBuilder csv; + // for debugging + private int errorCount = 0; + @Override public void initializeResults(Analysis analysis) { Analysis.Hooks.super.initializeResults(analysis); csv = new StringBuilder(); csv.append(UnparseEvaluation.makeHeader(CSV.DEFAULT_CSV_DELIMITER)).append(StringUtils.LINEBREAK); + + errorCount = 0; } @Override @@ -67,7 +72,8 @@ public boolean analyzeVariationDiff(Analysis analysis) throws Exception { } if (!boolOr(treeBeforeTest)) { error = error * 3; - errorSave[1] = codeBefore; + reportErrorToFile(analysis, codeBefore); + // errorSave[1] = codeBefore; } if (!boolOr(treeAfterTest)) { error = error * 5; @@ -98,6 +104,13 @@ public void endBatch(Analysis analysis) throws IOException { csv.toString()); } + private void reportErrorToFile(Analysis analysis, String errorMessage) throws IOException { + IO.write( + FileUtils.addExtension(analysis.getOutputFile().resolve("_error" + errorCount), ".txt"), + errorMessage); + ++errorCount; + } + public static String removeWhitespace1(String string) { string = string.replaceAll("\n(\\s*\n)+", "\n"); string = string.replaceAll("\n\\s+", "\n"); From dee62f810a25286aede2b0a408016a169fad4f65 Mon Sep 17 00:00:00 2001 From: eshul Date: Sat, 21 Dec 2024 15:35:07 +0100 Subject: [PATCH 018/126] fix: store endifs depending on the time --- .../diffdetective/variation/diff/DiffNode.java | 12 +++++++----- .../diffdetective/variation/diff/Projection.java | 2 +- .../diff/parse/VariationDiffParser.java | 16 +++++++++++++--- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/variantsync/diffdetective/variation/diff/DiffNode.java b/src/main/java/org/variantsync/diffdetective/variation/diff/DiffNode.java index a8ff8e0d0..f2f2101c1 100644 --- a/src/main/java/org/variantsync/diffdetective/variation/diff/DiffNode.java +++ b/src/main/java/org/variantsync/diffdetective/variation/diff/DiffNode.java @@ -51,7 +51,7 @@ public class DiffNode implements HasNodeType { private Node featureMapping; - private List endIf = null; + private List[] endIf = new List[2]; /** * The parents {@link DiffNode} before and after the edit. @@ -150,18 +150,20 @@ public void setLabel(L newLabel) { /** * Returns the line with the endif of the corresponding if, if the node is an if node, otherwise null + * @param time when the returned endif exists * @return String, the Line with endif */ - public List getEndIf() { - return endIf; + public List getEndIf(Time time) { + return endIf[time.ordinal()]; } /** * Sets the line with the endif of the corresponding if, if the node is an if node * @param endIf String, the Line with endif + * @param time when {@code endIf} exists */ - public void setEndIf(List endIf) { - this.endIf = endIf; + public void setEndIf(List endIf, Time time) { + this.endIf[time.ordinal()] = endIf; } /** diff --git a/src/main/java/org/variantsync/diffdetective/variation/diff/Projection.java b/src/main/java/org/variantsync/diffdetective/variation/diff/Projection.java index f9ac7a02d..9b89ee826 100644 --- a/src/main/java/org/variantsync/diffdetective/variation/diff/Projection.java +++ b/src/main/java/org/variantsync/diffdetective/variation/diff/Projection.java @@ -124,6 +124,6 @@ public int getID() { @Override public List getEndIf() { - return getBackingNode().getEndIf(); + return getBackingNode().getEndIf(getTime()); } }; diff --git a/src/main/java/org/variantsync/diffdetective/variation/diff/parse/VariationDiffParser.java b/src/main/java/org/variantsync/diffdetective/variation/diff/parse/VariationDiffParser.java index 5154bdb08..002714dde 100644 --- a/src/main/java/org/variantsync/diffdetective/variation/diff/parse/VariationDiffParser.java +++ b/src/main/java/org/variantsync/diffdetective/variation/diff/parse/VariationDiffParser.java @@ -329,7 +329,7 @@ private void parseLine( // Do not create a node for ENDIF, but update the line numbers of the closed if-chain // and remove that if-chain from the relevant stacks. diffType.forAllTimesOfExistence(beforeStack, afterStack, stack -> - popIfChain(stack, fromLine, line) + popIfChain(stack, fromLine, line, diffType) ); } else if (options.collapseMultipleCodeLines() && annotation.type() == AnnotationType.None @@ -369,17 +369,27 @@ private void parseLine( private void popIfChain( Stack> stack, DiffLineNumber elseLineNumber, - LogicalLine line + LogicalLine line, + DiffType diffType ) throws DiffParseException { DiffLineNumber previousLineNumber = elseLineNumber; do { DiffNode annotation = stack.peek(); + + // Save endif if (annotation.isIf()) { List list = new ArrayList<>(); for (int i = 0; i < line.getLines().size(); i++) { list.add(line.getLines().get(i).content()); } - annotation.setEndIf(list); + if (diffType.existsBefore() && diffType.existsAfter()) { + annotation.setEndIf(list, Time.BEFORE); + annotation.setEndIf(list, Time.AFTER); + } else if (diffType.existsBefore()) { + annotation.setEndIf(list, Time.BEFORE); + } else { + annotation.setEndIf(list, Time.AFTER); + } } // Set the line number of now closed annotations to the beginning of the From dd5485930c7800c75a8d6ee0cef554b390bc55bb Mon Sep 17 00:00:00 2001 From: eshul Date: Sat, 21 Dec 2024 15:36:43 +0100 Subject: [PATCH 019/126] test: add an example of failing to unparse --- src/test/resources/unparser/diff/diff.diff | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 src/test/resources/unparser/diff/diff.diff diff --git a/src/test/resources/unparser/diff/diff.diff b/src/test/resources/unparser/diff/diff.diff new file mode 100644 index 000000000..e45efbf26 --- /dev/null +++ b/src/test/resources/unparser/diff/diff.diff @@ -0,0 +1,4 @@ + #if A + Code ++#endif /* G_OS_WIN32 */ +-#endif /* WIN32 */ From e312978db8c06a86489b4d256e56fe1b13805e2c Mon Sep 17 00:00:00 2001 From: eshul Date: Sat, 21 Dec 2024 15:37:42 +0100 Subject: [PATCH 020/126] fix: add new lines to the unparsed code --- .../variantsync/diffdetective/variation/VariationUnparser.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java b/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java index 9f98d48d0..0ea4c0f18 100644 --- a/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java +++ b/src/main/java/org/variantsync/diffdetective/variation/VariationUnparser.java @@ -98,8 +98,10 @@ public static String undiff(String text, Time time) { for (String line : textSplit) { if (line.isEmpty()) { result.append(line); + result.append("\n"); } else if (line.charAt(0) != zeichen) { result.append(line.substring(1)); + result.append("\n"); } } if (result.isEmpty()) { From f743e2cbe2f369cd7510cb3a6c12a519db84cb15 Mon Sep 17 00:00:00 2001 From: eshul Date: Sat, 21 Dec 2024 15:49:56 +0100 Subject: [PATCH 021/126] feat: rework the unparse experiment evaluation --- docs/datasets/eugen-bachelor-thesis.md | 4 +- .../experiments/thesis_es/Main.java | 134 ++++++++------ .../thesis_es/UnparseAnalysis.java | 165 +++++++++++++----- .../thesis_es/UnparseEvaluation.java | 16 +- 4 files changed, 206 insertions(+), 113 deletions(-) diff --git a/docs/datasets/eugen-bachelor-thesis.md b/docs/datasets/eugen-bachelor-thesis.md index 1baa9ac75..52f415472 100644 --- a/docs/datasets/eugen-bachelor-thesis.md +++ b/docs/datasets/eugen-bachelor-thesis.md @@ -1,3 +1,5 @@ Project name | Domain | Source code available (\*\*y\*\*es/\*\*n\*\*o)? | Is it a git repository (\*\*y\*\*es/\*\*n\*\*o)? | Repository URL | Clone URL | Estimated number of commits -------------------|-------------------------|-------------------------------------------------|--------------------------------------------------|--------------------------------------------------------------|----------------------------------------------------|--------------------------------- -xfig | vector graphics editor | y | y | https://github.com/hhoeflin/xfig | https://github.com/hhoeflin/xfig.git | 9 +berkeley-db-libdb | database system | y | y | https://github.com/berkeleydb/libdb | https://github.com/berkeleydb/libdb.git | 7 +sylpheed | e-mail client | y | y | https://github.com/jan0sch/sylpheed | https://github.com/jan0sch/sylpheed.git | 2,682 +vim | text editor | y | y | https://github.com/vim/vim | https://github.com/vim/vim.git | 17,109 diff --git a/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/Main.java b/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/Main.java index abbc864ee..089b21885 100644 --- a/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/Main.java +++ b/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/Main.java @@ -11,6 +11,8 @@ import java.util.stream.Stream; import org.variantsync.diffdetective.AnalysisRunner; import org.variantsync.diffdetective.analysis.Analysis; +import org.variantsync.diffdetective.datasets.DatasetDescription; +import org.variantsync.diffdetective.datasets.DefaultDatasets; import org.variantsync.diffdetective.datasets.PatchDiffParseOptions; import org.variantsync.diffdetective.datasets.PatchDiffParseOptions.DiffStoragePolicy; import org.variantsync.diffdetective.datasets.Repository; @@ -19,11 +21,9 @@ public class Main { - public static String dataSetPath = ""; - public static void main(String[] args) throws IOException { startAnalysis(); - // evaluationAnalysis(Path.of("docs","datasets","eugen-bachelor-thesis.md")); + evaluationAnalysis(Path.of("docs", "datasets", "eugen-bachelor-thesis.md")); } @@ -54,24 +54,32 @@ protected static BiConsumer extractionRunner() { }; } + /** + * Verarbeitet die Ergebnisse der Analyse um aus einzelnen + * Angaben eine gesamt Übersicht zu bekommen + * + * @param path Pfad zu der markdown datei, aus der die repositories für die + * Analyse stammen + * @throws IOException + */ private static void evaluationAnalysis(Path path) throws IOException { int count = 0; - int error = 0; int[] diffTest = { 0, 0, 0, 0, 0, 0, 0, 0 }; int[] diffSemEqTest = { 0, 0, 0, 0, }; int[] treeTest = { 0, 0, 0, 0, 0, 0, 0, 0 }; - List errorList = new ArrayList<>(); - String data = Files.readString(path); - String[] splitPathData = data.split("\n"); - for (int i = 2; i < splitPathData.length; i++) { - String name = splitPathData[i].split("|")[0].replaceAll("\\s+", ""); + final List datasets = DefaultDatasets.loadDatasets(path); + + for (DatasetDescription description : datasets) { Stream files = Files - .list(Path.of("results", "thesis_es", name)) + .list(Path.of("results", "thesis_es", description.name())) .filter(filename -> filename.getFileName().toString().endsWith(".thesis_es.csv")); for (Path tempPath : files.toList()) { String[] splitFileData = Files.readString(tempPath).split("\n"); for (int j = 1; j < splitFileData.length; j++) { String[] splitLineData = splitFileData[j].split(";"); + for (int i = 0; i < splitLineData.length; i++) { + splitLineData[i] = parseNumberStringToIntWithLengthGreaterOne(splitLineData[i]); + } count = count + 1; diffTest[0] = diffTest[0] + Integer.parseInt(splitLineData[8]); diffTest[1] = diffTest[1] + Integer.parseInt(splitLineData[9]); @@ -101,55 +109,77 @@ private static void evaluationAnalysis(Path path) throws IOException { treeTest[5] = treeTest[5] + Integer.parseInt(splitLineData[33]); treeTest[6] = treeTest[6] + Integer.parseInt(splitLineData[34]); treeTest[7] = treeTest[7] + Integer.parseInt(splitLineData[35]); - int errorCode = Integer.parseInt(splitLineData[36]); - if (errorCode != 1) { - if (errorCode % 2 == 0) { - errorList.add(splitLineData[37]); - error = error + 1; - } - if (errorCode % 3 == 0) { - errorList.add(splitLineData[38]); - error = error + 1; - } - if (errorCode % 5 == 0) { - errorList.add(splitLineData[39]); - error = error + 1; - } - } + } } } List result = new ArrayList<>(); - result.add("Anzahl geprüfter Diffs : " + count); - result.add("Anzahl syntaktisch korrekter Diffs mit MultiLine0 und EmptyLine0 : " + diffTest[0]); - result.add("Anzahl syntaktisch korrekter Diffs mit MultiLine1 und EmptyLine0 : " + diffTest[1]); - result.add("Anzahl syntaktisch korrekter Diffs mit MultiLine0 und EmptyLine1 : " + diffTest[2]); - result.add("Anzahl syntaktisch korrekter Diffs mit MultiLine1 und EmptyLine1 : " + diffTest[3]); - result.add("Anzahl syntaktisch korrekter Diffs ohne Whitespace mit MultiLine0 und EmptyLine0 : " + diffTest[4]); - result.add("Anzahl syntaktisch korrekter Diffs ohne Whitespace mit MultiLine1 und EmptyLine0 : " + diffTest[5]); - result.add("Anzahl syntaktisch korrekter Diffs ohne Whitespace mit MultiLine0 und EmptyLine1 : " + diffTest[6]); - result.add("Anzahl syntaktisch korrekter Diffs ohne Whitespace mit MultiLine1 und EmptyLine1 : " + diffTest[7]); - result.add("Anzahl semantisch korrekter Diffs mit MultiLine0 und EmptyLine0 : " + diffSemEqTest[0]); - result.add("Anzahl semantisch korrekter Diffs mit MultiLine1 und EmptyLine0 : " + diffSemEqTest[1]); - result.add("Anzahl semantisch korrekter Diffs mit MultiLine0 und EmptyLine1 : " + diffSemEqTest[2]); - result.add("Anzahl semantisch korrekter Diffs mit MultiLine1 und EmptyLine1 : " + diffSemEqTest[3]); - result.add("Anzahl syntaktisch korrekter Trees mit MultiLine0 und EmptyLine0 : " + treeTest[0]); - result.add("Anzahl syntaktisch korrekter Trees mit MultiLine1 und EmptyLine0 : " + treeTest[1]); - result.add("Anzahl syntaktisch korrekter Trees mit MultiLine0 und EmptyLine1 : " + treeTest[2]); - result.add("Anzahl syntaktisch korrekter Trees mit MultiLine1 und EmptyLine1 : " + treeTest[3]); - result.add("Anzahl syntaktisch korrekter Trees ohne Whitespace mit MultiLine0 und EmptyLine0 : " + treeTest[4]); - result.add("Anzahl syntaktisch korrekter Trees ohne Whitespace mit MultiLine1 und EmptyLine0 : " + treeTest[5]); - result.add("Anzahl syntaktisch korrekter Trees ohne Whitespace mit MultiLine0 und EmptyLine1 : " + treeTest[6]); - result.add("Anzahl syntaktisch korrekter Trees ohne Whitespace mit MultiLine1 und EmptyLine1 : " + treeTest[7]); - result.add("Anzahl Fehler : " + error); + result.add("Anzahl geprüfter Diffs : " + count + "\n"); + result.add("Anzahl syntaktisch korrekter Diffs mit MultiLine0 und EmptyLine0 : " + diffTest[0] + "\n"); + result.add("Anzahl syntaktisch korrekter Diffs ohne Whitespace mit MultiLine0 und EmptyLine0 : " + diffTest[4] + + "\n"); + result.add("Anzahl semantisch korrekter Diffs mit MultiLine0 und EmptyLine0 : " + diffSemEqTest[0] + "\n"); + result.add("Anzahl von Diffs mit MultiLine0 und EmptyLine0, welche keine Korrektheitskriterium erfühlt haben : " + + (count - diffSemEqTest[0]) + "\n"); + + result.add("Anzahl syntaktisch korrekter Diffs mit MultiLine1 und EmptyLine0 : " + diffTest[1] + "\n"); + result.add("Anzahl syntaktisch korrekter Diffs ohne Whitespace mit MultiLine1 und EmptyLine0 : " + diffTest[5] + + "\n"); + result.add("Anzahl semantisch korrekter Diffs mit MultiLine1 und EmptyLine0 : " + diffSemEqTest[1] + "\n"); + result.add("Anzahl von Diffs mit MultiLine1 und EmptyLine0, welche keine Korrektheitskriterium erfühlt haben : " + + (count - diffSemEqTest[1]) + "\n"); + + result.add("Anzahl syntaktisch korrekter Diffs mit MultiLine0 und EmptyLine1 : " + diffTest[2] + "\n"); + result.add("Anzahl syntaktisch korrekter Diffs ohne Whitespace mit MultiLine0 und EmptyLine1 : " + diffTest[6] + + "\n"); + result.add("Anzahl semantisch korrekter Diffs mit MultiLine0 und EmptyLine1 : " + diffSemEqTest[2] + "\n"); + result.add("Anzahl von Diffs mit MultiLine0 und EmptyLine1, welche keine Korrektheitskriterium erfühlt haben : " + + (count - diffSemEqTest[2]) + "\n"); + + result.add("Anzahl syntaktisch korrekter Diffs mit MultiLine1 und EmptyLine1 : " + diffTest[3] + "\n"); + result.add("Anzahl syntaktisch korrekter Diffs ohne Whitespace mit MultiLine1 und EmptyLine1 : " + diffTest[7] + + "\n"); + result.add("Anzahl semantisch korrekter Diffs mit MultiLine1 und EmptyLine1 : " + diffSemEqTest[3] + "\n"); + result.add("Anzahl von Diffs mit MultiLine1 und EmptyLine1, welche keine Korrektheitskriterium erfühlt haben : " + + (count - diffSemEqTest[3]) + "\n"); + + result.add("-------------------------------------------------------------------------------------------"); + result.add("Anzahl geprüfter Trees : " + count * 2 + "\n"); + result.add("Anzahl syntaktisch korrekter Trees mit MultiLine0 und EmptyLine0 : " + treeTest[0] + "\n"); + result.add("Anzahl syntaktisch korrekter Trees ohne Whitespace mit MultiLine0 und EmptyLine0 : " + treeTest[4] + + "\n"); + result.add("Anzahl von Trees mit MultiLine0 und EmptyLine0, welche keine Korrektheitskriterium erfühlt haben : " + + (2 * count - treeTest[4]) + "\n"); + + result.add("Anzahl syntaktisch korrekter Trees mit MultiLine1 und EmptyLine0 : " + treeTest[1] + "\n"); + result.add("Anzahl syntaktisch korrekter Trees ohne Whitespace mit MultiLine1 und EmptyLine0 : " + treeTest[5] + + "\n"); + result.add("Anzahl von Trees mit MultiLine1 und EmptyLine0, welche keine Korrektheitskriterium erfühlt haben : " + + (2 * count - treeTest[5]) + "\n"); + + result.add("Anzahl syntaktisch korrekter Trees mit MultiLine0 und EmptyLine1 : " + treeTest[2] + "\n"); + result.add("Anzahl syntaktisch korrekter Trees ohne Whitespace mit MultiLine0 und EmptyLine1 : " + treeTest[6] + + "\n"); + result.add("Anzahl von Trees mit MultiLine0 und EmptyLine1, welche keine Korrektheitskriterium erfühlt haben : " + + (2 * count - treeTest[6]) + "\n"); + + result.add("Anzahl syntaktisch korrekter Trees mit MultiLine1 und EmptyLine1 : " + treeTest[3] + "\n"); + result.add("Anzahl syntaktisch korrekter Trees ohne Whitespace mit MultiLine1 und EmptyLine1 : " + treeTest[7] + + "\n"); + result.add("Anzahl von Trees mit MultiLine1 und EmptyLine1, welche keine Korrektheitskriterium erfühlt haben : " + + (2 * count - treeTest[7]) + "\n"); Files.write(Path.of("results", "thesis_es", "resultOfAnalysis.txt"), result); - int c = 1; - for (String line : errorList) { - Files.writeString(Path.of("results", "thesis_es", "errors", c + "error.txt"), line); - c = c + 1; - } } + private static String parseNumberStringToIntWithLengthGreaterOne(String string) { + string = string.trim(); + for (char c : string.toCharArray()) { + if (Character.isDigit(c)) { + return Character.toString(c); + } + } + return ""; + } } diff --git a/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java b/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java index 315e57139..516ff5cab 100644 --- a/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java +++ b/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java @@ -1,6 +1,8 @@ package org.variantsync.diffdetective.experiments.thesis_es; +import java.io.BufferedReader; import java.io.IOException; +import java.io.StringReader; import org.eclipse.jgit.diff.DiffAlgorithm.SupportedAlgorithm; import org.variantsync.diffdetective.analysis.Analysis; import org.variantsync.diffdetective.diff.git.PatchDiff; @@ -42,57 +44,131 @@ public void initializeResults(Analysis analysis) { public boolean analyzeVariationDiff(Analysis analysis) throws Exception { PatchDiff patch = analysis.getCurrentPatch(); String textDiff = patch.getDiff(); - String codeBefore = ""; - String codeAfter = ""; - /* - if (count < 10) { - Path pathTreeBefore = Paths.get("results", "thesis_es", "treeBefore" + count + ".txt"); - Path pathTreeAfter = Paths.get("results", "thesis_es", "treeAfter" + count + ".txt"); - Path pathDiff = Paths.get("results", "thesis_es", "diff" + count + ".txt"); - Files.writeString(pathTreeBefore, codeBefore); - Files.writeString(pathTreeAfter, codeAfter); - Files.writeString(pathDiff, textDiff); - count++; - } - */ - - codeBefore = VariationUnparser.undiff(patch.getDiff(), Time.BEFORE); - codeAfter = VariationUnparser.undiff(patch.getDiff(), Time.AFTER); - // codeBefore = codeBefore.replaceAll("\\r\\n", "\n"); - // codeAfter = codeAfter.replaceAll("\\r\\n", "\n"); + try { + BufferedReader in = new BufferedReader(new StringReader(textDiff)); + String line = ""; + StringBuilder tempString = new StringBuilder(); + while ((line = in.readLine()) != null) { + tempString.append(line); + tempString.append("\n"); + } + textDiff = tempString.toString(); + } catch (Exception e) { + e.printStackTrace(); + } + + String codeBefore = VariationUnparser.undiff(patch.getDiff(), Time.BEFORE); + String codeAfter = VariationUnparser.undiff(patch.getDiff(), Time.AFTER); + boolean[][] diffTestAll = runTestsDiff(textDiff); boolean[] treeBeforeTest = runTestsTree(codeBefore); boolean[] treeAfterTest = runTestsTree(codeAfter); boolean[] dataTests = runDataTest(textDiff, codeBefore, codeAfter); int error = 1; - String[] errorSave = new String[] { "", "", "" }; + // Test das es überhaupt funktioniert if (!(boolOr(diffTestAll[0]) || boolOr(diffTestAll[1]))) { error = error * 2; - // errorSave[0] = ; + reportErrorToFile(analysis, "textDiff: \n" + textDiff); } if (!boolOr(treeBeforeTest)) { error = error * 3; - reportErrorToFile(analysis, codeBefore); - // errorSave[1] = codeBefore; + reportErrorToFile(analysis, "treeBefore: \n" + codeBefore); } if (!boolOr(treeAfterTest)) { error = error * 5; - // errorSave[2] = codeAfter; + reportErrorToFile(analysis, "treeAfter: \n" + codeAfter); + } + // Testen der implikationen + // für Diff + // syntaktische Gleicheheit folgt syntaktische ohne Whitespace Gleichheit + // für CollapseMultyLine false und IgnoreEmptyLine false + if (diffTestAll[0][0] && !diffTestAll[0][4]) { + reportErrorToFile(analysis, "diffTestAll[0][0] && !diffTestAll[0][4] " + textDiff); + } + // syntaktische Gleicheheit ohne Whitespace folgt semantische Gleichheit + // für CollapseMultyLine false und IgnoreEmptyLine false + if (diffTestAll[0][4] && !diffTestAll[1][0]) { + reportErrorToFile(analysis, "diffTestAll[0][4] && !diffTestAll[1][0] " + textDiff); + } + // syntaktische Gleicheheit folgt syntaktische ohne Whitespace Gleichheit + // für CollapseMultyLine true und IgnoreEmptyLine false + if (diffTestAll[0][1] && !diffTestAll[0][5]) { + reportErrorToFile(analysis, "diffTestAll[0][1] && !diffTestAll[0][5] " + textDiff); + } + // syntaktische Gleicheheit ohne Whitespace folgt semantische Gleichheit + // für CollapseMultyLine true und IgnoreEmptyLine false + if (diffTestAll[0][5] && !diffTestAll[1][1]) { + reportErrorToFile(analysis, "diffTestAll[0][5] && !diffTestAll[1][1] " + textDiff); + } + // syntaktische Gleicheheit folgt syntaktische ohne Whitespace Gleichheit + // für CollapseMultyLine false und IgnoreEmptyLine true + if (diffTestAll[0][2] && !diffTestAll[0][6]) { + reportErrorToFile(analysis, "diffTestAll[0][2] && !diffTestAll[0][6] " + textDiff); + } + // syntaktische Gleicheheit ohne Whitespace folgt semantische Gleichheit + // für CollapseMultyLine false und IgnoreEmptyLine true + if (diffTestAll[0][6] && !diffTestAll[1][2]) { + reportErrorToFile(analysis, "diffTestAll[0][6] && !diffTestAll[1][2] " + textDiff); } - /* - if (!boolOr(dataTests)) { - error = error * 7; + // syntaktische Gleicheheit folgt syntaktische ohne Whitespace Gleichheit + // für CollapseMultyLine true und IgnoreEmptyLine true + if (diffTestAll[0][3] && !diffTestAll[0][7]) { + reportErrorToFile(analysis, "diffTestAll[0][3] && !diffTestAll[0][7] " + textDiff); + } + // syntaktische Gleicheheit ohne Whitespace folgt semantische Gleichheit + // für CollapseMultyLine true und IgnoreEmptyLine true + if (diffTestAll[0][7] && !diffTestAll[1][3]) { + reportErrorToFile(analysis, "diffTestAll[0][7] && !diffTestAll[1][3] " + textDiff); + } + // für Trees + // syntaktische Gleicheheit folgt syntaktische ohne Whitespace Gleichheit + // für CollapseMultyLine false und IgnoreEmptyLine false + if (treeBeforeTest[0] && !treeBeforeTest[4]) { + reportErrorToFile(analysis, "treeBeforeTest[0] && !treeBeforeTest[4] " + textDiff); + } + // syntaktische Gleicheheit folgt syntaktische ohne Whitespace Gleichheit + // für CollapseMultyLine true und IgnoreEmptyLine false + if (treeBeforeTest[1] && !treeBeforeTest[5]) { + reportErrorToFile(analysis, "treeBeforeTest[1] && !treeBeforeTest[5] " + textDiff); + } + // syntaktische Gleicheheit folgt syntaktische ohne Whitespace Gleichheit + // für CollapseMultyLine false und IgnoreEmptyLine true + if (treeBeforeTest[2] && !treeBeforeTest[6]) { + reportErrorToFile(analysis, "treeBeforeTest[2] && !treeBeforeTest[6] " + textDiff); + } + // syntaktische Gleicheheit folgt syntaktische ohne Whitespace Gleichheit + // für CollapseMultyLine true und IgnoreEmptyLine true + if (treeBeforeTest[3] && !treeBeforeTest[7]) { + reportErrorToFile(analysis, "treeBeforeTest[3] && !treeBeforeTest[7] " + textDiff); + } + // syntaktische Gleicheheit folgt syntaktische ohne Whitespace Gleichheit + // für CollapseMultyLine false und IgnoreEmptyLine false + if (treeAfterTest[0] && !treeAfterTest[4]) { + reportErrorToFile(analysis, "treeAfterTest[0] && !treeAfterTest[4] " + textDiff); + } + // syntaktische Gleicheheit folgt syntaktische ohne Whitespace Gleichheit + // für CollapseMultyLine true und IgnoreEmptyLine false + if (treeAfterTest[1] && !treeAfterTest[5]) { + reportErrorToFile(analysis, "treeAfterTest[1] && !treeAfterTest[5] " + textDiff); + } + // syntaktische Gleicheheit folgt syntaktische ohne Whitespace Gleichheit + // für CollapseMultyLine false und IgnoreEmptyLine true + if (treeAfterTest[2] && !treeAfterTest[6]) { + reportErrorToFile(analysis, "treeAfterTest[2] && !treeAfterTest[6] " + textDiff); + } + // syntaktische Gleicheheit folgt syntaktische ohne Whitespace Gleichheit + // für CollapseMultyLine true und IgnoreEmptyLine true + if (treeAfterTest[3] && !treeAfterTest[7]) { + reportErrorToFile(analysis, "treeAfterTest[3] && !treeAfterTest[7] " + textDiff); } - */ + // Speichere ergebniss final UnparseEvaluation ue = new UnparseEvaluation( boolToInt(dataTests), boolToInt(diffTestAll[0]), boolToInt(diffTestAll[1]), boolToInt(treeBeforeTest), - boolToInt(treeAfterTest), - error, - errorSave); + boolToInt(treeAfterTest)); csv.append(ue.toCSV()).append(StringUtils.LINEBREAK); return Analysis.Hooks.super.analyzeVariationDiff(analysis); } @@ -106,34 +182,29 @@ public void endBatch(Analysis analysis) throws IOException { private void reportErrorToFile(Analysis analysis, String errorMessage) throws IOException { IO.write( - FileUtils.addExtension(analysis.getOutputFile().resolve("_error" + errorCount), ".txt"), + FileUtils.addExtension(analysis.getOutputFile().resolve("error" + errorCount), ".txt"), errorMessage); ++errorCount; } - public static String removeWhitespace1(String string) { - string = string.replaceAll("\n(\\s*\n)+", "\n"); - string = string.replaceAll("\n\\s+", "\n"); - if (!string.isEmpty()) { - if (string.charAt(string.length() - 1) == '\n') { - return string.substring(0, string.length() - 1); - } - } - return string; - } - public static String removeWhitespace(String string) { if (string.isEmpty()) { return ""; } else { StringBuilder result = new StringBuilder(); - for (String line : string.split("\n")) { - if (!line.replaceAll("\\s+", "").isEmpty()) { - result.append(line.trim()); - result.append("\n"); + try { + BufferedReader in = new BufferedReader(new StringReader(string)); + String line = ""; + while ((line = in.readLine()) != null) { + if (!line.replaceAll("\\s+", "").isEmpty()) { + result.append(line.trim()); + result.append("\n"); + } } + } catch (Exception e) { + e.printStackTrace(); } - return result.substring(0, result.length() - 1); + return result.toString(); } } diff --git a/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseEvaluation.java b/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseEvaluation.java index 41c6328e2..ba0ff3ba5 100644 --- a/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseEvaluation.java +++ b/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseEvaluation.java @@ -9,9 +9,7 @@ public record UnparseEvaluation( int[] diffTest, int[] diffSemEqTest, int[] treeBeforeTest, - int[] treeAfterTest, - int error, - String[] errorSave) implements CSV { + int[] treeAfterTest) implements CSV { @Override public String toCSV(String delimiter) { @@ -51,11 +49,7 @@ public String toCSV(String delimiter) { treeAfterTest[4], treeAfterTest[5], treeAfterTest[6], - treeAfterTest[7], - error, - errorSave[0], - errorSave[1], - errorSave[2]); + treeAfterTest[7]); } public static String makeHeader(String delimiter) { @@ -95,11 +89,7 @@ public static String makeHeader(String delimiter) { "treeAfterEqTestMultiL0EmptyL0White", "treeAfterEqTestMultiL1EmptyL0White", "treeAfterEqTestMultiL0EmptyL1White", - "treeAfterEqTestMultiL1EmptyL1White", - "errorTyp", - "errorData1", - "errorData2", - "errorData3"); + "treeAfterEqTestMultiL1EmptyL1White"); } } From 53b82e19e390850f42e28fa091ec274f76e3ff1f Mon Sep 17 00:00:00 2001 From: eshul Date: Sat, 21 Dec 2024 15:52:04 +0100 Subject: [PATCH 022/126] test: add a test for comparing unparsed diffs semantically --- src/test/java/VariationUnparserTest.java | 27 +++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/test/java/VariationUnparserTest.java b/src/test/java/VariationUnparserTest.java index 857c33b26..a0b32c662 100644 --- a/src/test/java/VariationUnparserTest.java +++ b/src/test/java/VariationUnparserTest.java @@ -56,7 +56,7 @@ public void testDiffDir(Path basename) throws IOException, DiffParseException { } @Test - public void testDiff() { + public void testTree() { String source = ""; String temp = ""; try { @@ -74,6 +74,31 @@ public void testDiff() { } } + @Test + public void testDiffSemEq() { + String source = ""; + String temp = ""; + try { + + source = Files + .readString(Path.of("src", "test", "resources", "unparser", "diff", "diff.diff")); + VariationDiff diff = VariationDiff.fromDiff(source, VariationDiffParseOptions.Default); + temp = VariationUnparser.variationDiffUnparser(diff); + System.out.println(removeWhitespace(source).equals(removeWhitespace(temp))); + System.out.println(removeWhitespace(VariationUnparser.undiff(source, Time.BEFORE)) + .equals(removeWhitespace(VariationUnparser.undiff(temp, Time.BEFORE)))); + System.out.println(removeWhitespace(VariationUnparser.undiff(source, Time.AFTER)) + .equals(removeWhitespace(VariationUnparser.undiff(temp, Time.AFTER)))); + + // System.out.println(removeWhitespace(source)); + // System.out.println("Ende"); + // System.out.println(removeWhitespace(temp)); + // System.out.println("Ende"); + } catch (Exception e) { + e.printStackTrace(); + } + } + public static void testCaseTree(Path testCasePath) { String temp = ""; try { From ff4699922fdcfc7619bc4e268fe3f62329304203 Mon Sep 17 00:00:00 2001 From: eshul Date: Mon, 10 Feb 2025 10:21:35 +0100 Subject: [PATCH 023/126] fix: fix removeWhitespace for diffs --- .../thesis_es/UnparseAnalysis.java | 43 +++++++++-------- src/test/java/VariationUnparserTest.java | 48 +++++++++---------- 2 files changed, 48 insertions(+), 43 deletions(-) diff --git a/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java b/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java index 516ff5cab..a1fedae95 100644 --- a/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java +++ b/src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java @@ -187,7 +187,7 @@ private void reportErrorToFile(Analysis analysis, String errorMessage) throws IO ++errorCount; } - public static String removeWhitespace(String string) { + public static String removeWhitespace(String string, boolean diff) { if (string.isEmpty()) { return ""; } else { @@ -197,7 +197,11 @@ public static String removeWhitespace(String string) { String line = ""; while ((line = in.readLine()) != null) { if (!line.replaceAll("\\s+", "").isEmpty()) { - result.append(line.trim()); + String temp = line.trim(); + if (diff && !(temp.charAt(0) == '+' || temp.charAt(0) == '-')) { + temp = " " + temp; + } + result.append(temp); result.append("\n"); } } @@ -242,11 +246,11 @@ public static VariationDiffParseOptions optionsSetter(int i) { } } - public static boolean equalsText(String text1, String text2, boolean whitespace) { + public static boolean equalsText(String text1, String text2, boolean whitespace, boolean diff) { if (whitespace) { return text1.equals(text2); } else { - return removeWhitespace(text1).equals(removeWhitespace(text2)); + return removeWhitespace(text1, diff).equals(removeWhitespace(text2, diff)); } } @@ -254,16 +258,16 @@ public static boolean[][] runTestsDiff(String text) { boolean[][] array = new boolean[2][8]; for (int i = 0; i < 4; i++) { String diff = parseUnparseDiff(text, optionsSetter(i)); - array[0][i] = equalsText(text, diff, true); - array[0][i + 4] = equalsText(text, diff, false); + array[0][i] = equalsText(text, diff, true, true); + array[0][i + 4] = equalsText(text, diff, false, true); array[1][i] = (equalsText(VariationUnparser.undiff(text, Time.BEFORE), - VariationUnparser.undiff(diff, Time.BEFORE), true) + VariationUnparser.undiff(diff, Time.BEFORE), true, true) && equalsText(VariationUnparser.undiff(text, Time.AFTER), - VariationUnparser.undiff(diff, Time.AFTER), true)) + VariationUnparser.undiff(diff, Time.AFTER), true, true)) || (equalsText(VariationUnparser.undiff(text, Time.BEFORE), - VariationUnparser.undiff(diff, Time.BEFORE), false) + VariationUnparser.undiff(diff, Time.BEFORE), false, true) && equalsText(VariationUnparser.undiff(text, Time.AFTER), - VariationUnparser.undiff(diff, Time.AFTER), false)); + VariationUnparser.undiff(diff, Time.AFTER), false, true)); array[1][i + 4] = false; } return array; @@ -273,8 +277,8 @@ public static boolean[] runTestsTree(String text) { boolean[] array = new boolean[8]; for (int i = 0; i < 4; i++) { String temp = parseUnparseTree(text, optionsSetter(i)); - array[i] = equalsText(text, temp, true); - array[i + 4] = equalsText(text, temp, false); + array[i] = equalsText(text, temp, true, false); + array[i + 4] = equalsText(text, temp, false, false); } return array; } @@ -282,16 +286,17 @@ public static boolean[] runTestsTree(String text) { public static boolean[] runDataTest(String textDiff, String treeBefore, String treeAfter) throws IOException { boolean[] array = new boolean[8]; array[0] = JGitDiff.textDiff(treeBefore, treeAfter, SupportedAlgorithm.MYERS).equals(textDiff); - array[1] = removeWhitespace(JGitDiff.textDiff(treeBefore, treeAfter, SupportedAlgorithm.MYERS)) - .equals(removeWhitespace(textDiff)); + array[1] = removeWhitespace(JGitDiff.textDiff(treeBefore, treeAfter, SupportedAlgorithm.MYERS), true) + .equals(removeWhitespace(textDiff, true)); array[2] = JGitDiff.textDiff(treeBefore, treeAfter, SupportedAlgorithm.HISTOGRAM).equals(textDiff); - array[3] = removeWhitespace(JGitDiff.textDiff(treeBefore, treeAfter, SupportedAlgorithm.HISTOGRAM)) - .equals(removeWhitespace(textDiff)); + array[3] = removeWhitespace(JGitDiff.textDiff(treeBefore, treeAfter, SupportedAlgorithm.HISTOGRAM), true) + .equals(removeWhitespace(textDiff, true)); array[4] = treeBefore.equals(VariationUnparser.undiff(textDiff, Time.BEFORE)); - array[5] = removeWhitespace(treeBefore) - .equals(removeWhitespace(VariationUnparser.undiff(textDiff, Time.BEFORE))); + array[5] = removeWhitespace(treeBefore, false) + .equals(removeWhitespace(VariationUnparser.undiff(textDiff, Time.BEFORE), false)); array[6] = treeAfter.equals(VariationUnparser.undiff(textDiff, Time.AFTER)); - array[7] = removeWhitespace(treeAfter).equals(removeWhitespace(VariationUnparser.undiff(textDiff, Time.AFTER))); + array[7] = removeWhitespace(treeAfter, false) + .equals(removeWhitespace(VariationUnparser.undiff(textDiff, Time.AFTER), false)); return array; } diff --git a/src/test/java/VariationUnparserTest.java b/src/test/java/VariationUnparserTest.java index a0b32c662..62474655e 100644 --- a/src/test/java/VariationUnparserTest.java +++ b/src/test/java/VariationUnparserTest.java @@ -64,7 +64,7 @@ public void testTree() { VariationTree tree = VariationTree.fromText(source, VariationTreeSource.Unknown, VariationDiffParseOptions.Default); temp = VariationUnparser.variationTreeUnparser(tree); - System.out.println(removeWhitespace(source).equals(removeWhitespace(temp))); + System.out.println(removeWhitespace(source, false).equals(removeWhitespace(temp, false))); // System.out.println(removeWhitespace(source)); // System.out.println("Ende"); // System.out.println(removeWhitespace(temp)); @@ -84,11 +84,11 @@ public void testDiffSemEq() { .readString(Path.of("src", "test", "resources", "unparser", "diff", "diff.diff")); VariationDiff diff = VariationDiff.fromDiff(source, VariationDiffParseOptions.Default); temp = VariationUnparser.variationDiffUnparser(diff); - System.out.println(removeWhitespace(source).equals(removeWhitespace(temp))); - System.out.println(removeWhitespace(VariationUnparser.undiff(source, Time.BEFORE)) - .equals(removeWhitespace(VariationUnparser.undiff(temp, Time.BEFORE)))); - System.out.println(removeWhitespace(VariationUnparser.undiff(source, Time.AFTER)) - .equals(removeWhitespace(VariationUnparser.undiff(temp, Time.AFTER)))); + System.out.println(removeWhitespace(source, true).equals(removeWhitespace(temp, true))); + System.out.println(removeWhitespace(VariationUnparser.undiff(source, Time.BEFORE), false) + .equals(removeWhitespace(VariationUnparser.undiff(temp, Time.BEFORE), false))); + System.out.println(removeWhitespace(VariationUnparser.undiff(source, Time.AFTER), false) + .equals(removeWhitespace(VariationUnparser.undiff(temp, Time.AFTER), false))); // System.out.println(removeWhitespace(source)); // System.out.println("Ende"); @@ -114,11 +114,11 @@ public static void testCaseTree(Path testCasePath) { String unparse4 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(true, true)); System.out.println(temp.equals(unparse1) + " " + temp.equals(unparse2) + " " + temp.equals(unparse3) + " " + temp.equals(unparse4)); - temp = removeWhitespace(temp); - unparse1 = removeWhitespace(unparse1); - unparse2 = removeWhitespace(unparse2); - unparse3 = removeWhitespace(unparse3); - unparse4 = removeWhitespace(unparse4); + temp = removeWhitespace(temp, false); + unparse1 = removeWhitespace(unparse1, false); + unparse2 = removeWhitespace(unparse2, false); + unparse3 = removeWhitespace(unparse3, false); + unparse4 = removeWhitespace(unparse4, false); System.out.println(temp.equals(unparse1) + " " + temp.equals(unparse2) + " " + temp.equals(unparse3) + " " + temp.equals(unparse4)); } @@ -151,19 +151,19 @@ public static void testCaseDiff(Path testCasePath) { System.out.println(temp1.equals(unparse11) + " " + temp2.equals(unparse12) + " " + temp1.equals(unparse21) + " " + temp2.equals(unparse22) + " " + temp1.equals(unparse31) + " " + temp2.equals(unparse32) + " " + temp1.equals(unparse41) + " " + temp2.equals(unparse42)); - System.out.println(removeWhitespace(temp1).equals(removeWhitespace(unparse11)) + " " - + removeWhitespace(temp2).equals(removeWhitespace(unparse12)) + " " - + removeWhitespace(temp1).equals(removeWhitespace(unparse21)) + " " - + removeWhitespace(temp2).equals(removeWhitespace(unparse22)) + " " - + removeWhitespace(temp1).equals(removeWhitespace(unparse31)) + " " - + removeWhitespace(temp2).equals(removeWhitespace(unparse32)) + " " - + removeWhitespace(temp1).equals(removeWhitespace(unparse41)) + " " - + removeWhitespace(temp2).equals(removeWhitespace(unparse42))); - temp = removeWhitespace(temp); - unparse1 = removeWhitespace(unparse1); - unparse2 = removeWhitespace(unparse2); - unparse3 = removeWhitespace(unparse3); - unparse4 = removeWhitespace(unparse4); + System.out.println(removeWhitespace(temp1, false).equals(removeWhitespace(unparse11, false)) + " " + + removeWhitespace(temp2, false).equals(removeWhitespace(unparse12, false)) + " " + + removeWhitespace(temp1, false).equals(removeWhitespace(unparse21, false)) + " " + + removeWhitespace(temp2, false).equals(removeWhitespace(unparse22, false)) + " " + + removeWhitespace(temp1, false).equals(removeWhitespace(unparse31, false)) + " " + + removeWhitespace(temp2, false).equals(removeWhitespace(unparse32, false)) + " " + + removeWhitespace(temp1, false).equals(removeWhitespace(unparse41, false)) + " " + + removeWhitespace(temp2, false).equals(removeWhitespace(unparse42, false))); + temp = removeWhitespace(temp, true); + unparse1 = removeWhitespace(unparse1, true); + unparse2 = removeWhitespace(unparse2, true); + unparse3 = removeWhitespace(unparse3, true); + unparse4 = removeWhitespace(unparse4, true); System.out.println(temp.equals(unparse1) + " " + temp.equals(unparse2) + " " + temp.equals(unparse3) + " " + temp.equals(unparse4)); } From 6ac344067f3b9a3828cafdcc72d81fe2056134a3 Mon Sep 17 00:00:00 2001 From: Benjamin Moosherr Date: Sat, 9 Aug 2025 23:20:18 +0200 Subject: [PATCH 024/126] test: fail tests if an exception is thrown --- src/test/java/VariationUnparserTest.java | 118 ++++++++--------------- 1 file changed, 39 insertions(+), 79 deletions(-) diff --git a/src/test/java/VariationUnparserTest.java b/src/test/java/VariationUnparserTest.java index 62474655e..8fbe1fb92 100644 --- a/src/test/java/VariationUnparserTest.java +++ b/src/test/java/VariationUnparserTest.java @@ -24,15 +24,10 @@ public class VariationUnparserTest { private final static String testCaseSuffixDiff = ".diff"; - protected static Stream findTestCases(Path dir, String testCaseSuffix) { - try { - return Files - .list(dir) - .filter(filename -> filename.getFileName().toString().endsWith(testCaseSuffix)); - } catch (Exception e) { - e.printStackTrace(); - } - return null; + protected static Stream findTestCases(Path dir, String testCaseSuffix) throws IOException { + return Files + .list(dir) + .filter(filename -> filename.getFileName().toString().endsWith(testCaseSuffix)); } public static Stream testsTree() throws IOException { @@ -56,56 +51,38 @@ public void testDiffDir(Path basename) throws IOException, DiffParseException { } @Test - public void testTree() { - String source = ""; - String temp = ""; - try { - source = Files.readString(Path.of("src", "test", "resources", "unparser", "test8.txt")); - VariationTree tree = VariationTree.fromText(source, VariationTreeSource.Unknown, - VariationDiffParseOptions.Default); - temp = VariationUnparser.variationTreeUnparser(tree); - System.out.println(removeWhitespace(source, false).equals(removeWhitespace(temp, false))); - // System.out.println(removeWhitespace(source)); - // System.out.println("Ende"); - // System.out.println(removeWhitespace(temp)); - // System.out.println("Ende"); - } catch (Exception e) { - e.printStackTrace(); - } + public void testTree() throws IOException, DiffParseException { + String source = Files.readString(Path.of("src", "test", "resources", "unparser", "test8.txt")); + VariationTree tree = VariationTree.fromText(source, VariationTreeSource.Unknown, + VariationDiffParseOptions.Default); + String temp = VariationUnparser.variationTreeUnparser(tree); + System.out.println(removeWhitespace(source, false).equals(removeWhitespace(temp, false))); + // System.out.println(removeWhitespace(source)); + // System.out.println("Ende"); + // System.out.println(removeWhitespace(temp)); + // System.out.println("Ende"); } @Test - public void testDiffSemEq() { - String source = ""; - String temp = ""; - try { - - source = Files - .readString(Path.of("src", "test", "resources", "unparser", "diff", "diff.diff")); - VariationDiff diff = VariationDiff.fromDiff(source, VariationDiffParseOptions.Default); - temp = VariationUnparser.variationDiffUnparser(diff); - System.out.println(removeWhitespace(source, true).equals(removeWhitespace(temp, true))); - System.out.println(removeWhitespace(VariationUnparser.undiff(source, Time.BEFORE), false) - .equals(removeWhitespace(VariationUnparser.undiff(temp, Time.BEFORE), false))); - System.out.println(removeWhitespace(VariationUnparser.undiff(source, Time.AFTER), false) - .equals(removeWhitespace(VariationUnparser.undiff(temp, Time.AFTER), false))); - - // System.out.println(removeWhitespace(source)); - // System.out.println("Ende"); - // System.out.println(removeWhitespace(temp)); - // System.out.println("Ende"); - } catch (Exception e) { - e.printStackTrace(); - } + public void testDiffSemEq() throws IOException, DiffParseException { + String source = Files + .readString(Path.of("src", "test", "resources", "unparser", "diff", "diff.diff")); + VariationDiff diff = VariationDiff.fromDiff(source, VariationDiffParseOptions.Default); + String temp = VariationUnparser.variationDiffUnparser(diff); + System.out.println(removeWhitespace(source, true).equals(removeWhitespace(temp, true))); + System.out.println(removeWhitespace(VariationUnparser.undiff(source, Time.BEFORE), false) + .equals(removeWhitespace(VariationUnparser.undiff(temp, Time.BEFORE), false))); + System.out.println(removeWhitespace(VariationUnparser.undiff(source, Time.AFTER), false) + .equals(removeWhitespace(VariationUnparser.undiff(temp, Time.AFTER), false))); + + // System.out.println(removeWhitespace(source)); + // System.out.println("Ende"); + // System.out.println(removeWhitespace(temp)); + // System.out.println("Ende"); } - public static void testCaseTree(Path testCasePath) { - String temp = ""; - try { - temp = Files.readString(testCasePath); - } catch (Exception e) { - e.printStackTrace(); - } + public static void testCaseTree(Path testCasePath) throws IOException, DiffParseException { + String temp = Files.readString(testCasePath); System.out.println(testCasePath); temp = temp.replaceAll("\\r\\n", "\n"); String unparse1 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(false, false)); @@ -123,13 +100,8 @@ public static void testCaseTree(Path testCasePath) { + temp.equals(unparse4)); } - public static void testCaseDiff(Path testCasePath) { - String temp = ""; - try { - temp = Files.readString(testCasePath); - } catch (Exception e) { - e.printStackTrace(); - } + public static void testCaseDiff(Path testCasePath) throws IOException, DiffParseException { + String temp = Files.readString(testCasePath); System.out.println(testCasePath); temp = temp.replaceAll("\\r\\n", "\n"); String unparse1 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(false, false)); @@ -168,25 +140,13 @@ public static void testCaseDiff(Path testCasePath) { + temp.equals(unparse4)); } - public static String parseUnparseTree(Path path, VariationDiffParseOptions option) { - String temp = "b"; - try { - VariationTree tree = VariationTree.fromFile(path, option); - temp = VariationUnparser.variationTreeUnparser(tree); - } catch (Exception e) { - e.printStackTrace(); - } - return temp; + public static String parseUnparseTree(Path path, VariationDiffParseOptions option) throws IOException, DiffParseException { + VariationTree tree = VariationTree.fromFile(path, option); + return VariationUnparser.variationTreeUnparser(tree); } - public static String parseUnparseDiff(Path path, VariationDiffParseOptions option) { - String temp = "b"; - try { - VariationDiff diff = VariationDiff.fromFile(path, option); - temp = VariationUnparser.variationDiffUnparser(diff); - } catch (Exception e) { - e.printStackTrace(); - } - return temp; + public static String parseUnparseDiff(Path path, VariationDiffParseOptions option) throws IOException, DiffParseException { + VariationDiff diff = VariationDiff.fromFile(path, option); + return VariationUnparser.variationDiffUnparser(diff); } } From d18386fb71b9314a3d2a8cab14e9b201528bf088 Mon Sep 17 00:00:00 2001 From: Benjamin Moosherr Date: Sun, 10 Aug 2025 00:36:59 +0200 Subject: [PATCH 025/126] test: use asserts instead of manually checking stdout Note that some printed checks failed expectedly (those that didn't ignore whitespace or didn't compare diffs semantically) and thus are removed without replacement. --- src/test/java/VariationUnparserTest.java | 73 ++++++++++-------------- 1 file changed, 30 insertions(+), 43 deletions(-) diff --git a/src/test/java/VariationUnparserTest.java b/src/test/java/VariationUnparserTest.java index 8fbe1fb92..48384ab44 100644 --- a/src/test/java/VariationUnparserTest.java +++ b/src/test/java/VariationUnparserTest.java @@ -14,6 +14,7 @@ import org.variantsync.diffdetective.variation.VariationUnparser; import org.variantsync.diffdetective.variation.tree.source.VariationTreeSource; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.variantsync.diffdetective.experiments.thesis_es.UnparseAnalysis.removeWhitespace; public class VariationUnparserTest { @@ -56,11 +57,8 @@ public void testTree() throws IOException, DiffParseException { VariationTree tree = VariationTree.fromText(source, VariationTreeSource.Unknown, VariationDiffParseOptions.Default); String temp = VariationUnparser.variationTreeUnparser(tree); - System.out.println(removeWhitespace(source, false).equals(removeWhitespace(temp, false))); - // System.out.println(removeWhitespace(source)); - // System.out.println("Ende"); - // System.out.println(removeWhitespace(temp)); - // System.out.println("Ende"); + + assertEquals(removeWhitespace(source, false), removeWhitespace(temp, false)); } @Test @@ -69,47 +67,45 @@ public void testDiffSemEq() throws IOException, DiffParseException { .readString(Path.of("src", "test", "resources", "unparser", "diff", "diff.diff")); VariationDiff diff = VariationDiff.fromDiff(source, VariationDiffParseOptions.Default); String temp = VariationUnparser.variationDiffUnparser(diff); - System.out.println(removeWhitespace(source, true).equals(removeWhitespace(temp, true))); - System.out.println(removeWhitespace(VariationUnparser.undiff(source, Time.BEFORE), false) - .equals(removeWhitespace(VariationUnparser.undiff(temp, Time.BEFORE), false))); - System.out.println(removeWhitespace(VariationUnparser.undiff(source, Time.AFTER), false) - .equals(removeWhitespace(VariationUnparser.undiff(temp, Time.AFTER), false))); - - // System.out.println(removeWhitespace(source)); - // System.out.println("Ende"); - // System.out.println(removeWhitespace(temp)); - // System.out.println("Ende"); + + assertEquals( + removeWhitespace(VariationUnparser.undiff(source, Time.BEFORE), false), + removeWhitespace(VariationUnparser.undiff(temp, Time.BEFORE), false)); + assertEquals( + removeWhitespace(VariationUnparser.undiff(source, Time.AFTER), false), + removeWhitespace(VariationUnparser.undiff(temp, Time.AFTER), false)); } public static void testCaseTree(Path testCasePath) throws IOException, DiffParseException { String temp = Files.readString(testCasePath); - System.out.println(testCasePath); temp = temp.replaceAll("\\r\\n", "\n"); + String unparse1 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(false, false)); String unparse2 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(false, true)); String unparse3 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(true, false)); String unparse4 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(true, true)); - System.out.println(temp.equals(unparse1) + " " + temp.equals(unparse2) + " " + temp.equals(unparse3) + " " - + temp.equals(unparse4)); + temp = removeWhitespace(temp, false); unparse1 = removeWhitespace(unparse1, false); unparse2 = removeWhitespace(unparse2, false); unparse3 = removeWhitespace(unparse3, false); unparse4 = removeWhitespace(unparse4, false); - System.out.println(temp.equals(unparse1) + " " + temp.equals(unparse2) + " " + temp.equals(unparse3) + " " - + temp.equals(unparse4)); + + assertEquals(temp, unparse1); + assertEquals(temp, unparse2); + assertEquals(temp, unparse3); + assertEquals(temp, unparse4); } public static void testCaseDiff(Path testCasePath) throws IOException, DiffParseException { String temp = Files.readString(testCasePath); - System.out.println(testCasePath); temp = temp.replaceAll("\\r\\n", "\n"); + String unparse1 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(false, false)); String unparse2 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(false, true)); String unparse3 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(true, false)); String unparse4 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(true, true)); - System.out.println(temp.equals(unparse1) + " " + temp.equals(unparse2) + " " + temp.equals(unparse3) + " " - + temp.equals(unparse4)); + String temp1 = VariationUnparser.undiff(temp, Time.BEFORE); String temp2 = VariationUnparser.undiff(temp, Time.AFTER); String unparse11 = VariationUnparser.undiff(unparse1, Time.BEFORE); @@ -118,26 +114,17 @@ public static void testCaseDiff(Path testCasePath) throws IOException, DiffParse String unparse22 = VariationUnparser.undiff(unparse2, Time.AFTER); String unparse31 = VariationUnparser.undiff(unparse3, Time.BEFORE); String unparse32 = VariationUnparser.undiff(unparse3, Time.AFTER); - String unparse41 = VariationUnparser.undiff(unparse1, Time.BEFORE); - String unparse42 = VariationUnparser.undiff(unparse1, Time.AFTER); - System.out.println(temp1.equals(unparse11) + " " + temp2.equals(unparse12) + " " + temp1.equals(unparse21) + " " - + temp2.equals(unparse22) + " " + temp1.equals(unparse31) + " " + temp2.equals(unparse32) + " " - + temp1.equals(unparse41) + " " + temp2.equals(unparse42)); - System.out.println(removeWhitespace(temp1, false).equals(removeWhitespace(unparse11, false)) + " " - + removeWhitespace(temp2, false).equals(removeWhitespace(unparse12, false)) + " " - + removeWhitespace(temp1, false).equals(removeWhitespace(unparse21, false)) + " " - + removeWhitespace(temp2, false).equals(removeWhitespace(unparse22, false)) + " " - + removeWhitespace(temp1, false).equals(removeWhitespace(unparse31, false)) + " " - + removeWhitespace(temp2, false).equals(removeWhitespace(unparse32, false)) + " " - + removeWhitespace(temp1, false).equals(removeWhitespace(unparse41, false)) + " " - + removeWhitespace(temp2, false).equals(removeWhitespace(unparse42, false))); - temp = removeWhitespace(temp, true); - unparse1 = removeWhitespace(unparse1, true); - unparse2 = removeWhitespace(unparse2, true); - unparse3 = removeWhitespace(unparse3, true); - unparse4 = removeWhitespace(unparse4, true); - System.out.println(temp.equals(unparse1) + " " + temp.equals(unparse2) + " " + temp.equals(unparse3) + " " - + temp.equals(unparse4)); + String unparse41 = VariationUnparser.undiff(unparse4, Time.BEFORE); + String unparse42 = VariationUnparser.undiff(unparse4, Time.AFTER); + + assertEquals(removeWhitespace(temp1, false), removeWhitespace(unparse11, false)); + assertEquals(removeWhitespace(temp2, false), removeWhitespace(unparse12, false)); + assertEquals(removeWhitespace(temp1, false), removeWhitespace(unparse21, false)); + assertEquals(removeWhitespace(temp2, false), removeWhitespace(unparse22, false)); + assertEquals(removeWhitespace(temp1, false), removeWhitespace(unparse31, false)); + assertEquals(removeWhitespace(temp2, false), removeWhitespace(unparse32, false)); + assertEquals(removeWhitespace(temp1, false), removeWhitespace(unparse41, false)); + assertEquals(removeWhitespace(temp2, false), removeWhitespace(unparse42, false)); } public static String parseUnparseTree(Path path, VariationDiffParseOptions option) throws IOException, DiffParseException { From 0a35f6228de601af66d713279f3af137e89429b9 Mon Sep 17 00:00:00 2001 From: Benjamin Moosherr Date: Sun, 10 Aug 2025 00:49:37 +0200 Subject: [PATCH 026/126] test: remove unnecessary wrappers around unparser --- src/test/java/VariationUnparserTest.java | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/test/java/VariationUnparserTest.java b/src/test/java/VariationUnparserTest.java index 48384ab44..16ef3b66c 100644 --- a/src/test/java/VariationUnparserTest.java +++ b/src/test/java/VariationUnparserTest.java @@ -39,18 +39,6 @@ public static Stream testsDiff() throws IOException { return findTestCases(testDirDiff, testCaseSuffixDiff); } - @ParameterizedTest - @MethodSource("testsTree") - public void testTreeDir(Path basename) throws IOException, DiffParseException { - testCaseTree(basename); - } - - @ParameterizedTest - @MethodSource("testsDiff") - public void testDiffDir(Path basename) throws IOException, DiffParseException { - testCaseDiff(basename); - } - @Test public void testTree() throws IOException, DiffParseException { String source = Files.readString(Path.of("src", "test", "resources", "unparser", "test8.txt")); @@ -76,7 +64,9 @@ public void testDiffSemEq() throws IOException, DiffParseException { removeWhitespace(VariationUnparser.undiff(temp, Time.AFTER), false)); } - public static void testCaseTree(Path testCasePath) throws IOException, DiffParseException { + @ParameterizedTest + @MethodSource("testsTree") + public void testCaseTree(Path testCasePath) throws IOException, DiffParseException { String temp = Files.readString(testCasePath); temp = temp.replaceAll("\\r\\n", "\n"); @@ -97,7 +87,9 @@ public static void testCaseTree(Path testCasePath) throws IOException, DiffParse assertEquals(temp, unparse4); } - public static void testCaseDiff(Path testCasePath) throws IOException, DiffParseException { + @ParameterizedTest + @MethodSource("testsDiff") + public void testCaseDiff(Path testCasePath) throws IOException, DiffParseException { String temp = Files.readString(testCasePath); temp = temp.replaceAll("\\r\\n", "\n"); From b9fa82b4c763ed028638e2bc3d85da198575305d Mon Sep 17 00:00:00 2001 From: Benjamin Moosherr Date: Sun, 10 Aug 2025 00:51:12 +0200 Subject: [PATCH 027/126] test: reuse directory constants in the unparsing tests --- src/test/java/VariationUnparserTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/VariationUnparserTest.java b/src/test/java/VariationUnparserTest.java index 16ef3b66c..01c57e790 100644 --- a/src/test/java/VariationUnparserTest.java +++ b/src/test/java/VariationUnparserTest.java @@ -41,7 +41,7 @@ public static Stream testsDiff() throws IOException { @Test public void testTree() throws IOException, DiffParseException { - String source = Files.readString(Path.of("src", "test", "resources", "unparser", "test8.txt")); + String source = Files.readString(testDirTree.resolve("test8.txt")); VariationTree tree = VariationTree.fromText(source, VariationTreeSource.Unknown, VariationDiffParseOptions.Default); String temp = VariationUnparser.variationTreeUnparser(tree); @@ -52,7 +52,7 @@ public void testTree() throws IOException, DiffParseException { @Test public void testDiffSemEq() throws IOException, DiffParseException { String source = Files - .readString(Path.of("src", "test", "resources", "unparser", "diff", "diff.diff")); + .readString(testDirTree.resolve("diff").resolve("diff.diff")); VariationDiff diff = VariationDiff.fromDiff(source, VariationDiffParseOptions.Default); String temp = VariationUnparser.variationDiffUnparser(diff); From bb590584492f0c31824d270ea385b4bababe9c7e Mon Sep 17 00:00:00 2001 From: Benjamin Moosherr Date: Sun, 10 Aug 2025 00:59:02 +0200 Subject: [PATCH 028/126] test: improve the variable names in the unparse tests --- src/test/java/VariationUnparserTest.java | 135 +++++++++++------------ 1 file changed, 66 insertions(+), 69 deletions(-) diff --git a/src/test/java/VariationUnparserTest.java b/src/test/java/VariationUnparserTest.java index 01c57e790..6058652a3 100644 --- a/src/test/java/VariationUnparserTest.java +++ b/src/test/java/VariationUnparserTest.java @@ -18,105 +18,102 @@ import static org.variantsync.diffdetective.experiments.thesis_es.UnparseAnalysis.removeWhitespace; public class VariationUnparserTest { - private final static Path testDirTree = Constants.RESOURCE_DIR.resolve("unparser"); - + private final static Path treeDir = Constants.RESOURCE_DIR.resolve("unparser"); private final static Path testDirDiff = Constants.RESOURCE_DIR.resolve("diffs").resolve("parser"); - private final static String testCaseSuffixTree = ".txt"; - - private final static String testCaseSuffixDiff = ".diff"; + private final static String treeSuffix = ".txt"; + private final static String diffSuffix = ".diff"; - protected static Stream findTestCases(Path dir, String testCaseSuffix) throws IOException { + protected static Stream findTestCases(Path dir, String filenameSuffix) throws IOException { return Files .list(dir) - .filter(filename -> filename.getFileName().toString().endsWith(testCaseSuffix)); + .filter(filename -> filename.getFileName().toString().endsWith(filenameSuffix)); } public static Stream testsTree() throws IOException { - return findTestCases(testDirTree, testCaseSuffixTree); + return findTestCases(treeDir, treeSuffix); } public static Stream testsDiff() throws IOException { - return findTestCases(testDirDiff, testCaseSuffixDiff); + return findTestCases(testDirDiff, diffSuffix); } @Test public void testTree() throws IOException, DiffParseException { - String source = Files.readString(testDirTree.resolve("test8.txt")); - VariationTree tree = VariationTree.fromText(source, VariationTreeSource.Unknown, - VariationDiffParseOptions.Default); - String temp = VariationUnparser.variationTreeUnparser(tree); + String original = Files.readString(treeDir.resolve("test8.txt")); + VariationTree tree = + VariationTree.fromText(original, VariationTreeSource.Unknown, VariationDiffParseOptions.Default); + String unparsed = VariationUnparser.variationTreeUnparser(tree); - assertEquals(removeWhitespace(source, false), removeWhitespace(temp, false)); + assertEquals(removeWhitespace(original, false), removeWhitespace(unparsed, false)); } @Test public void testDiffSemEq() throws IOException, DiffParseException { - String source = Files - .readString(testDirTree.resolve("diff").resolve("diff.diff")); - VariationDiff diff = VariationDiff.fromDiff(source, VariationDiffParseOptions.Default); - String temp = VariationUnparser.variationDiffUnparser(diff); + String original = Files.readString(treeDir.resolve("diff").resolve("diff.diff")); + VariationDiff diff = VariationDiff.fromDiff(original, VariationDiffParseOptions.Default); + String unparsed = VariationUnparser.variationDiffUnparser(diff); assertEquals( - removeWhitespace(VariationUnparser.undiff(source, Time.BEFORE), false), - removeWhitespace(VariationUnparser.undiff(temp, Time.BEFORE), false)); + removeWhitespace(VariationUnparser.undiff(original, Time.BEFORE), false), + removeWhitespace(VariationUnparser.undiff(unparsed, Time.BEFORE), false)); assertEquals( - removeWhitespace(VariationUnparser.undiff(source, Time.AFTER), false), - removeWhitespace(VariationUnparser.undiff(temp, Time.AFTER), false)); + removeWhitespace(VariationUnparser.undiff(original, Time.AFTER), false), + removeWhitespace(VariationUnparser.undiff(unparsed, Time.AFTER), false)); } @ParameterizedTest @MethodSource("testsTree") - public void testCaseTree(Path testCasePath) throws IOException, DiffParseException { - String temp = Files.readString(testCasePath); - temp = temp.replaceAll("\\r\\n", "\n"); - - String unparse1 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(false, false)); - String unparse2 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(false, true)); - String unparse3 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(true, false)); - String unparse4 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(true, true)); - - temp = removeWhitespace(temp, false); - unparse1 = removeWhitespace(unparse1, false); - unparse2 = removeWhitespace(unparse2, false); - unparse3 = removeWhitespace(unparse3, false); - unparse4 = removeWhitespace(unparse4, false); - - assertEquals(temp, unparse1); - assertEquals(temp, unparse2); - assertEquals(temp, unparse3); - assertEquals(temp, unparse4); + public void testTreeUnparse(Path testCasePath) throws IOException, DiffParseException { + String original = Files.readString(testCasePath); + original = original.replaceAll("\\r\\n", "\n"); + + String unparsed1 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(false, false)); + String unparsed2 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(false, true)); + String unparsed3 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(true, false)); + String unparsed4 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(true, true)); + + original = removeWhitespace(original, false); + unparsed1 = removeWhitespace(unparsed1, false); + unparsed2 = removeWhitespace(unparsed2, false); + unparsed3 = removeWhitespace(unparsed3, false); + unparsed4 = removeWhitespace(unparsed4, false); + + assertEquals(original, unparsed1); + assertEquals(original, unparsed2); + assertEquals(original, unparsed3); + assertEquals(original, unparsed4); } @ParameterizedTest @MethodSource("testsDiff") - public void testCaseDiff(Path testCasePath) throws IOException, DiffParseException { - String temp = Files.readString(testCasePath); - temp = temp.replaceAll("\\r\\n", "\n"); - - String unparse1 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(false, false)); - String unparse2 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(false, true)); - String unparse3 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(true, false)); - String unparse4 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(true, true)); - - String temp1 = VariationUnparser.undiff(temp, Time.BEFORE); - String temp2 = VariationUnparser.undiff(temp, Time.AFTER); - String unparse11 = VariationUnparser.undiff(unparse1, Time.BEFORE); - String unparse12 = VariationUnparser.undiff(unparse1, Time.AFTER); - String unparse21 = VariationUnparser.undiff(unparse2, Time.BEFORE); - String unparse22 = VariationUnparser.undiff(unparse2, Time.AFTER); - String unparse31 = VariationUnparser.undiff(unparse3, Time.BEFORE); - String unparse32 = VariationUnparser.undiff(unparse3, Time.AFTER); - String unparse41 = VariationUnparser.undiff(unparse4, Time.BEFORE); - String unparse42 = VariationUnparser.undiff(unparse4, Time.AFTER); - - assertEquals(removeWhitespace(temp1, false), removeWhitespace(unparse11, false)); - assertEquals(removeWhitespace(temp2, false), removeWhitespace(unparse12, false)); - assertEquals(removeWhitespace(temp1, false), removeWhitespace(unparse21, false)); - assertEquals(removeWhitespace(temp2, false), removeWhitespace(unparse22, false)); - assertEquals(removeWhitespace(temp1, false), removeWhitespace(unparse31, false)); - assertEquals(removeWhitespace(temp2, false), removeWhitespace(unparse32, false)); - assertEquals(removeWhitespace(temp1, false), removeWhitespace(unparse41, false)); - assertEquals(removeWhitespace(temp2, false), removeWhitespace(unparse42, false)); + public void testDiffUnparse(Path testCasePath) throws IOException, DiffParseException { + String original = Files.readString(testCasePath); + original = original.replaceAll("\\r\\n", "\n"); + + String unparsed1 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(false, false)); + String unparsed2 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(false, true)); + String unparsed3 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(true, false)); + String unparsed4 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(true, true)); + + String original1 = VariationUnparser.undiff(original, Time.BEFORE); + String original2 = VariationUnparser.undiff(original, Time.AFTER); + String unparsed11 = VariationUnparser.undiff(unparsed1, Time.BEFORE); + String unparsed12 = VariationUnparser.undiff(unparsed1, Time.AFTER); + String unparsed21 = VariationUnparser.undiff(unparsed2, Time.BEFORE); + String unparsed22 = VariationUnparser.undiff(unparsed2, Time.AFTER); + String unparsed31 = VariationUnparser.undiff(unparsed3, Time.BEFORE); + String unparsed32 = VariationUnparser.undiff(unparsed3, Time.AFTER); + String unparsed41 = VariationUnparser.undiff(unparsed4, Time.BEFORE); + String unparsed42 = VariationUnparser.undiff(unparsed4, Time.AFTER); + + assertEquals(removeWhitespace(original1, false), removeWhitespace(unparsed11, false)); + assertEquals(removeWhitespace(original2, false), removeWhitespace(unparsed12, false)); + assertEquals(removeWhitespace(original1, false), removeWhitespace(unparsed21, false)); + assertEquals(removeWhitespace(original2, false), removeWhitespace(unparsed22, false)); + assertEquals(removeWhitespace(original1, false), removeWhitespace(unparsed31, false)); + assertEquals(removeWhitespace(original2, false), removeWhitespace(unparsed32, false)); + assertEquals(removeWhitespace(original1, false), removeWhitespace(unparsed41, false)); + assertEquals(removeWhitespace(original2, false), removeWhitespace(unparsed42, false)); } public static String parseUnparseTree(Path path, VariationDiffParseOptions option) throws IOException, DiffParseException { From 79b0a27e363eb9206334d8d18dbaa8d04357baa6 Mon Sep 17 00:00:00 2001 From: Benjamin Moosherr Date: Sun, 10 Aug 2025 01:11:00 +0200 Subject: [PATCH 029/126] test: factor out duplicate code in the unparser tests --- src/test/java/VariationUnparserTest.java | 81 +++++++++--------------- 1 file changed, 31 insertions(+), 50 deletions(-) diff --git a/src/test/java/VariationUnparserTest.java b/src/test/java/VariationUnparserTest.java index 6058652a3..28d767211 100644 --- a/src/test/java/VariationUnparserTest.java +++ b/src/test/java/VariationUnparserTest.java @@ -12,7 +12,6 @@ import org.variantsync.diffdetective.variation.diff.parse.VariationDiffParseOptions; import org.variantsync.diffdetective.variation.tree.VariationTree; import org.variantsync.diffdetective.variation.VariationUnparser; -import org.variantsync.diffdetective.variation.tree.source.VariationTreeSource; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.variantsync.diffdetective.experiments.thesis_es.UnparseAnalysis.removeWhitespace; @@ -39,81 +38,54 @@ public static Stream testsDiff() throws IOException { @Test public void testTree() throws IOException, DiffParseException { - String original = Files.readString(treeDir.resolve("test8.txt")); - VariationTree tree = - VariationTree.fromText(original, VariationTreeSource.Unknown, VariationDiffParseOptions.Default); - String unparsed = VariationUnparser.variationTreeUnparser(tree); + Path file = treeDir.resolve("test8.txt"); - assertEquals(removeWhitespace(original, false), removeWhitespace(unparsed, false)); + assertEqualTree( + Files.readString(file), + parseUnparseTree(file, VariationDiffParseOptions.Default)); } @Test public void testDiffSemEq() throws IOException, DiffParseException { - String original = Files.readString(treeDir.resolve("diff").resolve("diff.diff")); - VariationDiff diff = VariationDiff.fromDiff(original, VariationDiffParseOptions.Default); - String unparsed = VariationUnparser.variationDiffUnparser(diff); - - assertEquals( - removeWhitespace(VariationUnparser.undiff(original, Time.BEFORE), false), - removeWhitespace(VariationUnparser.undiff(unparsed, Time.BEFORE), false)); - assertEquals( - removeWhitespace(VariationUnparser.undiff(original, Time.AFTER), false), - removeWhitespace(VariationUnparser.undiff(unparsed, Time.AFTER), false)); + Path file = treeDir.resolve("diff").resolve("diff.diff"); + + assertEqualDiff( + Files.readString(file), + parseUnparseDiff(file, VariationDiffParseOptions.Default)); } @ParameterizedTest @MethodSource("testsTree") public void testTreeUnparse(Path testCasePath) throws IOException, DiffParseException { - String original = Files.readString(testCasePath); - original = original.replaceAll("\\r\\n", "\n"); - String unparsed1 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(false, false)); String unparsed2 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(false, true)); String unparsed3 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(true, false)); String unparsed4 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(true, true)); - original = removeWhitespace(original, false); - unparsed1 = removeWhitespace(unparsed1, false); - unparsed2 = removeWhitespace(unparsed2, false); - unparsed3 = removeWhitespace(unparsed3, false); - unparsed4 = removeWhitespace(unparsed4, false); + String original = Files.readString(testCasePath); + original = original.replaceAll("\\r\\n", "\n"); - assertEquals(original, unparsed1); - assertEquals(original, unparsed2); - assertEquals(original, unparsed3); - assertEquals(original, unparsed4); + assertEqualTree(original, unparsed1); + assertEqualTree(original, unparsed2); + assertEqualTree(original, unparsed3); + assertEqualTree(original, unparsed4); } @ParameterizedTest @MethodSource("testsDiff") public void testDiffUnparse(Path testCasePath) throws IOException, DiffParseException { - String original = Files.readString(testCasePath); - original = original.replaceAll("\\r\\n", "\n"); - String unparsed1 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(false, false)); String unparsed2 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(false, true)); String unparsed3 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(true, false)); String unparsed4 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(true, true)); - String original1 = VariationUnparser.undiff(original, Time.BEFORE); - String original2 = VariationUnparser.undiff(original, Time.AFTER); - String unparsed11 = VariationUnparser.undiff(unparsed1, Time.BEFORE); - String unparsed12 = VariationUnparser.undiff(unparsed1, Time.AFTER); - String unparsed21 = VariationUnparser.undiff(unparsed2, Time.BEFORE); - String unparsed22 = VariationUnparser.undiff(unparsed2, Time.AFTER); - String unparsed31 = VariationUnparser.undiff(unparsed3, Time.BEFORE); - String unparsed32 = VariationUnparser.undiff(unparsed3, Time.AFTER); - String unparsed41 = VariationUnparser.undiff(unparsed4, Time.BEFORE); - String unparsed42 = VariationUnparser.undiff(unparsed4, Time.AFTER); - - assertEquals(removeWhitespace(original1, false), removeWhitespace(unparsed11, false)); - assertEquals(removeWhitespace(original2, false), removeWhitespace(unparsed12, false)); - assertEquals(removeWhitespace(original1, false), removeWhitespace(unparsed21, false)); - assertEquals(removeWhitespace(original2, false), removeWhitespace(unparsed22, false)); - assertEquals(removeWhitespace(original1, false), removeWhitespace(unparsed31, false)); - assertEquals(removeWhitespace(original2, false), removeWhitespace(unparsed32, false)); - assertEquals(removeWhitespace(original1, false), removeWhitespace(unparsed41, false)); - assertEquals(removeWhitespace(original2, false), removeWhitespace(unparsed42, false)); + String original = Files.readString(testCasePath); + original = original.replaceAll("\\r\\n", "\n"); + + assertEqualDiff(original, unparsed1); + assertEqualDiff(original, unparsed2); + assertEqualDiff(original, unparsed3); + assertEqualDiff(original, unparsed4); } public static String parseUnparseTree(Path path, VariationDiffParseOptions option) throws IOException, DiffParseException { @@ -125,4 +97,13 @@ public static String parseUnparseDiff(Path path, VariationDiffParseOptions optio VariationDiff diff = VariationDiff.fromFile(path, option); return VariationUnparser.variationDiffUnparser(diff); } + + private static void assertEqualDiff(String expected, String actual) { + assertEqualTree(VariationUnparser.undiff(expected, Time.BEFORE), VariationUnparser.undiff(actual, Time.BEFORE)); + assertEqualTree(VariationUnparser.undiff(expected, Time.AFTER), VariationUnparser.undiff(actual, Time.AFTER)); + } + + private static void assertEqualTree(String expected, String actual) { + assertEquals(removeWhitespace(expected, false), removeWhitespace(actual, false)); + } } From 6bcde64734b99b7fadea7995cd794b849168e3a8 Mon Sep 17 00:00:00 2001 From: Benjamin Moosherr Date: Sun, 10 Aug 2025 01:11:29 +0200 Subject: [PATCH 030/126] test: make all unparser test helper methods private --- src/test/java/VariationUnparserTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/VariationUnparserTest.java b/src/test/java/VariationUnparserTest.java index 28d767211..df972f01b 100644 --- a/src/test/java/VariationUnparserTest.java +++ b/src/test/java/VariationUnparserTest.java @@ -22,7 +22,7 @@ public class VariationUnparserTest { private final static String treeSuffix = ".txt"; private final static String diffSuffix = ".diff"; - protected static Stream findTestCases(Path dir, String filenameSuffix) throws IOException { + private static Stream findTestCases(Path dir, String filenameSuffix) throws IOException { return Files .list(dir) .filter(filename -> filename.getFileName().toString().endsWith(filenameSuffix)); @@ -88,12 +88,12 @@ public void testDiffUnparse(Path testCasePath) throws IOException, DiffParseExce assertEqualDiff(original, unparsed4); } - public static String parseUnparseTree(Path path, VariationDiffParseOptions option) throws IOException, DiffParseException { + private static String parseUnparseTree(Path path, VariationDiffParseOptions option) throws IOException, DiffParseException { VariationTree tree = VariationTree.fromFile(path, option); return VariationUnparser.variationTreeUnparser(tree); } - public static String parseUnparseDiff(Path path, VariationDiffParseOptions option) throws IOException, DiffParseException { + private static String parseUnparseDiff(Path path, VariationDiffParseOptions option) throws IOException, DiffParseException { VariationDiff diff = VariationDiff.fromFile(path, option); return VariationUnparser.variationDiffUnparser(diff); } From 84e5eabd52c59feb37917cda6c099c2f1244a4bf Mon Sep 17 00:00:00 2001 From: Benjamin Moosherr Date: Sun, 10 Aug 2025 01:34:48 +0200 Subject: [PATCH 031/126] test: refactor the unparser test case sources --- src/test/java/VariationUnparserTest.java | 47 +++++-------------- src/test/resources/unparser/diff/diff.diff | 4 -- src/test/resources/unparser/diffs/01.diff | 4 ++ .../unparser/{test1.txt => trees/01.txt} | 0 .../unparser/{test2.txt => trees/02.txt} | 0 .../unparser/{test3.txt => trees/03.txt} | 0 .../unparser/{test4.txt => trees/04.txt} | 0 .../unparser/{test5.txt => trees/05.txt} | 0 .../unparser/{test6.txt => trees/06.txt} | 0 .../unparser/{test7.txt => trees/07.txt} | 0 .../unparser/{test8.txt => trees/08.txt} | 0 11 files changed, 16 insertions(+), 39 deletions(-) delete mode 100644 src/test/resources/unparser/diff/diff.diff create mode 100644 src/test/resources/unparser/diffs/01.diff rename src/test/resources/unparser/{test1.txt => trees/01.txt} (100%) rename src/test/resources/unparser/{test2.txt => trees/02.txt} (100%) rename src/test/resources/unparser/{test3.txt => trees/03.txt} (100%) rename src/test/resources/unparser/{test4.txt => trees/04.txt} (100%) rename src/test/resources/unparser/{test5.txt => trees/05.txt} (100%) rename src/test/resources/unparser/{test6.txt => trees/06.txt} (100%) rename src/test/resources/unparser/{test7.txt => trees/07.txt} (100%) rename src/test/resources/unparser/{test8.txt => trees/08.txt} (100%) diff --git a/src/test/java/VariationUnparserTest.java b/src/test/java/VariationUnparserTest.java index df972f01b..31f785d8b 100644 --- a/src/test/java/VariationUnparserTest.java +++ b/src/test/java/VariationUnparserTest.java @@ -2,7 +2,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.stream.Stream; -import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.variantsync.diffdetective.diff.result.DiffParseException; @@ -17,45 +16,23 @@ import static org.variantsync.diffdetective.experiments.thesis_es.UnparseAnalysis.removeWhitespace; public class VariationUnparserTest { - private final static Path treeDir = Constants.RESOURCE_DIR.resolve("unparser"); - private final static Path testDirDiff = Constants.RESOURCE_DIR.resolve("diffs").resolve("parser"); - private final static String treeSuffix = ".txt"; - private final static String diffSuffix = ".diff"; + private final static Path unparserTestCaseDir = Constants.RESOURCE_DIR.resolve("unparser"); + private final static Path parserTestCaseDir = Constants.RESOURCE_DIR.resolve("diffs").resolve("parser"); + private final static String parserTestCaseSuffix = ".diff"; - private static Stream findTestCases(Path dir, String filenameSuffix) throws IOException { - return Files - .list(dir) - .filter(filename -> filename.getFileName().toString().endsWith(filenameSuffix)); + public static Stream treeTestCases() throws IOException { + return Files.list(unparserTestCaseDir.resolve("trees")); } - public static Stream testsTree() throws IOException { - return findTestCases(treeDir, treeSuffix); - } - - public static Stream testsDiff() throws IOException { - return findTestCases(testDirDiff, diffSuffix); - } - - @Test - public void testTree() throws IOException, DiffParseException { - Path file = treeDir.resolve("test8.txt"); - - assertEqualTree( - Files.readString(file), - parseUnparseTree(file, VariationDiffParseOptions.Default)); - } - - @Test - public void testDiffSemEq() throws IOException, DiffParseException { - Path file = treeDir.resolve("diff").resolve("diff.diff"); - - assertEqualDiff( - Files.readString(file), - parseUnparseDiff(file, VariationDiffParseOptions.Default)); + public static Stream diffTestCases() throws IOException { + return Stream.concat( + Files.list(unparserTestCaseDir.resolve("diffs")), + Files.list(parserTestCaseDir) + .filter(filename -> filename.getFileName().toString().endsWith(parserTestCaseSuffix))); } @ParameterizedTest - @MethodSource("testsTree") + @MethodSource("treeTestCases") public void testTreeUnparse(Path testCasePath) throws IOException, DiffParseException { String unparsed1 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(false, false)); String unparsed2 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(false, true)); @@ -72,7 +49,7 @@ public void testTreeUnparse(Path testCasePath) throws IOException, DiffParseExce } @ParameterizedTest - @MethodSource("testsDiff") + @MethodSource("diffTestCases") public void testDiffUnparse(Path testCasePath) throws IOException, DiffParseException { String unparsed1 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(false, false)); String unparsed2 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(false, true)); diff --git a/src/test/resources/unparser/diff/diff.diff b/src/test/resources/unparser/diff/diff.diff deleted file mode 100644 index e45efbf26..000000000 --- a/src/test/resources/unparser/diff/diff.diff +++ /dev/null @@ -1,4 +0,0 @@ - #if A - Code -+#endif /* G_OS_WIN32 */ --#endif /* WIN32 */ diff --git a/src/test/resources/unparser/diffs/01.diff b/src/test/resources/unparser/diffs/01.diff new file mode 100644 index 000000000..471bf6f0f --- /dev/null +++ b/src/test/resources/unparser/diffs/01.diff @@ -0,0 +1,4 @@ + #if A + Code ++#endif A +-#endif B diff --git a/src/test/resources/unparser/test1.txt b/src/test/resources/unparser/trees/01.txt similarity index 100% rename from src/test/resources/unparser/test1.txt rename to src/test/resources/unparser/trees/01.txt diff --git a/src/test/resources/unparser/test2.txt b/src/test/resources/unparser/trees/02.txt similarity index 100% rename from src/test/resources/unparser/test2.txt rename to src/test/resources/unparser/trees/02.txt diff --git a/src/test/resources/unparser/test3.txt b/src/test/resources/unparser/trees/03.txt similarity index 100% rename from src/test/resources/unparser/test3.txt rename to src/test/resources/unparser/trees/03.txt diff --git a/src/test/resources/unparser/test4.txt b/src/test/resources/unparser/trees/04.txt similarity index 100% rename from src/test/resources/unparser/test4.txt rename to src/test/resources/unparser/trees/04.txt diff --git a/src/test/resources/unparser/test5.txt b/src/test/resources/unparser/trees/05.txt similarity index 100% rename from src/test/resources/unparser/test5.txt rename to src/test/resources/unparser/trees/05.txt diff --git a/src/test/resources/unparser/test6.txt b/src/test/resources/unparser/trees/06.txt similarity index 100% rename from src/test/resources/unparser/test6.txt rename to src/test/resources/unparser/trees/06.txt diff --git a/src/test/resources/unparser/test7.txt b/src/test/resources/unparser/trees/07.txt similarity index 100% rename from src/test/resources/unparser/test7.txt rename to src/test/resources/unparser/trees/07.txt diff --git a/src/test/resources/unparser/test8.txt b/src/test/resources/unparser/trees/08.txt similarity index 100% rename from src/test/resources/unparser/test8.txt rename to src/test/resources/unparser/trees/08.txt From 69b406f0a633648a618883373af6d8ef3c550ad3 Mon Sep 17 00:00:00 2001 From: Benjamin Moosherr Date: Sun, 10 Aug 2025 01:59:18 +0200 Subject: [PATCH 032/126] test: split different parse options into separate unparse tests --- src/test/java/VariationUnparserTest.java | 55 +++++++++++------------- 1 file changed, 24 insertions(+), 31 deletions(-) diff --git a/src/test/java/VariationUnparserTest.java b/src/test/java/VariationUnparserTest.java index 31f785d8b..d982a7ccc 100644 --- a/src/test/java/VariationUnparserTest.java +++ b/src/test/java/VariationUnparserTest.java @@ -3,6 +3,7 @@ import java.nio.file.Path; import java.util.stream.Stream; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.variantsync.diffdetective.diff.result.DiffParseException; import org.variantsync.diffdetective.variation.DiffLinesLabel; @@ -20,49 +21,41 @@ public class VariationUnparserTest { private final static Path parserTestCaseDir = Constants.RESOURCE_DIR.resolve("diffs").resolve("parser"); private final static String parserTestCaseSuffix = ".diff"; - public static Stream treeTestCases() throws IOException { - return Files.list(unparserTestCaseDir.resolve("trees")); + public static Stream treeTestCases() throws IOException { + return withParseOptions(Files.list(unparserTestCaseDir.resolve("trees"))); } - public static Stream diffTestCases() throws IOException { - return Stream.concat( + public static Stream diffTestCases() throws IOException { + return withParseOptions(Stream.concat( Files.list(unparserTestCaseDir.resolve("diffs")), Files.list(parserTestCaseDir) - .filter(filename -> filename.getFileName().toString().endsWith(parserTestCaseSuffix))); + .filter(filename -> filename.getFileName().toString().endsWith(parserTestCaseSuffix)))); + } + + private static Stream withParseOptions(Stream paths) { + // Build a Cartesian product of all paths and parse options. + return + paths.flatMap(path -> Stream.of( + Arguments.of(path, new VariationDiffParseOptions(false, false)), + Arguments.of(path, new VariationDiffParseOptions(false, true)), + Arguments.of(path, new VariationDiffParseOptions(true, false)), + Arguments.of(path, new VariationDiffParseOptions(true, true)))); } @ParameterizedTest @MethodSource("treeTestCases") - public void testTreeUnparse(Path testCasePath) throws IOException, DiffParseException { - String unparsed1 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(false, false)); - String unparsed2 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(false, true)); - String unparsed3 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(true, false)); - String unparsed4 = parseUnparseTree(testCasePath, new VariationDiffParseOptions(true, true)); - - String original = Files.readString(testCasePath); - original = original.replaceAll("\\r\\n", "\n"); - - assertEqualTree(original, unparsed1); - assertEqualTree(original, unparsed2); - assertEqualTree(original, unparsed3); - assertEqualTree(original, unparsed4); + public void testTreeUnparse(Path testCasePath, VariationDiffParseOptions parseOptions) throws IOException, DiffParseException { + assertEqualTree( + Files.readString(testCasePath).replaceAll("\\r\\n", "\n"), + parseUnparseTree(testCasePath, parseOptions)); } @ParameterizedTest @MethodSource("diffTestCases") - public void testDiffUnparse(Path testCasePath) throws IOException, DiffParseException { - String unparsed1 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(false, false)); - String unparsed2 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(false, true)); - String unparsed3 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(true, false)); - String unparsed4 = parseUnparseDiff(testCasePath, new VariationDiffParseOptions(true, true)); - - String original = Files.readString(testCasePath); - original = original.replaceAll("\\r\\n", "\n"); - - assertEqualDiff(original, unparsed1); - assertEqualDiff(original, unparsed2); - assertEqualDiff(original, unparsed3); - assertEqualDiff(original, unparsed4); + public void testDiffUnparse(Path testCasePath, VariationDiffParseOptions parseOptions) throws IOException, DiffParseException { + assertEqualDiff( + Files.readString(testCasePath).replaceAll("\\r\\n", "\n"), + parseUnparseDiff(testCasePath, parseOptions)); } private static String parseUnparseTree(Path path, VariationDiffParseOptions option) throws IOException, DiffParseException { From a8065d9d82a78a24493227a10866bacd46cddd35 Mon Sep 17 00:00:00 2001 From: Benjamin Moosherr Date: Sun, 10 Aug 2025 12:38:01 +0200 Subject: [PATCH 033/126] fix: fix a bug in the variation diff `endif` parsing --- .../variation/diff/parse/VariationDiffParser.java | 14 +++----------- src/test/resources/unparser/diffs/02.diff | 7 +++++++ 2 files changed, 10 insertions(+), 11 deletions(-) create mode 100644 src/test/resources/unparser/diffs/02.diff diff --git a/src/main/java/org/variantsync/diffdetective/variation/diff/parse/VariationDiffParser.java b/src/main/java/org/variantsync/diffdetective/variation/diff/parse/VariationDiffParser.java index 002714dde..b2d24f02d 100644 --- a/src/main/java/org/variantsync/diffdetective/variation/diff/parse/VariationDiffParser.java +++ b/src/main/java/org/variantsync/diffdetective/variation/diff/parse/VariationDiffParser.java @@ -329,7 +329,7 @@ private void parseLine( // Do not create a node for ENDIF, but update the line numbers of the closed if-chain // and remove that if-chain from the relevant stacks. diffType.forAllTimesOfExistence(beforeStack, afterStack, stack -> - popIfChain(stack, fromLine, line, diffType) + popIfChain(stack, fromLine, line) ); } else if (options.collapseMultipleCodeLines() && annotation.type() == AnnotationType.None @@ -369,8 +369,7 @@ private void parseLine( private void popIfChain( Stack> stack, DiffLineNumber elseLineNumber, - LogicalLine line, - DiffType diffType + LogicalLine line ) throws DiffParseException { DiffLineNumber previousLineNumber = elseLineNumber; do { @@ -382,14 +381,7 @@ private void popIfChain( for (int i = 0; i < line.getLines().size(); i++) { list.add(line.getLines().get(i).content()); } - if (diffType.existsBefore() && diffType.existsAfter()) { - annotation.setEndIf(list, Time.BEFORE); - annotation.setEndIf(list, Time.AFTER); - } else if (diffType.existsBefore()) { - annotation.setEndIf(list, Time.BEFORE); - } else { - annotation.setEndIf(list, Time.AFTER); - } + annotation.setEndIf(list, stack == beforeStack ? Time.BEFORE : Time.AFTER); } // Set the line number of now closed annotations to the beginning of the diff --git a/src/test/resources/unparser/diffs/02.diff b/src/test/resources/unparser/diffs/02.diff new file mode 100644 index 000000000..74c2ff65e --- /dev/null +++ b/src/test/resources/unparser/diffs/02.diff @@ -0,0 +1,7 @@ + #ifndef A + code +-#endif + +-#ifndef A + code + #endif // A From 0db0fdd7a818202c36ac2ed3ddcdd5ea6aa76a43 Mon Sep 17 00:00:00 2001 From: Benjamin Moosherr Date: Sun, 10 Aug 2025 13:39:16 +0200 Subject: [PATCH 034/126] refactor: directly pass the time to popIfChain in the diff parser --- .../variation/diff/DiffType.java | 9 ++++--- .../diff/parse/VariationDiffParser.java | 26 ++++++++++--------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/variantsync/diffdetective/variation/diff/DiffType.java b/src/main/java/org/variantsync/diffdetective/variation/diff/DiffType.java index 4adc558a5..0a9b6c5fc 100644 --- a/src/main/java/org/variantsync/diffdetective/variation/diff/DiffType.java +++ b/src/main/java/org/variantsync/diffdetective/variation/diff/DiffType.java @@ -1,11 +1,10 @@ package org.variantsync.diffdetective.variation.diff; import org.apache.commons.lang3.function.FailableConsumer; -import org.tinylog.Logger; +import org.apache.commons.lang3.function.FailableRunnable; import java.util.Optional; import java.util.Set; -import java.util.function.Consumer; /** * Type of change made to an artifact (e.g., a line of text in a text-based diff). @@ -70,8 +69,9 @@ public static Optional thatExistsOnlyAtAll(final Set