Merge pull request #179 from amandascm/feat/js-dyn-analysis

Feat/js dyn analysis

src/main/injectors/DynamicAnalysisJsModifiedLinesDetectionModule.groovy

@@ -0,0 +1,45 @@
+package injectors
+
+import com.google.inject.*
+import com.google.inject.multibindings.Multibinder
+
+import interfaces.CommitFilter
+import interfaces.DataCollector
+import interfaces.OutputProcessor
+import interfaces.ProjectProcessor
+
+import services.commitFilters.InCommitListMutuallyModifiedMethodsFilter
+import services.commitFilters.MutuallyModifiedFilesCommitFilter
+import services.dataCollectors.buildRequester.BuildRequester
+import services.dataCollectors.MergeConflictCollector
+import services.dataCollectors.StatisticsCollector
+import services.dataCollectors.modifiedLinesCollector.ModifiedFilesLinesPerFileCollector
+import services.outputProcessors.FetchBuildsOutputProcessor
+import services.outputProcessors.GenerateSootInputFilesOutputProcessor
+import services.outputProcessors.WaitForBuildsOutputProcessor
+import services.outputProcessors.soot.RunSootAnalysisOutputProcessor
+import services.projectProcessors.FilterNonExistentProjectsProcessor
+import services.projectProcessors.ForkAndEnableCIProcessor
+import services.util.ci.CIPlatform
+import services.util.ci.GithubActionsPlatform
+import services.util.ci.TravisPlatform
+import services.util.FetchBuildsScript
+
+class DynamicAnalysisJsModifiedLinesDetectionModule extends AbstractModule {
+
+    @Override
+    protected void configure() {
+        Multibinder<DataCollector> dataCollectorBinder = Multibinder.newSetBinder(binder(), DataCollector.class)
+
+        dataCollectorBinder.addBinding().to(ModifiedFilesLinesPerFileCollector.class)
+        dataCollectorBinder.addBinding().to(StatisticsCollector.class)
+
+        bind(CommitFilter.class).to(MutuallyModifiedFilesCommitFilter.class)
+
+        Multibinder<ProjectProcessor> projectProcessorBinder = Multibinder.newSetBinder(binder(), ProjectProcessor.class)
+        projectProcessorBinder.addBinding().to(FilterNonExistentProjectsProcessor.class)
+
+        Multibinder<OutputProcessor> outputProcessorBinder = Multibinder.newSetBinder(binder(), OutputProcessor.class)
+    }
+
+}

src/main/services/commitFilters/MutuallyModifiedFilesCommitFilter.groovy

@@ -4,19 +4,20 @@ import interfaces.CommitFilter
 import project.MergeCommit
 import project.Project
 import util.ProcessRunner
+import app.MiningFramework
 
 import java.util.stream.Collectors
 
 class MutuallyModifiedFilesCommitFilter implements CommitFilter {
 
     @Override
     boolean applyFilter(Project project, MergeCommit mergeCommit) {
-        List<String> leftFilePaths = getModifiedJavaFilePaths(project, mergeCommit, mergeCommit.getLeftSHA())
-        List<String> rightFilePaths = getModifiedJavaFilePaths(project, mergeCommit, mergeCommit.getRightSHA())
+        List<String> leftFilePaths = getModifiedFilePaths(project, mergeCommit, mergeCommit.getLeftSHA())
+        List<String> rightFilePaths = getModifiedFilePaths(project, mergeCommit, mergeCommit.getRightSHA())
         return !leftFilePaths.disjoint(rightFilePaths)
     }
 
-    static List<String> getModifiedJavaFilePaths(Project project, MergeCommit mergeCommit, String commitSHA) {
+    static List<String> getModifiedFilePaths(Project project, MergeCommit mergeCommit, String commitSHA) {
         List<String> parameters = [ 'git', 'diff-tree', '--no-commit-id', '--name-status', '-r' ]
         parameters.addAll(mergeCommit.getAncestorSHA(), commitSHA)
 
@@ -29,7 +30,7 @@ class MutuallyModifiedFilesCommitFilter implements CommitFilter {
         return files.stream()
             .filter(line -> MutuallyModifiedFilesCommitFilter::isModifiedFile(getFileStatus(line)))
             .map(MutuallyModifiedFilesCommitFilter::getFilePath)
-            .filter(MutuallyModifiedFilesCommitFilter::isJavaFile)
+            .filter(MutuallyModifiedFilesCommitFilter::isExpectedFileExtension)
             .collect(Collectors.toList())
     }
 
@@ -45,8 +46,8 @@ class MutuallyModifiedFilesCommitFilter implements CommitFilter {
         return diffTreeOutputLine.substring(1).trim()
     }
 
-    private static boolean isJavaFile(String filePath) {
-        return filePath.endsWith('.java')
+    private static boolean isExpectedFileExtension(String filePath) {
+        return filePath.endsWith(MiningFramework.arguments.getFileExtension())
     }
 
 }

src/main/services/dataCollectors/S3MWithCSDiffCollector/FilesQuadruplesCollector.groovy

@@ -18,10 +18,10 @@ class FilesQuadruplesCollector {
 
     static List<Path> collectFilesQuadruples(Project project, MergeCommit mergeCommit) {
         String commitSHA = mergeCommit.getLeftSHA()
-        List<String> leftFilePaths = MutuallyModifiedFilesCommitFilter.getModifiedJavaFilePaths(project, mergeCommit, commitSHA)
+        List<String> leftFilePaths = MutuallyModifiedFilesCommitFilter.getModifiedFilePaths(project, mergeCommit, commitSHA)
 
         commitSHA = mergeCommit.getRightSHA()
-        List<String> rightFilePaths = MutuallyModifiedFilesCommitFilter.getModifiedJavaFilePaths(project, mergeCommit, commitSHA)
+        List<String> rightFilePaths = MutuallyModifiedFilesCommitFilter.getModifiedFilePaths(project, mergeCommit, commitSHA)
 
         List<String> mutuallyModifiedFilePaths = leftFilePaths.intersect(rightFilePaths)
         return mutuallyModifiedFilePaths.stream()

src/main/services/dataCollectors/modifiedLinesCollector/ModifiedFilesLinesPerFileCollector.groovy

@@ -0,0 +1,61 @@
+package services.dataCollectors.modifiedLinesCollector
+
+import interfaces.DataCollector
+import project.MergeCommit
+import project.Project
+import util.FileManager
+import services.dataCollectors.RevisionsFilesCollector
+import util.TypeNameHelper
+
+import static app.MiningFramework.arguments
+
+
+/**
+ * @requires: that a diffj cli is in the dependencies folder and that diff (textual diff tool) is installed
+ * @provides: a [outputPath]/data/results.csv file with the following format:
+ * project;merge commit;file path;left additions;left deletions;left changes;right additions;right deletions;right changes;repo url
+ */
+class ModifiedFilesLinesPerFileCollector extends ModifiedLinesCollectorAbstract {
+
+    public ModifiedFilesLinesPerFileCollector() {
+        modifiedMethodsHelper = new ModifiedMethodsHelper("diffj.jar");
+    }
+
+    void collectData(Project project, MergeCommit mergeCommit) {
+        createOutputFiles(arguments.getOutputPath())
+        Set<String> mutuallyModifiedFiles = getFilesModifiedByBothParents(project, mergeCommit);
+        println "${project.getName()} - ModifiedFilesLinesCollector collection ongoing with mutuallyModifiedFiles: ${mutuallyModifiedFiles}"
+
+        for (String filePath : mutuallyModifiedFiles) {
+            Set<ModifiedLine> leftModifiedLines = modifiedMethodsHelper.getModifiedLines(project, filePath, mergeCommit.getAncestorSHA(), mergeCommit.getLeftSHA())
+            Set<ModifiedLine> rightModifiedLines = modifiedMethodsHelper.getModifiedLines(project, filePath, mergeCommit.getAncestorSHA(), mergeCommit.getRightSHA())
+
+            collectFileData(leftModifiedLines, rightModifiedLines, project, mergeCommit, filePath)
+        }
+        println "${project.getName()} - ModifiedFilesLinesCollector collection finished"
+    }
+
+    void createExperimentalDataFiles(String outputPath) {
+        this.experimentalDataFile = new File(outputPath + "/data/results.csv")
+        if (!experimentalDataFile.exists()) {
+            this.experimentalDataFile << 'project;merge commit;file path;left additions;left deletions;left changes;right additions;right deletions;right changes;repo url\n'
+        }
+
+        if (arguments.isPushCommandActive()) {
+            this.experimentalDataFileWithLinks = new File("${outputPath}/data/result-links.csv");
+        }
+    }
+
+    private synchronized void printResults(Project project, MergeCommit mergeCommit, String filePath,
+                      HashSet<Integer> leftAddedLines, HashSet<Integer> leftDeletedLines, HashSet<Integer> leftChangedLines,
+                      HashSet<Integer> rightAddedLines, HashSet<Integer> rightDeletedLines, HashSet<Integer> rightChangedLines) {
+        String remoteRepositoryURL = project.getRemoteUrl()
+        experimentalDataFile << "${project.getName()};${mergeCommit.getSHA()};${filePath};${leftAddedLines};${leftDeletedLines};${leftChangedLines};${rightAddedLines};${rightDeletedLines};${rightChangedLines};${remoteRepositoryURL}\n"
+
+        // Add links.
+        if(arguments.isPushCommandActive())
+            addLinks(project.getName(), mergeCommit.getSHA(), filePath, leftAddedLines, leftDeletedLines, leftChangedLines, rightAddedLines, rightDeletedLines, rightChangedLines, arguments.getResultsRemoteRepositoryURL())
+
+    }
+
+}

src/main/services/dataCollectors/modifiedLinesCollector/ModifiedLinesCollectorAbstract.groovy

@@ -60,6 +60,38 @@ abstract class ModifiedLinesCollectorAbstract implements DataCollector {
         printResults(project, mergeCommit, className, mergeMethod.getSignature(), leftAddedLines, leftDeletedLines, rightAddedLines, rightDeletedLines);
     }
 
+    protected void collectFileData(Set<ModifiedLine> leftModifiedLines, Set<ModifiedLine> rightModifiedLines, Project project, MergeCommit mergeCommit, String filePath) {
+        Set<Integer> leftAddedLines = new HashSet<Integer>();
+        Set<Integer> leftDeletedLines = new HashSet<Integer>();
+        Set<Integer> leftChangedLines = new HashSet<Integer>();
+        Set<Integer> rightAddedLines = new HashSet<Integer>();
+        Set<Integer> rightDeletedLines = new HashSet<Integer>();
+        Set<Integer> rightChangedLines = new HashSet<Integer>();
+
+        for (def line : leftModifiedLines) {
+            if (line.getType() == ModifiedLine.ModificationType.Removed) {
+                leftDeletedLines.add(line.getNumber());
+            } else if (line.getType() == ModifiedLine.ModificationType.Added){
+                leftAddedLines.add(line.getNumber());
+            } else {
+                leftChangedLines.add(line.getNumber())
+            }
+        }
+
+        for (def line : rightModifiedLines) {
+            if (line.getType() == ModifiedLine.ModificationType.Removed) {
+                rightDeletedLines.add(line.getNumber());
+            } else if (line.getType() == ModifiedLine.ModificationType.Added){
+                rightAddedLines.add(line.getNumber());
+            } else {
+                rightChangedLines.add(line.getNumber())
+            }
+        }
+
+        // prints results to a csv file
+        printResults(project, mergeCommit, filePath, leftAddedLines, leftDeletedLines, leftChangedLines, rightAddedLines, rightDeletedLines, rightChangedLines);
+    }
+
     protected void createOutputFiles(String outputPath) {
         createExperimentalDataDir(outputPath)
         createExperimentalDataFiles(outputPath)
@@ -74,8 +106,8 @@ abstract class ModifiedLinesCollectorAbstract implements DataCollector {
     }
 
     protected Set<String> getFilesModifiedByBothParents(Project project, MergeCommit mergeCommit) {
-        Set<String> leftModifiedFiles = FileManager.getModifiedFiles(project, mergeCommit.getLeftSHA(), mergeCommit.getAncestorSHA())
-        Set<String> rightModifiedFiles = FileManager.getModifiedFiles(project, mergeCommit.getRightSHA(), mergeCommit.getAncestorSHA())
+        Set<String> leftModifiedFiles = FileManager.getModifiedFiles(project, mergeCommit.getLeftSHA(), mergeCommit.getAncestorSHA(), arguments.getFileExtension())
+        Set<String> rightModifiedFiles = FileManager.getModifiedFiles(project, mergeCommit.getRightSHA(), mergeCommit.getAncestorSHA(), arguments.getFileExtension())
 
         return leftModifiedFiles.intersect(rightModifiedFiles)
     }

src/main/services/dataCollectors/modifiedLinesCollector/ModifiedMethodsHelper.groovy

@@ -53,6 +53,21 @@ class ModifiedMethodsHelper {
         return modifiedMethodsMatcher.matchModifiedMethodsAndLines(parsedDiffJResult, parsedTextualDiffResult);
     }
 
+    public List<ModifiedLine> getModifiedLines(Project project, String filePath, String ancestorSHA, String targetSHA) {
+        File ancestorFile = FileManager.getFileInCommit(project, filePath, ancestorSHA)
+        File targetFile = FileManager.getFileInCommit(project, filePath, targetSHA)
+
+        List<String> diffJOutput = runDiffJ(ancestorFile, targetFile);
+        List<String> textualDiffOutput = runTextualDiff(ancestorFile, targetFile);
+
+        targetFile.delete()
+        ancestorFile.delete()
+
+        List<ModifiedLine> parsedTextualDiffResult = textualDiffParser.parse(textualDiffOutput);
+
+        return parsedTextualDiffResult
+    }
+
     private List<String> runDiffJ(File ancestorFile, File targetFile) {
         Process diffJ = ProcessRunner.runProcess(this.dependenciesPath, 'java', '-jar', this.diffJOption, "--brief", ancestorFile.getAbsolutePath(), targetFile.getAbsolutePath())
         BufferedReader reader = new BufferedReader(new InputStreamReader(diffJ.getInputStream()))

src/main/util/FileManager.groovy

@@ -22,6 +22,18 @@ final class FileManager {
         return modifiedFiles
     }
 
+    public static Set<String> getModifiedFiles(Project project, String childSHA, String ancestorSHA, String fileExtension) {
+        Set<String> modifiedFiles = new HashSet<String>()
+
+        Process gitDiff = ProcessRunner.runProcess(project.getPath(), 'git', 'diff', '--name-only', childSHA, ancestorSHA)
+        gitDiff.getInputStream().eachLine {
+            if(it.endsWith(fileExtension))
+                modifiedFiles.add(it)
+        }
+
+        return modifiedFiles
+    }
+
     public static Set<String> getModifiedFilesLocalOption(Project project, String pathCommitOne, String pathCommitTwo) {
         def command = "diff -qr "+pathCommitOne+" "+pathCommitTwo
         def proc = command.execute()