related helper function

helper.py

@@ -6,14 +6,14 @@
 
 ###############################################################
 
-g_test_file = "data/selected_file/mn_c2x/c2x_jl_test_correct_prediction_samefile.txt"
+g_deltas_types = ["token", "char"]
 g_simp_file = "data/tmp/sm_test.java"
+JAR_LOAD_JAVA_METHOD = "others/LoadJavaMethod/target/jar/LoadJavaMethod.jar"
 
-g_deltas_types = ["token", "char"]
+# TODO - update file_path and delta_type
+g_test_file = "data/selected_file/mn_c2x/c2x_jl_test_correct_prediction_samefile.txt"
 g_deltas_type = g_deltas_types[0]
 
-JAR_LOAD_JAVA_METHOD = "others/LoadJavaMethod/target/jar/LoadJavaMethod.jar"
-
 ###############################################################
 
 

others/GetAstNodes/.gitignore

@@ -0,0 +1,32 @@
+
+# OS file
+.DS_Store
+
+# Generated files
+/target/
+.idea/
+*.iml
+
+# Compiled class file
+*.class
+
+# Log file
+*.log
+
+# BlueJ files
+*.ctxt
+
+# Mobile Tools for Java (J2ME)
+.mtj.tmp/
+
+# Package Files #
+*.jar
+*.war
+*.nar
+*.ear
+*.zip
+*.tar.gz
+*.rar
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*

others/GetAstNodes/pom.xml

@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.uhserg.dd.getastnodes</groupId>
+    <artifactId>GetAstNodes</artifactId>
+    <version>1.0</version>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <configuration>
+                    <finalName>jar/GetAstNodes</finalName>
+                    <!-- TODO: mvn clean compile assembly:single -->
+                    <appendAssemblyId>false</appendAssemblyId>
+                    <archive>
+                        <manifest>
+                            <mainClass>Main</mainClass>
+                        </manifest>
+                    </archive>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>1.8</source>
+                    <target>1.8</target>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.github.javaparser</groupId>
+            <artifactId>javaparser-core</artifactId>
+            <version>3.14.5</version>
+        </dependency>
+    </dependencies>
+
+</project>

others/GetAstNodes/src/main/java/Main.java

@@ -0,0 +1,72 @@
+import com.github.javaparser.StaticJavaParser;
+import com.github.javaparser.ast.CompilationUnit;
+import com.github.javaparser.ast.Node;
+import com.github.javaparser.ast.body.MethodDeclaration;
+import com.github.javaparser.ast.visitor.TreeVisitor;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Input: string_program
+ * Output: ast_nodes
+ */
+public class Main {
+    static final String INNER_DELIMITER = " ___INNER___ ";
+    static final String OUTER_DELIMITER = " ___OUTER___ ";
+
+    public static void main(String[] args) {
+        String strCode = args[0];
+        CompilationUnit cu = getParseUnit(strCode);
+        List<String> allTerminals = getTerminalNodes(cu);
+        List<String> allClasses = getNodeClasses(cu);
+        String astData = String.join(INNER_DELIMITER, allTerminals)
+                + OUTER_DELIMITER
+                + String.join(INNER_DELIMITER, allClasses);
+        System.out.print(astData);
+    }
+
+    public static List<String> getTerminalNodes (CompilationUnit cu) {
+        List<String> allTerminals = new ArrayList<>();
+        MethodDeclaration md = cu.findAll(MethodDeclaration.class).get(0);
+        md.setName("METHOD_NAME"); // skip actual method name
+        new TreeVisitor() {
+            @Override
+            public void process(Node node) {
+                if (node.getChildNodes().size() == 0) {
+                    allTerminals.add(node.toString());
+                }
+            }
+        }.visitPreOrder(md);
+        return allTerminals;
+    }
+
+    public static List<String> getNodeClasses(CompilationUnit cu) {
+        List<String> allClasses = new ArrayList<>();
+        MethodDeclaration md = cu.findAll(MethodDeclaration.class).get(0);
+        new TreeVisitor() {
+            @Override
+            public void process(Node node) {
+                String clsStr = node.getClass().toString();
+                // clsStr = clsStr.replace("class com.github.javaparser.ast.", "");
+                String lastName = clsStr.substring(clsStr.lastIndexOf('.') + 1);
+                allClasses.add(lastName);
+            }
+        }.visitPreOrder(md);
+        return allClasses;
+    }
+
+    public static CompilationUnit getParseUnit(String txtCode) {
+        CompilationUnit cu = null;
+        try {
+            // add tmp class to parse
+            if(!txtCode.startsWith("class")) txtCode = "class T { \n" + txtCode + "\n}";
+            // remove comments
+            StaticJavaParser.getConfiguration().setAttributeComments(false);
+            // parse code
+            cu = StaticJavaParser.parse(txtCode);
+        } catch (Exception ignore) {}
+        return cu;
+    }
+
+}

others/LoadJavaMethod/src/main/java/Main.java

@@ -8,31 +8,29 @@
 import java.util.List;
 
 /**
- * Input: .../file.java
- * Output: "method_name" "method_body"
+ * Input: <file_path>.java
+ * Output: "method_name" "single_line"
  */
 public class Main {
 
     public static void main(String[] args) {
         File inputFile = new File(args[0]);
-        //System.out.println(inputFile);
         CompilationUnit root = getParseUnit(inputFile);
         List<String> contents = loadJavaMethod(root);
         System.out.print(String.join(" ", contents));
     }
 
     static List<String> loadJavaMethod(CompilationUnit root) {
-        List<String> contents = new ArrayList<String>();
+        List<String> contents = new ArrayList<>();
         if (root != null) {
             try {
                 MethodDeclaration methodDec = root.findAll(MethodDeclaration.class).get(0);
                 String method_name = methodDec.getName().toString();
                 contents.add(method_name);
-//                methodDec.setName("f"); // keep original name
-                String method_body = methodDec.toString()
+                String single_line = methodDec.toString()
                         .replaceAll("\\n", " ")
                         .replaceAll("\\r", " ");
-                contents.add(method_body);
+                contents.add(single_line);
             } catch (Exception ignore){}
         }
         return contents;
@@ -42,8 +40,11 @@ static CompilationUnit getParseUnit(File javaFile) {
         CompilationUnit root = null;
         try {
             String txtCode = new String(Files.readAllBytes(javaFile.toPath()));
-            if(!txtCode.startsWith("class")) txtCode = "class T { \n" + txtCode + "\n}"; // add tmp class to parse
-            StaticJavaParser.getConfiguration().setAttributeComments(false); // remove comments
+            // add tmp class to parse
+            if(!txtCode.startsWith("class")) txtCode = "class T { \n" + txtCode + "\n}";
+            // remove comments
+            StaticJavaParser.getConfiguration().setAttributeComments(false);
+            // parse code
             root = StaticJavaParser.parse(txtCode);
         } catch (Exception ignore) {}
         return root;

others/get_ast_nodes.py

@@ -0,0 +1,25 @@
+import subprocess
+from pathlib import Path
+
+JAR_JAVA_AST_DATA = "GetAstNodes/target/jar/GetAstNodes.jar"
+INNER_DELIMITER = " ___INNER___ "
+OUTER_DELIMITER = " ___OUTER___ "
+
+
+def get_ast_data(str_code):
+    cmd = ['java', '-jar', JAR_JAVA_AST_DATA, str_code]
+    content = subprocess.check_output(cmd, encoding="utf-8", close_fds=True)
+    [all_terminals, all_classes] = content.strip().split(OUTER_DELIMITER)
+    all_terminals = all_terminals.split(INNER_DELIMITER)
+    all_classes = all_classes.split(INNER_DELIMITER)
+    return all_terminals, all_classes
+
+
+if __name__ == '__main__':
+    program = Path('sample_input.java').read_text()
+    print("program:\n{}".format(program))
+    ast_terminals, ast_classes = get_ast_data(program)
+    print("ast_terminals = {}".format(ast_terminals))
+    print("ast_classes = {}".format(ast_classes))
+    ast_nodes = list(set(ast_terminals + ast_classes))
+    print("all_nodes = {}".format(ast_nodes))

others/get_tokens_java.py

@@ -0,0 +1,14 @@
+import javalang
+from pathlib import Path
+
+
+def get_tokens(str_code):
+    tokens = list(javalang.tokenizer.tokenize(str_code))
+    tokens = [token.value for token in tokens]
+    return tokens
+
+
+if __name__ == '__main__':
+    program = Path('sample_input.java').read_text()
+    print("program:\n{}".format(program))
+    print("tokens = {}".format(get_tokens(program)))

others/load_java_method.py

@@ -0,0 +1,21 @@
+import subprocess
+from pathlib import Path
+
+JAR_LOAD_JAVA_METHOD = "LoadJavaMethod/target/jar/LoadJavaMethod.jar"
+
+
+def load_method(file_path):
+    cmd = ['java', '-jar', JAR_LOAD_JAVA_METHOD, file_path]
+    contents = subprocess.check_output(cmd, encoding="utf-8", close_fds=True)
+    contents = contents.split()
+    name, body = contents[0], " ".join(contents[1:])
+    return name, body
+
+
+if __name__ == '__main__':
+    input_path = 'sample_input.java'
+    program = Path(input_path).read_text()
+    print("program:\n{}".format(program))
+    method_name, single_line = load_method(input_path)
+    print("method_name = {}".format(method_name))
+    print("single_line = {}".format(single_line))

others/sample_input.java

@@ -0,0 +1,3 @@
+void print() {
+    System.out.println("Hello World!");
+}