diff --git a/symbols_db/cli.py b/symbols_db/cli.py index f305a4b..5817ed4 100644 --- a/symbols_db/cli.py +++ b/symbols_db/cli.py @@ -1,36 +1,27 @@ import argparse import concurrent import concurrent.futures -from time import sleep import sqlite3 import traceback +from time import sleep +from symbols_db import BLINTDB_LOCATION, VCPKG_LOCATION, logger from symbols_db.handlers.blint_handler import ( - blint_on_crates_from_purl, - get_blint_internal_functions_exe, -) + blint_on_crates_from_purl, get_blint_internal_functions_exe) from symbols_db.handlers.cyclonedx_handler import get_purl_from_bom -from symbols_db.handlers.language_handlers.wrapdb_handler import get_wrapdb_projects -from symbols_db.handlers.language_handlers.cargo_handler import build_crates_from_purl -from symbols_db.handlers.language_handlers.vcpkg_handler import ( - get_vcpkg_projects, - vcpkg_build, - find_vcpkg_executables, -) +from symbols_db.handlers.language_handlers.cargo_handler import \ + build_crates_from_purl from symbols_db.handlers.language_handlers.meson_handler import ( - meson_build, - find_meson_executables, - strip_executables, -) -from symbols_db.handlers.sqlite_handler import ( - add_projects, - add_binary, - add_binary_export, -) - + find_meson_executables, meson_build, strip_executables) +from symbols_db.handlers.language_handlers.vcpkg_handler import ( + find_vcpkg_executables, get_vcpkg_projects, vcpkg_build) +from symbols_db.handlers.language_handlers.wrapdb_handler import \ + get_wrapdb_projects # TODO: clean up reset database command -from symbols_db.handlers.sqlite_handler import clear_sqlite_database, create_database -from symbols_db import BLINTDB_LOCATION, logger, VCPKG_LOCATION +from symbols_db.handlers.sqlite_handler import (add_binary, add_binary_export, + add_projects, + clear_sqlite_database, + create_database) clear_sqlite_database() create_database() diff --git a/symbols_db/handlers/blint_handler.py b/symbols_db/handlers/blint_handler.py index ff7c393..42be1f0 100644 --- a/symbols_db/handlers/blint_handler.py +++ b/symbols_db/handlers/blint_handler.py @@ -1,18 +1,15 @@ import json import os -from pathlib import Path import subprocess +from pathlib import Path -from symbols_db import BOM_LOCATION, CWD +from symbols_db import (BOM_LOCATION, CWD, DEBUG_MODE, DELIMETER_BOM, + WRAPDB_LOCATION, logger) from symbols_db.handlers.sqlite_handler import store_sbom_in_sqlite from symbols_db.utils.json import get_properties_internal -from symbols_db.utils.rust import ( - from_purl_to_rust_srcname, - get_all_index_names, - get_path_names_from_index_names, -) -from symbols_db import DEBUG_MODE, DELIMETER_BOM, WRAPDB_LOCATION -from symbols_db import logger +from symbols_db.utils.rust import (from_purl_to_rust_srcname, + get_all_index_names, + get_path_names_from_index_names) def run_blint_on_file(file_path): diff --git a/symbols_db/handlers/cyclonedx_handler.py b/symbols_db/handlers/cyclonedx_handler.py index 8def876..a3ea36c 100644 --- a/symbols_db/handlers/cyclonedx_handler.py +++ b/symbols_db/handlers/cyclonedx_handler.py @@ -1,4 +1,5 @@ import json + from symbols_db import logger diff --git a/symbols_db/handlers/git_handler.py b/symbols_db/handlers/git_handler.py index 4450d21..aedc05b 100644 --- a/symbols_db/handlers/git_handler.py +++ b/symbols_db/handlers/git_handler.py @@ -1,6 +1,6 @@ import subprocess -from symbols_db import DEBUG_MODE +from symbols_db import DEBUG_MODE # def git_clone_wrapdb(): # command = ( diff --git a/symbols_db/handlers/language_handlers/__init__.py b/symbols_db/handlers/language_handlers/__init__.py index 59c5b1b..9ed58e3 100644 --- a/symbols_db/handlers/language_handlers/__init__.py +++ b/symbols_db/handlers/language_handlers/__init__.py @@ -1,2 +1,12 @@ +import subprocess + +from symbols_db import WRAPDB_LOCATION + + class BaseHandler: - pass + strip = True + + def strip_executables(self, file_path, loc=WRAPDB_LOCATION): + if self.strip: + strip_command = f"strip --strip-all {file_path}".split(" ") + subprocess.run(strip_command, cwd=loc) diff --git a/symbols_db/handlers/language_handlers/cargo_handler.py b/symbols_db/handlers/language_handlers/cargo_handler.py index d40e6e6..580dffb 100644 --- a/symbols_db/handlers/language_handlers/cargo_handler.py +++ b/symbols_db/handlers/language_handlers/cargo_handler.py @@ -1,11 +1,9 @@ -from symbols_db.utils.rust import ( - from_purl_to_rust_srcname, - get_all_index_names, - get_path_names_from_index_names, -) +import os from symbols_db import logger -import os +from symbols_db.utils.rust import (from_purl_to_rust_srcname, + get_all_index_names, + get_path_names_from_index_names) def build_cargo_package(build_dir): diff --git a/symbols_db/handlers/language_handlers/meson_handler.py b/symbols_db/handlers/language_handlers/meson_handler.py index dc4ba03..88550fd 100644 --- a/symbols_db/handlers/language_handlers/meson_handler.py +++ b/symbols_db/handlers/language_handlers/meson_handler.py @@ -1,13 +1,14 @@ import os import subprocess - -from symbols_db import logger, WRAPDB_LOCATION, CWD from pathlib import Path -from symbols_db import logger + +from symbols_db import CWD, WRAPDB_LOCATION, logger from symbols_db.handlers.language_handlers import BaseHandler +from symbols_db.projects_compiler.meson import (ensure_meson_installed, + git_checkout_wrapdb_commit, + git_clone_wrapdb) from symbols_db.utils.utils import subprocess_run_debug - # TODO: debug DEBUG_MODE = True @@ -15,19 +16,49 @@ class MesonHandler(BaseHandler): def __init__(self): - pass + if meson_present := ensure_meson_installed(): + git_clone_wrapdb() + git_checkout_wrapdb_commit() + else: + ModuleNotFoundError("Meson was not found") def build(self, project_name): - pass + setup_command = ( + f"meson setup build/{project_name} -Dwraps={project_name}".split(" ") + ) + meson_setup = subprocess.run(setup_command, cwd=WRAPDB_LOCATION) + subprocess_run_debug(meson_setup, project_name) + compile_command = f"meson compile -C build/{project_name}".split(" ") + meson_compile = subprocess.run(compile_command, cwd=WRAPDB_LOCATION) + subprocess_run_debug(meson_compile, project_name) def find_executables(self, project_name): - pass + full_project_dir = WRAPDB_LOCATION / "build" / project_name / "subprojects" + executable_list = [] + for root, dir, files in os.walk(full_project_dir): + for file in files: + # what is the value of variable `root` + file_path = Path(root) / file + if os.access(file_path, os.X_OK): + full_path = CWD / file_path + file_output = subprocess.run( + ["file", full_path], capture_output=True + ) + if b"ELF" in file_output.stdout: + executable_list.append(full_path) + return executable_list def delete_project_files(self, project_name): pass def get_project_list(self): - pass + subproject_filenames = os.listdir(WRAPDB_LOCATION / "subprojects") + projects_list = [] + for file in subproject_filenames: + project_path = Path(file) + if project_path.suffix == ".wrap": + projects_list.append(project_path.stem) + return projects_list def meson_build(project_name): diff --git a/symbols_db/handlers/language_handlers/vcpkg_handler.py b/symbols_db/handlers/language_handlers/vcpkg_handler.py index c182db7..6a11568 100644 --- a/symbols_db/handlers/language_handlers/vcpkg_handler.py +++ b/symbols_db/handlers/language_handlers/vcpkg_handler.py @@ -1,53 +1,39 @@ import os import subprocess -from symbols_db import DEBUG_MODE, VCPKG_HASH, VCPKG_LOCATION, VCPKG_URL, logger -from symbols_db.handlers.git_handler import git_checkout_commit, git_clone +from symbols_db import VCPKG_LOCATION from symbols_db.handlers.language_handlers import BaseHandler +from symbols_db.projects_compiler.vcpkg import (exec_explorer, + git_checkout_vcpkg_commit, + git_clone_vcpkg, + run_vcpkg_install_command) from symbols_db.utils.utils import subprocess_run_debug class VcpkgHandler(BaseHandler): + strip = False def __init__(self): - pass + git_clone_vcpkg() + git_checkout_vcpkg_commit() + run_vcpkg_install_command() def build(self, project_name): - pass + inst_cmd = f"./vcpkg install {project_name}".split(" ") + inst_run = subprocess.run(inst_cmd, cwd=VCPKG_LOCATION, capture_output=True) + subprocess_run_debug(inst_run, project_name) def find_executables(self, project_name): - pass + project_path = f"{project_name}_x64-linux" + target_directory = VCPKG_LOCATION / "packages" / project_path + return exec_explorer(target_directory) def delete_project_files(self, project_name): pass def get_project_list(self): - pass - - -def git_clone_vcpkg(): - git_clone(VCPKG_URL, VCPKG_LOCATION) - - -def git_checkout_vcpkg_commit(): - git_checkout_commit(VCPKG_LOCATION, VCPKG_HASH) - - -def run_vcpkg_install_command(): - # Linux command - install_command = ["./bootstrap-vcpkg.sh"] - install_run = subprocess.run( - install_command, cwd=VCPKG_LOCATION, capture_output=True - ) - if DEBUG_MODE: - print(install_run.stdout) - logger.debug(f'"bootstrap-vcpkg.sh: ":{install_run.stdout.decode('ascii')}') - - int_command = "./vcpkg integrate install".split(" ") - int_run = subprocess.run(int_command, cwd=VCPKG_LOCATION, capture_output=True) - if DEBUG_MODE: - print(int_run.stdout) - logger.debug(f'"vcpkg integrate install: ":{int_run.stdout.decode('ascii')}') + ports_path = VCPKG_LOCATION / "ports" + return os.listdir(ports_path) def get_vcpkg_projects(): @@ -65,34 +51,6 @@ def vcpkg_build(project_name): subprocess_run_debug(inst_run, project_name) -def exec_explorer(directory): - """ - Walks through a directory and identifies executable files using the `file` command. - - Args: - directory: The directory to search. - - Returns: - A list of executable file paths. - """ - executables = [] - for root, _, files in os.walk(directory): - for file in files: - file_path = os.path.join(root, file) - try: - result = subprocess.run(["file", file_path], capture_output=True) - if b"ELF" in result.stdout: - executables.append(file_path) - if b"archive" in result.stdout: - executables.append(file_path) - except FileNotFoundError: - print( - "Error: 'file' command not found. Make sure it's installed and in your PATH." - ) - return [] - return executables - - def archive_explorer(directory): """ Walks through a directory and identifies executable files using the `file` command. diff --git a/symbols_db/handlers/language_handlers/wrapdb_handler.py b/symbols_db/handlers/language_handlers/wrapdb_handler.py index 497d559..e50ceb1 100644 --- a/symbols_db/handlers/language_handlers/wrapdb_handler.py +++ b/symbols_db/handlers/language_handlers/wrapdb_handler.py @@ -1,15 +1,9 @@ import os from pathlib import Path -from symbols_db import WRAPDB_HASH, WRAPDB_LOCATION, WRAPDB_URL -from symbols_db.handlers.git_handler import git_checkout_commit, git_clone - -def git_clone_wrapdb(): - git_clone(WRAPDB_URL, WRAPDB_LOCATION) - - -def git_checkout_wrapdb_commit(): - git_checkout_commit(WRAPDB_LOCATION, WRAPDB_HASH) +from symbols_db import WRAPDB_LOCATION +from symbols_db.projects_compiler.meson import (git_checkout_wrapdb_commit, + git_clone_wrapdb) def get_wrapdb_projects(): diff --git a/symbols_db/handlers/sqlite_handler.py b/symbols_db/handlers/sqlite_handler.py index 727ef2f..7f5b7da 100644 --- a/symbols_db/handlers/sqlite_handler.py +++ b/symbols_db/handlers/sqlite_handler.py @@ -1,11 +1,10 @@ import datetime import os import sqlite3 -from symbols_db import DEBUG_MODE, BLINTDB_LOCATION -from pathlib import PurePath -from symbols_db import logger from contextlib import closing -from symbols_db import SQLITE_TIMEOUT +from pathlib import PurePath + +from symbols_db import BLINTDB_LOCATION, DEBUG_MODE, SQLITE_TIMEOUT, logger def get_cursor(): diff --git a/symbols_db/projects_compiler/meson.py b/symbols_db/projects_compiler/meson.py new file mode 100644 index 0000000..ed2797a --- /dev/null +++ b/symbols_db/projects_compiler/meson.py @@ -0,0 +1,16 @@ +import shutil + +from symbols_db import WRAPDB_HASH, WRAPDB_LOCATION, WRAPDB_URL +from symbols_db.handlers.git_handler import git_checkout_commit, git_clone + + +def git_clone_wrapdb(): + git_clone(WRAPDB_URL, WRAPDB_LOCATION) + + +def git_checkout_wrapdb_commit(): + git_checkout_commit(WRAPDB_LOCATION, WRAPDB_HASH) + + +def ensure_meson_installed(): + return shutil.which("meson") is not None diff --git a/symbols_db/projects_compiler/vcpkg.py b/symbols_db/projects_compiler/vcpkg.py new file mode 100644 index 0000000..bc0e520 --- /dev/null +++ b/symbols_db/projects_compiler/vcpkg.py @@ -0,0 +1,58 @@ +import subprocess + +from symbols_db import (DEBUG_MODE, VCPKG_HASH, VCPKG_LOCATION, VCPKG_URL, + logger) +from symbols_db.handlers.git_handler import git_checkout_commit, git_clone + + +def git_clone_vcpkg(): + git_clone(VCPKG_URL, VCPKG_LOCATION) + + +def git_checkout_vcpkg_commit(): + git_checkout_commit(VCPKG_LOCATION, VCPKG_HASH) + + +def run_vcpkg_install_command(): + # Linux command + install_command = ["./bootstrap-vcpkg.sh"] + install_run = subprocess.run( + install_command, cwd=VCPKG_LOCATION, capture_output=True + ) + if DEBUG_MODE: + print(install_run.stdout) + logger.debug(f'"bootstrap-vcpkg.sh: ":{install_run.stdout.decode('ascii')}') + + int_command = "./vcpkg integrate install".split(" ") + int_run = subprocess.run(int_command, cwd=VCPKG_LOCATION, capture_output=True) + if DEBUG_MODE: + print(int_run.stdout) + logger.debug(f'"vcpkg integrate install: ":{int_run.stdout.decode('ascii')}') + + +def exec_explorer(directory): + """ + Walks through a directory and identifies executable files using the `file` command. + + Args: + directory: The directory to search. + + Returns: + A list of executable file paths. + """ + executables = [] + for root, _, files in os.walk(directory): + for file in files: + file_path = os.path.join(root, file) + try: + result = subprocess.run(["file", file_path], capture_output=True) + if b"ELF" in result.stdout: + executables.append(file_path) + if b"archive" in result.stdout: + executables.append(file_path) + except FileNotFoundError: + print( + "Error: 'file' command not found. Make sure it's installed and in your PATH." + ) + return [] + return executables diff --git a/symbols_db/utils/__init__.py b/symbols_db/utils/__init__.py index 80b8837..fc9e988 100644 --- a/symbols_db/utils/__init__.py +++ b/symbols_db/utils/__init__.py @@ -1,7 +1,7 @@ -from pathlib import Path import os -from symbols_db import DEBUG_MODE, WRAPDB_LOCATION -from symbols_db import BOM_LOCATION +from pathlib import Path + +from symbols_db import BOM_LOCATION, DEBUG_MODE, WRAPDB_LOCATION HOME_DIRECTORY = Path.home() diff --git a/symbols_db/utils/json.py b/symbols_db/utils/json.py index 98aad36..6d7998a 100644 --- a/symbols_db/utils/json.py +++ b/symbols_db/utils/json.py @@ -1,4 +1,5 @@ import json + from symbols_db import logger diff --git a/symbols_db/utils/rust.py b/symbols_db/utils/rust.py index 6659837..4cdf068 100644 --- a/symbols_db/utils/rust.py +++ b/symbols_db/utils/rust.py @@ -1,8 +1,8 @@ -from symbols_db.utils import HOME_DIRECTORY -from symbols_db import logger - import os +from symbols_db import logger +from symbols_db.utils import HOME_DIRECTORY + def from_purl_to_rust_srcname(purl): return purl.replace("@", "-").split("/")[1] diff --git a/symbols_db/utils/utils.py b/symbols_db/utils/utils.py index 64553fb..907f299 100644 --- a/symbols_db/utils/utils.py +++ b/symbols_db/utils/utils.py @@ -1,4 +1,4 @@ -from symbols_db import DEBUG_MODE, logger, WRAPDB_LOCATION +from symbols_db import DEBUG_MODE, WRAPDB_LOCATION, logger def subprocess_run_debug(setup_run, project_name): diff --git a/tests/scripts/explore_json_all_func.py b/tests/scripts/explore_json_all_func.py index 8fe7771..0ba1855 100644 --- a/tests/scripts/explore_json_all_func.py +++ b/tests/scripts/explore_json_all_func.py @@ -1,4 +1,5 @@ import json + from tests.scripts.match_internal_functions_withdb import get_pname_bname with open("all_ifunc_object.json", "r") as f: diff --git a/tests/scripts/get_internal_functions.py b/tests/scripts/get_internal_functions.py index 46760a8..93faed7 100644 --- a/tests/scripts/get_internal_functions.py +++ b/tests/scripts/get_internal_functions.py @@ -1,16 +1,14 @@ -import json import argparse -import os import concurrent import concurrent.futures +import json +import os from pathlib import Path from symbols_db.handlers.blint_handler import get_properties_internal -from tests.scripts.match_internal_functions_withdb import ( - get_export_id, - get_bid_using_fid, - get_bname, -) +from tests.scripts.match_internal_functions_withdb import (get_bid_using_fid, + get_bname, + get_export_id) def arguments_parser():