diff --git a/.gitmodules b/.gitmodules index c69dc75..db59cf3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "leveldb-mcpe"] path = submodules/leveldb-mcpe url = https://github.com/Amulet-Team/leveldb-mcpe +[submodule "tools/shared"] + path = tools/shared + url = https://github.com/Amulet-Team/cpp-tools diff --git a/tools/generate_pybind_stubs.py b/tools/generate_pybind_stubs.py index 5f5c0a5..cc24260 100644 --- a/tools/generate_pybind_stubs.py +++ b/tools/generate_pybind_stubs.py @@ -1,26 +1,7 @@ import os -import glob import importlib.util -import sys -import subprocess -import re -import pybind11_stubgen -from pybind11_stubgen.structs import Identifier -from pybind11_stubgen.parser.mixins.filter import FilterClassMembers -UnionPattern = re.compile( - r"^(?P[a-zA-Z_][a-zA-Z0-9_]*): types\.UnionType\s*#\s*value = (?P.*)$", - flags=re.MULTILINE, -) -VersionPattern = re.compile(r"(?P[a-zA-Z0-9_].*): str = '.*?'") - - -def union_sub_func(match: re.Match) -> str: - return f'{match.group("variable")}: typing.TypeAlias = {match.group("value")}' - - -def str_sub_func(match: re.Match) -> str: - return f"{match.group('var')}: str" +from shared.generate_pybind_stubs import run def get_module_path(name: str) -> str: @@ -35,143 +16,10 @@ def get_package_dir(name: str) -> str: return os.path.dirname(get_module_path(name)) -def patch_stubgen(): - # Is there a better way to add items to the blacklist? - # Pybind11 - FilterClassMembers._FilterClassMembers__class_member_blacklist.add( - Identifier("_pybind11_conduit_v1_") - ) - # Python - FilterClassMembers._FilterClassMembers__class_member_blacklist.add( - Identifier("__new__") - ) - FilterClassMembers._FilterClassMembers__class_member_blacklist.add( - Identifier("__subclasshook__") - ) - # Pickle - FilterClassMembers._FilterClassMembers__class_member_blacklist.add( - Identifier("__getnewargs__") - ) - FilterClassMembers._FilterClassMembers__class_member_blacklist.add( - Identifier("__getstate__") - ) - FilterClassMembers._FilterClassMembers__class_member_blacklist.add( - Identifier("__setstate__") - ) - # ABC - FilterClassMembers._FilterClassMembers__attribute_blacklist.add( - Identifier("__abstractmethods__") - ) - FilterClassMembers._FilterClassMembers__attribute_blacklist.add( - Identifier("__orig_bases__") - ) - FilterClassMembers._FilterClassMembers__attribute_blacklist.add( - Identifier("__parameters__") - ) - FilterClassMembers._FilterClassMembers__attribute_blacklist.add( - Identifier("_abc_impl") - ) - # Protocol - FilterClassMembers._FilterClassMembers__attribute_blacklist.add( - Identifier("__protocol_attrs__") - ) - FilterClassMembers._FilterClassMembers__attribute_blacklist.add( - Identifier("__non_callable_proto_members__") - ) - FilterClassMembers._FilterClassMembers__attribute_blacklist.add( - Identifier("_is_protocol") - ) - FilterClassMembers._FilterClassMembers__attribute_blacklist.add( - Identifier("_is_runtime_protocol") - ) - # dataclass - FilterClassMembers._FilterClassMembers__attribute_blacklist.add( - Identifier("__dataclass_fields__") - ) - FilterClassMembers._FilterClassMembers__attribute_blacklist.add( - Identifier("__dataclass_params__") - ) - FilterClassMembers._FilterClassMembers__attribute_blacklist.add( - Identifier("__match_args__") - ) - # Buffer protocol - FilterClassMembers._FilterClassMembers__class_member_blacklist.add( - Identifier("__buffer__") - ) - FilterClassMembers._FilterClassMembers__class_member_blacklist.add( - Identifier("__release_buffer__") - ) - - -def main() -> None: +def main(): path = get_package_dir("leveldb") src_path = os.path.dirname(path) - - # Remove all existing stub files - print("Removing stub files...") - for stub_path in glob.iglob( - os.path.join(glob.escape(src_path), "**", "*.pyi"), recursive=True - ): - os.remove(stub_path) - - # Extend pybind11-stubgen - patch_stubgen() - - # Call pybind11-stubgen - print("Running pybind11-stubgen...") - pybind11_stubgen.main([ - f"--output-dir={src_path}", - "leveldb", - ]) - - # Run normal stubgen on the python files - # print("Running stubgen...") - # stubgen.main([ - # *glob.glob( - # os.path.join(glob.escape(src_path), "**", "*.py"), recursive=True - # ), - # "-o", - # src_path, - # "--include-docstrings", - # ]) - - # Remove stub files generated for python modules - for stub_path in glob.iglob( - os.path.join(glob.escape(src_path), "**", "*.pyi"), recursive=True - ): - if os.path.isfile(stub_path[:-1]): - os.remove(stub_path) - - print("Patching stub files...") - # Fix some issues and reformat the stub files. - stub_paths = glob.glob( - os.path.join(glob.escape(src_path), "**", "*.pyi"), recursive=True - ) - for stub_path in stub_paths: - with open(stub_path, encoding="utf-8") as f: - pyi = f.read() - pyi = UnionPattern.sub(union_sub_func, pyi) - pyi = VersionPattern.sub(str_sub_func, pyi) - with open(stub_path, "w", encoding="utf-8") as f: - f.write(pyi) - - subprocess.run( - [ - "isort", - *stub_paths, - ] - ) - - subprocess.run( - [ - "autoflake", - "--in-place", - "--remove-unused-variables", - *stub_paths, - ] - ) - - subprocess.run([sys.executable, "-m", "black", src_path]) + run(src_path, "leveldb") if __name__ == "__main__": diff --git a/tools/generate_vs_sln.py b/tools/generate_vs_sln.py index 64ba60b..a6ed4fb 100644 --- a/tools/generate_vs_sln.py +++ b/tools/generate_vs_sln.py @@ -2,450 +2,13 @@ from __future__ import annotations import os -import re -import uuid -from dataclasses import dataclass, field -from enum import Enum import pybind11 import pybind11_extensions -import sys -import glob -import sysconfig -from collections.abc import Iterable -from hashlib import md5 +from shared.generate_vs_sln import ProjectData, CompileMode, get_files, PythonIncludeDir, PythonLibraryDir, write RootDir = os.path.dirname(os.path.dirname(__file__)) SrcDir = os.path.join(RootDir, "src") -ProjectPattern = re.compile( - r'Project\("{(?P[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})}"\) = "(?P[a-zA-Z0-9_-]+)", "(?P[a-zA-Z0-9\\/_-]+\.vcxproj)", "{(?P[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})}"' -) - -PythonExtensionModuleSuffix = ( - f".cp{sys.version_info.major}{sys.version_info.minor}-win_amd64.pyd" -) - -PythonIncludeDir = sysconfig.get_paths()["include"] -PythonLibraryDir = os.path.join(os.path.dirname(PythonIncludeDir), "libs") - - -VCXProjSource = """\ - """ -VCXProjInclude = """\ - """ - -VCXProj = r""" - - - - Debug - x64 - - - Release - x64 - - - - 17.0 - {{{project_guid}}} - Win32Proj - 10.0 - - - - {library_type} - true - v143 - - - {library_type} - false - v143 - - - - - - - - - - - - - - - $(SolutionDir)$(Platform)\$(Configuration)\int\{project_name}\ - {out_dir} - {file_extension} - $(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64);{library_path} - {ext_name} - - - $(SolutionDir)$(Platform)\$(Configuration)\int\{project_name}\ - {out_dir} - {file_extension} - $(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64);{library_path} - {ext_name} - - - - stdcpp20 - {include_dirs}%(AdditionalIncludeDirectories) - {preprocessor_definitions}%(PreprocessorDefinitions) - - - $(CoreLibraryDependencies);%(AdditionalDependencies);{libraries} - - - - - stdcpp20 - {include_dirs}%(AdditionalIncludeDirectories) - {preprocessor_definitions}%(PreprocessorDefinitions) - - - $(CoreLibraryDependencies);%(AdditionalDependencies);{libraries} - - - -{source_files} - - -{include_files} - - - - -""" - - -VCXProjFiltersSource = """\ - - {rel_path} - """ - - -VCXProjFiltersSourceGroup = """\ - - {{{uuid}}} - -""" - - -VCXProjFiltersInclude = """\ - - {rel_path} - """ - - -VCXProjFiltersIncludeGroup = """\ - - {{{uuid}}} - -""" - - -VCXProjFilters = r""" - - -{filter_groups} - -{source_files} - - -{include_files} - -""" - - -VCXProjUser = r""" - - -""" - - -SolutionHeader = """Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.2.32630.192 -MinimumVisualStudioVersion = 10.0.40219.1 -""" - -SolutionProjectDependency = """\ - {{{dependency_guid}}} = {{{dependency_guid}}} -""" -SolutionProjectDependencies = """\ - ProjectSection(ProjectDependencies) = postProject -{project_dependencies}\ - EndProjectSection -""" - -SolutionProject = """Project("{{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}}") = "{project_name}", "{project_name}.vcxproj", "{{{project_guid}}}" -{project_dependencies}EndProject -""" - -SolutionGlobalConfigurationPlatforms = """\ - {{{project_guid}}}.Debug|x64.ActiveCfg = Debug|x64 - {{{project_guid}}}.Debug|x64.Build.0 = Debug|x64 - {{{project_guid}}}.Debug|x86.ActiveCfg = Debug|x64 - {{{project_guid}}}.Debug|x86.Build.0 = Debug|x64 - {{{project_guid}}}.Release|x64.ActiveCfg = Release|x64 - {{{project_guid}}}.Release|x64.Build.0 = Release|x64 - {{{project_guid}}}.Release|x86.ActiveCfg = Release|x64 - {{{project_guid}}}.Release|x86.Build.0 = Release|x64""" - - -SolutionGlobal = """\ -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|x64 = Debug|x64 - Debug|x86 = Debug|x86 - Release|x64 = Release|x64 - Release|x86 = Release|x86 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution -{configuration_platforms} - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {{6B72B1FC-8248-4021-B089-D05ED8CBCE73}} - EndGlobalSection -EndGlobal -""" - - -class CompileMode(Enum): - DynamicLibrary = "DynamicLibrary" - StaticLibrary = "StaticLibrary" - PythonExtension = "pyd" - - -@dataclass(kw_only=True) -class ProjectData: - name: str - compile_mode: CompileMode - source_files: list[tuple[str, str, str]] = field(default_factory=list) - include_files: list[tuple[str, str, str]] = field(default_factory=list) - include_dirs: list[str] = field(default_factory=list) - preprocessor_definitions: list[str] = field(default_factory=list) - library_dirs: list[str] = field(default_factory=list) - dependencies: list[ProjectData | str] = field(default_factory=list) - py_package: str | None = None - package_dir: str | None = None - - def project_guid(self) -> str: - encoded_name = self.name.encode() - digest = md5(encoded_name).hexdigest() - return ( - digest[0:8].upper() - + "-" - + digest[8:12].upper() - + "-" - + digest[12:16].upper() - + "-" - + digest[16:20].upper() - + "-" - + digest[20:32].upper() - ) - - -def write( - src_dir: str, solution_dir: str, solution_name: str, projects: list[ProjectData] -) -> None: - os.makedirs(solution_dir, exist_ok=True) - for project in projects: - project_guid = project.project_guid() - vcxproj_sources = "\n".join( - VCXProjSource.format(path=os.path.join(*path)) - for path in project.source_files - ) - vcxproj_includes = "\n".join( - VCXProjInclude.format(path=os.path.join(*path)) - for path in project.include_files - ) - library_type = project.compile_mode - project_name = project.name - if library_type == CompileMode.PythonExtension: - extension = PythonExtensionModuleSuffix - library_type = CompileMode.DynamicLibrary - if project.py_package: - module_path = project.py_package.replace(".", "\\") - out_dir = f"{project.package_dir or src_dir}\\{module_path}\\" - project_name = f"{project.py_package}.{project_name}" - else: - out_dir = f"{project.package_dir or src_dir}\\" - elif library_type == CompileMode.StaticLibrary: - extension = ".lib" - out_dir = ( - f"$(SolutionDir)$(Platform)\\$(Configuration)\\out\\{project.name}\\" - ) - # elif library_type == CompileMode.DynamicLibrary: - # extension = ".dll" - # out_dir = f"$(SolutionDir)$(Platform)\\$(Configuration)\\out\\{project.name}\\" - else: - raise RuntimeError - with open( - os.path.join(solution_dir, f"{project_name}.vcxproj"), "w", encoding="utf8" - ) as f: - f.write( - VCXProj.format( - project_name=project_name, - ext_name=project.name, - source_files=vcxproj_sources, - include_files=vcxproj_includes, - include_dirs="".join(f"{path};" for path in project.include_dirs), - preprocessor_definitions="".join(f"{ppd};" for ppd in project.preprocessor_definitions), - library_path="".join( - [f"{path};" for path in project.library_dirs] - + [ - f"$(SolutionDir)$(Platform)\\$(Configuration)\\out\\{dep.name if isinstance(dep, ProjectData) else dep}\\;" - for dep in project.dependencies - if ( - dep.compile_mode == CompileMode.StaticLibrary and dep.source_files - if isinstance(dep, ProjectData) else True - ) - ] - ), - libraries="".join( - f"{dep.name if isinstance(dep, ProjectData) else dep}.lib;" - for dep in project.dependencies - if ( - dep.compile_mode == CompileMode.StaticLibrary and dep.source_files - if isinstance(dep, ProjectData) else True - ) - ), - library_type=library_type.value, - project_guid=project_guid, - file_extension=extension, - out_dir=out_dir, - ) - ) - filter_sources = [] - filter_includes = [] - filter_sources_groups = dict[str, str]() - filter_includes_groups = dict[str, str]() - for path in project.source_files: - filter_sources.append( - VCXProjFiltersSource.format( - path=os.path.join(*path), rel_path=path[1] or "" - ) - ) - rel_path = path[1] - while rel_path: - if rel_path not in filter_sources_groups: - filter_sources_groups[rel_path] = VCXProjFiltersSourceGroup.format( - path=rel_path, uuid=str(uuid.uuid4()) - ) - rel_path = os.path.dirname(rel_path) - for path in project.include_files: - filter_includes.append( - VCXProjFiltersInclude.format( - path=os.path.join(*path), rel_path=path[1] or "" - ) - ) - rel_path = path[1] - while rel_path: - if rel_path not in filter_includes_groups: - filter_includes_groups[rel_path] = ( - VCXProjFiltersIncludeGroup.format( - path=rel_path, uuid=str(uuid.uuid4()) - ) - ) - rel_path = os.path.dirname(rel_path) - with open( - os.path.join(solution_dir, f"{project_name}.vcxproj.filters"), - "w", - encoding="utf8", - ) as f: - f.write( - VCXProjFilters.format( - source_files="\n".join(filter_sources), - include_files="\n".join(filter_includes), - filter_groups="".join(filter_sources_groups.values()) - + "".join(filter_includes_groups.values()), - ) - ) - with open( - os.path.join(solution_dir, f"{project_name}.vcxproj.user"), - "w", - encoding="utf8", - ) as f: - f.write(VCXProjUser) - - # write solution file - with open( - os.path.join(solution_dir, f"{solution_name}.sln"), "w", encoding="utf8" - ) as f: - f.write(SolutionHeader) - global_configuration_platforms = [] - for project in projects: - project_guid = project.project_guid() - project_name = ( - f"{project.py_package}.{project.name}" - if project.py_package - else project.name - ) - if project.dependencies: - project_dependencies = "".join( - SolutionProjectDependency.format(dependency_guid=dep.project_guid()) - for dep in project.dependencies - if isinstance(dep, ProjectData) - ) - project_dependencies = SolutionProjectDependencies.format( - project_dependencies=project_dependencies - ) - else: - project_dependencies = "" - f.write( - SolutionProject.format( - project_name=project_name, - project_guid=project_guid, - project_dependencies=project_dependencies, - ) - ) - global_configuration_platforms.append( - SolutionGlobalConfigurationPlatforms.format(project_guid=project_guid) - ) - f.write( - SolutionGlobal.format( - configuration_platforms="\n".join(global_configuration_platforms) - ) - ) - - -def get_files( - *, - root_dir: str, - ext: str, - root_dir_suffix: str = "", - exclude_dirs: Iterable[str] = (), - exclude_exts: Iterable[str] = (), -) -> list[tuple[str, str, str]]: - """ - Get file paths split into - 1) containing folder ("your/path") - 2) relative path to parent directory within containing folder ("amulet/io") - 3) file name. ("binary_reader.hpp") - get_files("your/path", "hpp") - """ - paths = list[tuple[str, str, str]]() - search_path = root_dir - if root_dir_suffix: - search_path = os.path.join(search_path, root_dir_suffix) - for path in glob.iglob( - os.path.join(glob.escape(search_path), "**", f"*.{ext}"), recursive=True - ): - if any(map(path.startswith, exclude_dirs)): - continue - if any(map(path.endswith, exclude_exts)): - continue - rel_path = os.path.relpath(path, root_dir) - paths.append((root_dir, os.path.dirname(rel_path), os.path.basename(rel_path))) - return paths - def main() -> None: zlib_path = os.path.join(RootDir, "submodules", "zlib") diff --git a/tools/shared b/tools/shared new file mode 160000 index 0000000..145e496 --- /dev/null +++ b/tools/shared @@ -0,0 +1 @@ +Subproject commit 145e496e113179a7e3b805ceb0fd083c2384c1a9