From 0861e5a270e154b1db34dfaca3c73f177732673a Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Fri, 16 Feb 2024 16:09:40 -0500 Subject: [PATCH] Feat: OpenAPI endpoint line numbers (#22) * Add line numbers to x-atom-usages for now. Signed-off-by: Caroline Russell * Bump version, move flake8 config, update docstrings and types. Signed-off-by: Caroline Russell --------- Signed-off-by: Caroline Russell --- .flake8 | 4 + atom_tools/cli/commands/convert.py | 20 +- atom_tools/lib/converter.py | 157 ++++++----- pyproject.toml | 18 +- test/test_converter.py | 407 +++++++++++++++++++---------- 5 files changed, 366 insertions(+), 240 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..9c00701 --- /dev/null +++ b/.flake8 @@ -0,0 +1,4 @@ +[flake8] +max-line-length = 99 +max-complexity = 18 +exclude = ["atom_tools/cli/"] \ No newline at end of file diff --git a/atom_tools/cli/commands/convert.py b/atom_tools/cli/commands/convert.py index e1f828d..3cf4307 100644 --- a/atom_tools/cli/commands/convert.py +++ b/atom_tools/cli/commands/convert.py @@ -80,33 +80,23 @@ def handle(self): """ Executes the convert command and performs the conversion. """ - supported_types = [ - 'java', 'jar', 'python', 'py', 'javascript', 'js', - 'typescript', 'ts' - ] + supported_types = ['java', 'jar', 'python', 'py', 'javascript', 'js', 'typescript', 'ts'] if self.option('type') not in supported_types: - raise ValueError( - f'Unknown origin type: {self.option("type")}' - ) + raise ValueError(f'Unknown origin type: {self.option("type")}') match self.option('format'): case 'openapi3.1.0' | 'openapi3.0.1': converter = OpenAPI( self.option('format'), self.option('type'), self.option('usages-slice'), - # self.option('server'), - # self.option('reachables-slice'), ) - if not (result := converter.endpoints_to_openapi( - self.option('server'))): + if not (result := converter.endpoints_to_openapi(self.option('server'))): logging.error('No results produced!') return 1 - with open(self.option('output-file'), 'w', - encoding='utf-8') as f: + with open(self.option('output-file'), 'w', encoding='utf-8') as f: json.dump(result, f, indent=4, sort_keys=True) - logging.info(f'OpenAPI document written to ' - f'{self.option("output-file")}.') + logging.info(f'OpenAPI document written to {self.option("output-file")}.') case _: raise ValueError( f'Unknown destination format: {self.option("format")}' diff --git a/atom_tools/lib/converter.py b/atom_tools/lib/converter.py index 3795650..a9fa93e 100644 --- a/atom_tools/lib/converter.py +++ b/atom_tools/lib/converter.py @@ -81,9 +81,12 @@ class OpenAPI: regex (RegexCollection): collection of regular expressions Methods: + _create_ln_entries: Creates an x-atom-usages entry. + _filter_matches: Filters a list of matches based on certain criteria. _js_helper: Formats path sections which are parameters correctly. _process_methods_helper: Utility for process_methods. _query_calls_helper: A helper function to query calls. + _remove_nested_parameters: Removes nested path parameters from the get/post/etc. calls_to_params: Transforms a call and endpoint into parameter object. collect_methods: Collects and combines methods that may be endpoints. convert_usages: Converts usages to OpenAPI. @@ -93,7 +96,6 @@ class OpenAPI: endpoints_to_openapi: Generates an OpenAPI document. extract_endpoints: Extracts endpoints from the given code. filter_calls: Filters invokedCalls and argToCalls. - filter_matches: Filters a list of matches based on certain criteria. methods_to_endpoints: Converts a method map to a map of endpoints. populate_endpoints: Populates the endpoints based on the method_map. process_calls: Processes calls and returns a new method map. @@ -112,6 +114,7 @@ def __init__( self.openapi_version = dest_format.replace('openapi', '') self.title = f'OpenAPI Specification for {Path(usages).parent.stem}' self.regex = RegexCollection() + self.file_endpoint_map: Dict = {} def endpoints_to_openapi(self, server: str = '') -> Any: """ @@ -128,12 +131,32 @@ def endpoints_to_openapi(self, server: str = '') -> Any: return output + def create_file_to_method_dict(self, method_map): + """ + Creates a dictionary of endpoints and methods. + """ + full_names = list(method_map.get('full_names').keys()) + file_endpoint_map = {i: [] for i in full_names} + for full_name in full_names: + for values in method_map['full_names'][full_name]['resolved_methods'].values(): + file_endpoint_map[full_name].extend(values.get('endpoints')) + for k, v in file_endpoint_map.items(): + filename = k.split(':')[0] + endpoints = set(v) + for i in endpoints: + if self.file_endpoint_map.get(i): + self.file_endpoint_map[i].add(filename) + else: + self.file_endpoint_map[i] = {filename} + self.file_endpoint_map = {k: list(v) for k, v in self.file_endpoint_map.items()} + def convert_usages(self) -> Dict[str, Any]: """ Converts usages to OpenAPI. """ methods = self.process_methods() methods = self.methods_to_endpoints(methods) + self.create_file_to_method_dict(methods) methods = self.process_calls(methods) return self.populate_endpoints(methods) @@ -177,32 +200,26 @@ def process_methods(self) -> Dict[str, List[str]]: Create a dictionary of full names and their corresponding methods. """ method_map = self._process_methods_helper( - 'objectSlices[].{fullName: fullName, resolvedMethods: usages[' - '].*.resolvedMethod[]}') + 'objectSlices[].{fullName: fullName, resolvedMethods: usages[].*.resolvedMethod[]}') calls = self._process_methods_helper( - 'objectSlices[].{fullName: fullName, resolvedMethods: usages[].*[' - '][?resolvedMethod].resolvedMethod[]}') + 'objectSlices[].{fullName: fullName, resolvedMethods: usages[].*[][?resolvedMethod].' + 'resolvedMethod[]}') user_defined_types = self._process_methods_helper( - 'userDefinedTypes[].{fullName: name, resolvedMethods: fields[' - '].name}') + 'userDefinedTypes[].{fullName: name, resolvedMethods: fields[].name}') for key, value in calls.items(): if method_map.get(key): - method_map[key]['resolved_methods'].extend( - value.get('resolved_methods')) + method_map[key]['resolved_methods'].extend(value.get('resolved_methods')) else: - method_map[key] = { - 'resolved_methods': value.get('resolved_methods')} + method_map[key] = {'resolved_methods': value.get('resolved_methods')} for key, value in user_defined_types.items(): if method_map.get(key): - method_map[key]['resolved_methods'].extend( - value.get('resolved_methods')) + method_map[key]['resolved_methods'].extend(value.get('resolved_methods')) else: - method_map[key] = { - 'resolved_methods': value.get('resolved_methods')} + method_map[key] = {'resolved_methods': value.get('resolved_methods')} for k, v in method_map.items(): method_map[k] = list(set(v.get('resolved_methods'))) @@ -228,8 +245,7 @@ def query_calls(self, full_name: str, resolved_methods: List[str]) -> List: calls.append(call) return calls - def _query_calls_helper( - self, full_name: str, call_type_str: str) -> List[Dict]: + def _query_calls_helper(self, full_name: str, call_type_str: str) -> List[Dict]: """ A function to help query calls. @@ -240,8 +256,7 @@ def _query_calls_helper( Returns: list: The result of searching for the calls pattern in the usages. """ - pattern = (f'objectSlices[].usages[?fullName==' - f'{json.dumps(full_name)}{call_type_str}') + pattern = f'objectSlices[].usages[?fullName=={json.dumps(full_name)}{call_type_str}' compiled_pattern = jmespath.compile(pattern) return compiled_pattern.search(self.usages.content) @@ -254,20 +269,18 @@ def process_calls(self, method_map: Dict) -> Dict[str, Any]: dict: A new method map containing calls. """ for full_name, resolved_methods in method_map['full_names'].items(): - if res := self.query_calls( - full_name, resolved_methods['resolved_methods'].keys()): + if res := self.query_calls(full_name, resolved_methods['resolved_methods'].keys()): mmap = self.filter_calls(res, resolved_methods) else: mmap = self.filter_calls([], resolved_methods) - method_map['full_names'][full_name]['resolved_methods'] = ( - mmap.get('resolved_methods')) + + method_map['full_names'][full_name]['resolved_methods'] = mmap.get('resolved_methods') return method_map @staticmethod def filter_calls( - queried_calls: List[Dict[str, Any]], resolved_methods: Dict - ) -> Dict[str, List]: + queried_calls: List[Dict[str, Any]], resolved_methods: Dict) -> Dict[str, List]: """ Iterate through the invokedCalls and argToCalls and create a relevant dictionary of endpoints and calls. @@ -282,12 +295,15 @@ def filter_calls( i for i in queried_calls if i.get('resolvedMethod', '') == method ] - resolved_methods['resolved_methods'][method].update( - {'calls': calls}) + lns = [ + i.get('lineNumber') + for i in calls + if i.get('lineNumber') and i.get('resolvedMethod', '') == method + ] + resolved_methods['resolved_methods'][method].update({'calls': calls, 'line_nos': lns}) return resolved_methods - def methods_to_endpoints( - self, method_map: Dict[str, Any]) -> Dict[str, Any]: + def methods_to_endpoints(self, method_map: Dict[str, Any]) -> Dict[str, Any]: """ Convert a method map to a map of endpoints. @@ -340,8 +356,10 @@ def _process_methods_helper(self, pattern: str) -> Dict[str, Any]: """ dict_resolved_pattern = jmespath.compile(pattern) - result = [i for i in dict_resolved_pattern.search(self.usages.content) - if i.get('resolvedMethods')] + result = [ + i for i in dict_resolved_pattern.search(self.usages.content) + if i.get('resolvedMethods') + ] resolved: Dict = {} for r in result: @@ -364,9 +382,9 @@ def populate_endpoints(self, method_map: Dict) -> Dict[str, Any]: """ paths_object: Dict = {} for resolved_methods in method_map.values(): - for value in resolved_methods.values(): + for key, value in resolved_methods.items(): for m in value['resolved_methods'].items(): - new_path_item = self.create_paths_item(m) + new_path_item = self.create_paths_item(key, m) if paths_object: paths_object = self.merge_path_objects( paths_object, new_path_item) @@ -394,10 +412,11 @@ def merge_path_objects(p1: Dict, p2: Dict) -> Dict: p1[key] = value return p1 - def create_paths_item(self, paths_dict: Dict) -> Dict: + def create_paths_item(self, filename: str, paths_dict: Dict) -> Dict: """ Create paths item object based on provided endpoints and calls. Args: + filename (str): The name of the file paths_dict (dict): The object containing endpoints and calls Returns: dict: The paths item object @@ -405,6 +424,7 @@ def create_paths_item(self, paths_dict: Dict) -> Dict: """ endpoints = paths_dict[1].get('endpoints') calls = paths_dict[1].get('calls') + line_numbers = paths_dict[1].get('line_nos') paths_object: Dict = {} for ep in endpoints: @@ -424,6 +444,9 @@ def create_paths_item(self, paths_dict: Dict) -> Dict: paths_item_object |= self.calls_to_params(ep, call) else: paths_item_object |= self.calls_to_params(ep, None) + if line_numbers and (line_nos := self._create_ln_entries( + filename, list(set(line_numbers)))): + paths_item_object |= line_nos if paths_item_object: if paths_object.get(ep): paths_object[ep] |= paths_item_object @@ -432,10 +455,25 @@ def create_paths_item(self, paths_dict: Dict) -> Dict: else: paths_object[ep] = {} - return self.remove_nested_parameters(paths_object) + return self._remove_nested_parameters(paths_object) + + @staticmethod + def _create_ln_entries(filename, line_numbers): + """ + Creates line number entries for a given filename and line numbers. + + Args: + filename (str): The name of the file. + line_numbers (list): A list of line numbers. + + Returns: + dict: A dictionary containing line number entries. + """ + fn = filename.split(':')[0] + return {'x-atom-usages': {fn: line_numbers}} @staticmethod - def remove_nested_parameters(data: Dict) -> Dict[str, Dict | List]: + def _remove_nested_parameters(data: Dict) -> Dict[str, Dict | List]: """ Removes nested path parameters from the given data. @@ -447,8 +485,7 @@ def remove_nested_parameters(data: Dict) -> Dict[str, Dict | List]: """ for value in data.values(): for v in value.values(): - if isinstance(v, dict) and "parameters" in v and isinstance( - v["parameters"], list): + if isinstance(v, dict) and "parameters" in v and isinstance(v["parameters"], list): v["parameters"] = [param for param in v["parameters"] if param.get("in") != "path"] return data @@ -467,15 +504,10 @@ def determine_operations(call: Dict, params: List) -> Dict[str, Any]: parameters and responses. """ ops = {'get', 'put', 'post', 'delete', 'options', 'head', 'patch'} - if found := [ - op for op in ops if op in call.get('resolvedMethod', '').lower() - ]: + if found := [op for op in ops if op in call.get('resolvedMethod', '').lower()]: if params: - return {op: { - 'parameters': params, 'responses': {}} for op in found - } - return { - op: {'responses': {}} for op in found} + return {op: {'parameters': params, 'responses': {}} for op in found} + return {op: {'responses': {}} for op in found} return {'parameters': params} if params else {} def calls_to_params(self, ep: str, call: Dict | None) -> Dict[str, Any]: @@ -517,15 +549,9 @@ def create_param_object(self, ep: str, call: Dict | None) -> List[Dict]: if not params and call: ptypes = set(call.get('paramTypes', [])) if len(ptypes) > 1: - params = [ - {'name': param, 'in': 'header'} - for param in ptypes if param != 'ANY' - ] + params = [{'name': param, 'in': 'header'} for param in ptypes if param != 'ANY'] else: - params = [ - {'name': param, 'in': 'header'} - for param in ptypes - ] + params = [{'name': param, 'in': 'header'} for param in ptypes] return params def collect_methods(self) -> List: @@ -537,14 +563,11 @@ def collect_methods(self) -> List: list: A list of unique methods. """ # Surely there is a way to combine these... - target_obj_pattern = jmespath.compile( - 'objectSlices[].usages[].targetObj.resolvedMethod') - defined_by_pattern = jmespath.compile( - 'objectSlices[].usages[].definedBy.resolvedMethod') - invoked_calls_pattern = jmespath.compile( - 'objectSlices[].usages[].invokedCalls[].resolvedMethod') - udt_jmespath_query = jmespath.compile( - 'userDefinedTypes[].fields[].name') + target_obj_pattern = jmespath.compile('objectSlices[].usages[].targetObj.resolvedMethod') + defined_by_pattern = jmespath.compile('objectSlices[].usages[].definedBy.resolvedMethod') + invoked_calls_pattern = jmespath.compile('objectSlices[].usages[].invokedCalls[].resolved' + 'Method') + udt_jmespath_query = jmespath.compile('userDefinedTypes[].fields[].name') methods = target_obj_pattern.search(self.usages.content) or [] methods.extend(defined_by_pattern.search(self.usages.content) or []) methods.extend(invoked_calls_pattern.search(self.usages.content) or []) @@ -567,10 +590,10 @@ def extract_endpoints(self, method: str) -> List[str]: return endpoints if not (matches := re.findall(self.regex.endpoints, method)): return endpoints - matches = self.filter_matches(matches, method) + matches = self._filter_matches(matches, method) return [v for v in matches if v] - def filter_matches(self, matches: List[str], code: str) -> List[str]: + def _filter_matches(self, matches: List[str], code: str) -> List[str]: """ Filters a list of matches based on certain criteria. @@ -592,11 +615,7 @@ def filter_matches(self, matches: List[str], code: str) -> List[str]: ): return filtered_matches case 'js' | 'ts' | 'javascript' | 'typescript': - if ( - 'app.' not in code and - 'route' not in code and - 'ftp' not in code - ): + if 'app.' not in code and 'route' not in code and 'ftp' not in code: return filtered_matches for m in matches: diff --git a/pyproject.toml b/pyproject.toml index 61af7b7..52aa87f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,14 +1,11 @@ [project] name = "atom-tools" -version = "0.1.2" +version = "0.2.0" description = "Collection of tools for use with AppThreat/atom." authors = [ - { name = "Team AppThreat", email = "hello@appthreat.com" }, -] -dependencies = ["cleo>=1.0.0", "jmespath>=1.0.0"] -maintainers = [ { name = "Caroline Russell", email = "caroline@appthreat.dev" }, ] +dependencies = ["cleo>=1.0.0", "jmespath>=1.0.0"] license = { text = "Apache-2.0" } readme = "README.md" requires-python = ">=3.10" @@ -47,7 +44,7 @@ dev = [ ] [build-system] -requires = ["setuptools>=61", "wheel"] +requires = ["setuptools>=65", "wheel"] build-backend = "setuptools.build_meta" [tool.setuptools] @@ -59,13 +56,8 @@ packages = [ ] include-package-data = true -[tool.flake8] -max-line-length = 79 -max-complexity = 18 -exclude = ["atom_tools/cli/"] - [tool.black] -line-length = 79 +line-length = 99 [tool.isort] profile = "black" @@ -74,7 +66,7 @@ profile = "black" max-args = 6 [tool.pylint.format] -max-line-length = 79 +max-line-length = 99 [tool.pylint.logging] logging-format-style = "new" diff --git a/test/test_converter.py b/test/test_converter.py index b8ac146..d5b9c5c 100644 --- a/test/test_converter.py +++ b/test/test_converter.py @@ -383,7 +383,7 @@ def test_populate_endpoints(js_usages_1, js_usages_2): '/we/may/also/instruct/you/to/refuse/all/reasonably/necessary/responsibility', '/x-powered-by'] assert list( - result['/rest/continue-code-findIt/apply/{continueCode}'].keys()) == ['parameters', 'put'] + result['/rest/continue-code-findIt/apply/{continueCode}'].keys()) == ['parameters', 'put', 'x-atom-usages'] methods = js_usages_2.process_methods() methods = js_usages_2.methods_to_endpoints(methods) @@ -419,7 +419,7 @@ def test_populate_endpoints(js_usages_1, js_usages_2): '/maxAge']}}}}} methods = js_usages_2.process_calls(methods) result = js_usages_2.populate_endpoints(methods) - assert len(list(result['/login'].keys())) == 2 + assert len(list(result['/login'].keys())) == 3 result = list(result.keys()) result.sort() assert result == ['/', '/allocations/{userId}', '/app/assets/favicon.ico', @@ -435,150 +435,261 @@ def test_usages_class(java_usages_1): def test_convert_usages(java_usages_1, java_usages_2, js_usages_1, js_usages_2, py_usages_1, py_usages_2): - assert java_usages_1.convert_usages() == {'/': {'post': {'responses': {}}}, + assert java_usages_1.convert_usages() == {'/': {'post': {'responses': {}}, + 'x-atom-usages': {'com.piggymetrics.account.controller.AccountController.createNewAccount': [35]}}, '/accounts/{accountName}': {'get': {'responses': {}}, 'parameters': [{'in': 'path', 'name': 'accountName', - 'required': True}]}, - '/current': {'get': {'responses': {}}, 'put': {'responses': {}}}, - '/latest': {'get': {'responses': {}}}, + 'required': True}], + 'x-atom-usages': {'com.piggymetrics.notification.client.AccountServiceClient.getAccount': [12]}}, + '/current': {'get': {'responses': {}}, + 'put': {'responses': {}}, + 'x-atom-usages': {'com.piggymetrics.statistics.controller.StatisticsController.getCurrentAccountStatistics': [20, + 22]}}, + '/latest': {'get': {'responses': {}}, + 'x-atom-usages': {'com.piggymetrics.statistics.client.ExchangeRatesClient.getRates': [13]}}, '/statistics/{accountName}': {'parameters': [{'in': 'path', 'name': 'accountName', 'required': True}], - 'put': {'responses': {}}}, - '/uaa/users': {'post': {'responses': {}}}, + 'put': {'responses': {}}, + 'x-atom-usages': {'com.piggymetrics.account.client.StatisticsServiceClient.updateStatistics': [13]}}, + '/uaa/users': {'post': {'responses': {}}, + 'x-atom-usages': {'com.piggymetrics.account.client.AuthServiceClient.createUser': [12]}}, '/{accountName}': {'get': {'responses': {}}, 'parameters': [{'in': 'path', 'name': 'accountName', 'required': True}], - 'put': {'responses': {}}}, + 'put': {'responses': {}}, + 'x-atom-usages': {'com.piggymetrics.statistics.controller.StatisticsController.saveAccountStatistics': [32]}}, '/{name}': {'get': {'responses': {}}, - 'parameters': [{'in': 'path', 'name': 'name', 'required': True}]}} - assert java_usages_2.convert_usages() == {'/': {'get': {'responses': {}}}, - '/*': {'get': {'responses': {}}}, - '/Digester/sec': {'post': {'responses': {}}}, - '/Digester/vuln': {'post': {'responses': {}}}, - '/DocumentBuilder/Sec': {'post': {'responses': {}}}, - '/DocumentBuilder/vuln': {'post': {'responses': {}}}, - '/DocumentBuilder/xinclude/sec': {'post': {'responses': {}}}, - '/DocumentBuilder/xinclude/vuln': {'post': {'responses': {}}}, - '/DocumentHelper/vuln': {'post': {'responses': {}}}, - '/HttpSyncClients/vuln': {'get': {'responses': {}}}, - '/HttpURLConnection/sec': {'get': {'responses': {}}}, - '/HttpURLConnection/vuln': {'get': {'responses': {}}}, - '/IOUtils/sec': {'get': {'responses': {}}}, - '/ImageIO/sec': {'get': {'responses': {}}}, - '/Jsoup/sec': {'get': {'responses': {}}}, - '/ProcessBuilder': {'get': {'responses': {}}}, - '/SAXBuilder/sec': {'post': {'responses': {}}}, - '/SAXBuilder/vuln': {'post': {'responses': {}}}, - '/SAXParser/sec': {'post': {'responses': {}}}, - '/SAXParser/vuln': {'post': {'responses': {}}}, - '/SAXReader/sec': {'post': {'responses': {}}}, - '/SAXReader/vuln': {'post': {'responses': {}}}, - '/XMLReader/sec': {'post': {'responses': {}}}, - '/XMLReader/vuln': {'post': {'responses': {}}}, - '/aa': {}, - '/any': {'get': {'responses': {}}}, - '/appInfo': {}, - '/application/javascript': {'get': {'responses': {}}}, - '/classloader': {}, - '/codeinject': {'get': {'responses': {}}}, - '/codeinject/host': {'get': {'responses': {}}}, - '/codeinject/sec': {'get': {'responses': {}}}, - '/commonsHttpClient/sec': {'get': {'responses': {}}}, - '/createToken': {'get': {'responses': {}}}, - '/deserialize': {'post': {'responses': {}}}, - '/dnsrebind/vuln': {'get': {'responses': {}}}, - '/exclued/vuln': {'get': {'responses': {}}}, - '/fastjsonp/getToken': {'get': {'responses': {}}}, - '/forward': {}, - '/getName': {'get': {'responses': {}}}, - '/getToken': {'get': {'responses': {}}}, - '/groovy': {'get': {'responses': {}}}, - '/httpclient/sec': {'get': {'responses': {}}}, - '/hutool/vuln': {'get': {'responses': {}}}, - '/index': {}, - '/jdbc/ps/vuln': {}, - '/jdbc/sec': {}, - '/jdbc/vuln': {}, - '/jscmd': {'get': {'responses': {}}}, - '/log4j': {'get': {'responses': {}}}, - '/login': {}, - '/logout': {'get': {'responses': {}}}, - '/mybatis/orderby/sec04': {'get': {'responses': {}}}, - '/mybatis/orderby/vuln03': {'get': {'responses': {}}}, - '/mybatis/sec01': {'get': {'responses': {}}}, - '/mybatis/sec02': {'get': {'responses': {}}}, - '/mybatis/sec03': {'get': {'responses': {}}}, - '/mybatis/vuln01': {'get': {'responses': {}}}, - '/mybatis/vuln02': {'get': {'responses': {}}}, - '/noproxy': {}, - '/object2jsonp': {}, - '/okhttp/sec': {'get': {'responses': {}}}, - '/openStream': {'get': {'responses': {}}}, - '/path_traversal/sec': {'get': {'responses': {}}}, - '/path_traversal/vul': {'get': {'responses': {}}}, - '/pic': {'get': {'responses': {}}}, - '/post': {'post': {'responses': {}}}, - '/postgresql': {'post': {'responses': {}}}, - '/proxy': {}, - '/readxlsx': {'post': {'responses': {}}}, - '/redirect': {'get': {'responses': {}}}, - '/reflect': {}, - '/rememberMe/security': {}, - '/rememberMe/vuln': {}, - '/request/sec': {'get': {'responses': {}}}, - '/restTemplate/vuln1': {'get': {'responses': {}}}, - '/restTemplate/vuln2': {'get': {'responses': {}}}, - '/runtime/exec': {'get': {'responses': {}}}, - '/safe': {}, - '/safecode': {}, - '/sec': {'get': {'responses': {}}}, - '/sec/array_indexOf': {'get': {'responses': {}}}, - '/sec/checkOrigin': {'get': {'responses': {}}}, - '/sec/checkReferer': {}, - '/sec/corsFilter': {}, - '/sec/crossOrigin': {'get': {'responses': {}}}, - '/sec/httpCors': {'get': {'responses': {}}}, - '/sec/originFilter': {'get': {'responses': {}}}, - '/sec/webMvcConfigurer': {'get': {'responses': {}}}, - '/sec/yarm': {'get': {'responses': {}}}, - '/sendRedirect': {}, - '/sendRedirect/sec': {}, - '/setHeader': {'head': {'responses': {}}}, - '/spel/vuln': {'get': {'responses': {}}}, - '/status': {'get': {'responses': {}}}, - '/stored/show': {}, - '/stored/store': {}, - '/upload': {'get': {'responses': {}}, 'post': {'responses': {}}}, - '/upload/picture': {'post': {'responses': {}}}, - '/urlConnection/sec': {'get': {'responses': {}}}, - '/urlConnection/vuln': {'get': {'responses': {}}, 'post': {'responses': {}}}, - '/velocity': {'get': {'responses': {}}}, - '/vuln/contains': {'get': {'responses': {}}}, - '/vuln/crossOrigin': {}, - '/vuln/emptyReferer': {}, - '/vuln/endsWith': {'get': {'responses': {}}}, - '/vuln/mappingJackson2JsonView': {}, - '/vuln/origin': {'get': {'responses': {}}}, - '/vuln/referer': {}, - '/vuln/regex': {'get': {'responses': {}}}, - '/vuln/setHeader': {'get': {'responses': {}}, 'head': {'responses': {}}}, - '/vuln/url_bypass': {'get': {'responses': {}}}, - '/vuln/yarm': {'get': {'responses': {}}}, - '/vuln01': {'get': {'responses': {}}}, - '/vuln02': {'get': {'responses': {}}}, - '/vuln03': {'get': {'responses': {}}}, - '/vuln04': {'get': {'responses': {}}}, - '/vuln05': {'get': {'responses': {}}}, - '/vuln06': {'get': {'responses': {}}}, - '/websocket/cmd': {}, - '/websocket/proxy': {}, - '/xmlReader/sec': {'post': {'responses': {}}}, - '/xmlReader/vuln': {'post': {'responses': {}}}, - '/xmlbeam/vuln': {'post': {'responses': {}}}, - '/xstream': {'post': {'responses': {}}}} + 'parameters': [{'in': 'path', 'name': 'name', 'required': True}], + 'x-atom-usages': {'com.piggymetrics.account.controller.AccountController.getAccountByName': [20]}}} + assert java_usages_2.convert_usages() == {'/': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Test.Index': [15]}}, + '/*': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Cors.vuls3': [41]}}, + '/Digester/sec': {'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.XXE.DigesterSec': [213]}}, + '/Digester/vuln': {'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.XXE.DigesterVuln': [198]}}, + '/DocumentBuilder/Sec': {'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.XXE.DocumentBuilderSec': [263]}}, + '/DocumentBuilder/vuln': {'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.XXE.DocumentBuilderVuln': [236]}}, + '/DocumentBuilder/xinclude/sec': {'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.XXE.DocumentBuilderXincludeSec': [312]}}, + '/DocumentBuilder/xinclude/vuln': {'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.XXE.DocumentBuilderXincludeVuln': [286]}}, + '/DocumentHelper/vuln': {'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.XXE.DocumentHelper': [388]}}, + '/HttpSyncClients/vuln': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SSRF.HttpSyncClients': [265]}}, + '/HttpURLConnection/sec': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SSRF.httpURLConnection': [74]}}, + '/HttpURLConnection/vuln': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SSRF.httpURLConnectionVuln': [87]}}, + '/IOUtils/sec': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SSRF.IOUtils': [246]}}, + '/ImageIO/sec': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SSRF.ImageIO': [153]}}, + '/Jsoup/sec': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SSRF.Jsoup': [226]}}, + '/ProcessBuilder': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Rce.processBuilder': [64]}}, + '/SAXBuilder/sec': {'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.XXE.SAXBuilderSec': [102]}}, + '/SAXBuilder/vuln': {'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.XXE.SAXBuilderVuln': [86]}}, + '/SAXParser/sec': {'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.XXE.SAXParserSec': [178]}}, + '/SAXParser/vuln': {'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.XXE.SAXParserVuln': [160]}}, + '/SAXReader/sec': {'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.XXE.SAXReaderSec': [141]}}, + '/SAXReader/vuln': {'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.XXE.SAXReaderVuln': [123]}}, + '/XMLReader/sec': {'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.XXE.XMLReaderSec': [362]}}, + '/XMLReader/vuln': {'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.XXE.XMLReaderVuln': [342]}}, + '/aa': {'x-atom-usages': {'org.joychou.controller.Test.test': [27]}}, + '/any': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.FileUpload.index': [39]}}, + '/appInfo': {'x-atom-usages': {'org.joychou.controller.Index.appInfo': [24]}}, + '/application/javascript': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Jsonp.safecode': [102]}}, + '/classloader': {'x-atom-usages': {'org.joychou.controller.ClassDataLoader.classData': [15]}}, + '/codeinject': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.CommandInject.codeInject': [24]}}, + '/codeinject/host': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.CommandInject.codeInjectHost': [39]}}, + '/codeinject/sec': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.CommandInject.codeInjectSec': [51]}}, + '/commonsHttpClient/sec': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SSRF.commonsHttpClient': [207]}}, + '/createToken': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Jwt.createToken': [31]}}, + '/deserialize': {'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Fastjson.Deserialize': [17]}}, + '/dnsrebind/vuln': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SSRF.DnsRebind': [308]}}, + '/exclued/vuln': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.GetRequestURI.exclued': [34]}}, + '/fastjsonp/getToken': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Jsonp.getCsrfToken2': [128]}}, + '/forward': {'x-atom-usages': {'org.joychou.controller.URLRedirect.forward': [64]}}, + '/getName': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Jwt.getNickname': [56]}}, + '/getToken': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Jsonp.getCsrfToken1': [118]}}, + '/groovy': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Rce.groovyshell': [128]}}, + '/httpclient/sec': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SSRF.HttpClient': [187]}}, + '/hutool/vuln': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SSRF.hutoolHttp': [298]}}, + '/index': {'x-atom-usages': {'org.joychou.controller.Index.index': [46]}}, + '/jdbc/ps/vuln': {'x-atom-usages': {'org.joychou.controller.SQLI.jdbc_ps_vuln': [138]}}, + '/jdbc/sec': {'x-atom-usages': {'org.joychou.controller.SQLI.jdbc_sqli_sec': [94]}}, + '/jdbc/vuln': {'x-atom-usages': {'org.joychou.controller.SQLI.jdbc_sqli_vul': [51]}}, + '/jscmd': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Rce.jsEngine': [96]}}, + '/log4j': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Log4j.log4j': [19]}}, + '/login': {'x-atom-usages': {'org.joychou.controller.Login.login': [22]}}, + '/logout': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Login.logoutPage': [27]}}, + '/mybatis/orderby/sec04': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SQLI.mybatisOrderBySec04': [240]}}, + '/mybatis/orderby/vuln03': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SQLI.mybatisVuln03': [201]}}, + '/mybatis/sec01': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SQLI.mybatisSec01': [211]}}, + '/mybatis/sec02': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SQLI.mybatisSec02': [220]}}, + '/mybatis/sec03': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SQLI.mybatisSec03': [230]}}, + '/mybatis/vuln01': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SQLI.mybatisVuln01': [181]}}, + '/mybatis/vuln02': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SQLI.mybatisVuln02': [191]}}, + '/noproxy': {'x-atom-usages': {'org.joychou.controller.IPForge.noProxy': [20]}}, + '/object2jsonp': {'x-atom-usages': {'org.joychou.controller.Jsonp.advice': [76]}}, + '/okhttp/sec': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SSRF.okhttp': [168]}}, + '/openStream': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SSRF.openStream': [118]}}, + '/path_traversal/sec': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.PathTraversal.getImageSec': [29]}}, + '/path_traversal/vul': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.PathTraversal.getImage': [24]}}, + '/pic': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.FileUpload.uploadPic': [45]}}, + '/post': {'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.CSRF.post': [24]}}, + '/postgresql': {'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Rce.postgresql': [137]}}, + '/proxy': {'x-atom-usages': {'org.joychou.controller.IPForge.proxy': [31]}}, + '/readxlsx': {'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.othervulns.xlsxStreamerXXE.xllx_streamer_xxe': [35, + 43]}}, + '/redirect': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.URLRedirect.redirect': [31]}}, + '/reflect': {'x-atom-usages': {'org.joychou.controller.XSS.reflect': [27]}}, + '/rememberMe/security': {'x-atom-usages': {'org.joychou.controller.Deserialize.rememberMeBlackClassCheck': [60]}}, + '/rememberMe/vuln': {'x-atom-usages': {'org.joychou.controller.Deserialize.rememberMeVul': [35]}}, + '/request/sec': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SSRF.request': [97]}}, + '/restTemplate/vuln1': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SSRF.RestTemplateUrlBanRedirects': [277]}}, + '/restTemplate/vuln2': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SSRF.RestTemplateUrl': [285]}}, + '/runtime/exec': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Rce.CommandExec': [31]}}, + '/safe': {'x-atom-usages': {'org.joychou.controller.XSS.safe': [65]}}, + '/safecode': {'x-atom-usages': {'org.joychou.controller.CRLFInjection.crlf': [20]}}, + '/sec': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.URLWhiteList.sec': [125]}}, + '/sec/array_indexOf': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.URLWhiteList.sec_array_indexOf': [151]}}, + '/sec/checkOrigin': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Cors.seccode': [104]}}, + '/sec/checkReferer': {'x-atom-usages': {'org.joychou.controller.Jsonp.safecode': [102]}}, + '/sec/corsFilter': {'x-atom-usages': {'org.joychou.controller.Cors.getCsrfToken_04': [98]}}, + '/sec/crossOrigin': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Cors.secCrossOrigin': [54]}}, + '/sec/httpCors': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Cors.getCsrfToken_02': [76]}}, + '/sec/originFilter': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Cors.getCsrfToken_03': [87]}}, + '/sec/webMvcConfigurer': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Cors.getCsrfToken_01': [65]}}, + '/sec/yarm': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Rce.secYarm': [118]}}, + '/sendRedirect': {'x-atom-usages': {'org.joychou.controller.URLRedirect.sendRedirect': [52]}}, + '/sendRedirect/sec': {'x-atom-usages': {'org.joychou.controller.URLRedirect.sendRedirect_seccode': [81]}}, + '/setHeader': {'head': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.URLRedirect.setHeader': [40]}}, + '/spel/vuln': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SpEL.rce': [24]}}, + '/status': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.FileUpload.uploadStatus': [76]}}, + '/stored/show': {'x-atom-usages': {'org.joychou.controller.XSS.show': [55]}}, + '/stored/store': {'x-atom-usages': {'org.joychou.controller.XSS.store': [40]}}, + '/upload': {'get': {'responses': {}}, + 'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.othervulns.xlsxStreamerXXE.index': [29, + 37]}}, + '/upload/picture': {'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.FileUpload.uploadPicture': [82]}}, + '/urlConnection/sec': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SSRF.URLConnectionSec': [50]}}, + '/urlConnection/vuln': {'get': {'responses': {}}, + 'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SSRF.URLConnectionVuln': [44]}}, + '/velocity': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.SSTI.velocity': [26]}}, + '/vuln/contains': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.URLWhiteList.contains': [56]}}, + '/vuln/crossOrigin': {'x-atom-usages': {'org.joychou.controller.Cors.vuls3': [42]}}, + '/vuln/emptyReferer': {'x-atom-usages': {'org.joychou.controller.Jsonp.emptyReferer': [57]}}, + '/vuln/endsWith': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.URLWhiteList.endsWith': [36]}}, + '/vuln/mappingJackson2JsonView': {'x-atom-usages': {'org.joychou.controller.Jsonp.mappingJackson2JsonView': [89]}}, + '/vuln/origin': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Cors.vuls1': [25]}}, + '/vuln/referer': {'x-atom-usages': {'org.joychou.controller.Jsonp.referer': [45]}}, + '/vuln/regex': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.URLWhiteList.regex': [74]}}, + '/vuln/setHeader': {'get': {'responses': {}}, + 'head': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Cors.vuls2': [33]}}, + '/vuln/url_bypass': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.URLWhiteList.url_bypass': [98]}}, + '/vuln/yarm': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Rce.yarm': [112]}}, + '/vuln01': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Cookies.vuln01': [25]}}, + '/vuln02': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Cookies.vuln02': [32]}}, + '/vuln03': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Cookies.vuln03': [45]}}, + '/vuln04': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Cookies.vuln04': [61]}}, + '/vuln05': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Cookies.vuln05': [76]}}, + '/vuln06': {'get': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.Cookies.vuln06': [82]}}, + '/websocket/cmd': {'x-atom-usages': {'org.joychou.controller.WebSockets.cmdInject': [30]}}, + '/websocket/proxy': {'x-atom-usages': {'org.joychou.controller.WebSockets.proxyInject': [53]}}, + '/xmlReader/sec': {'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.XXE.xmlReaderSec': [63]}}, + '/xmlReader/vuln': {'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.XXE.xmlReaderVuln': [48]}}, + '/xmlbeam/vuln': {'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.XXE.post': [419]}}, + '/xstream': {'post': {'responses': {}}, + 'x-atom-usages': {'org.joychou.controller.XStreamRce.parseXml': [23]}}} assert len(js_usages_1.convert_usages()) == 142 assert len(js_usages_2.convert_usages()) == 21 assert py_usages_2.convert_usages() == {'/': {}, @@ -644,24 +755,34 @@ def test_endpoints_to_openapi(java_usages_1): assert java_usages_1.endpoints_to_openapi() == {'info': {'title': 'OpenAPI Specification for data', 'version': '1.0.0'}, 'openapi': '3.1.0', - 'paths': {'/': {'post': {'responses': {}}}, + 'paths': {'/': {'post': {'responses': {}}, + 'x-atom-usages': {'com.piggymetrics.account.controller.AccountController.createNewAccount': [35]}}, '/accounts/{accountName}': {'get': {'responses': {}}, 'parameters': [{'in': 'path', 'name': 'accountName', - 'required': True}]}, - '/current': {'get': {'responses': {}}, 'put': {'responses': {}}}, - '/latest': {'get': {'responses': {}}}, + 'required': True}], + 'x-atom-usages': {'com.piggymetrics.notification.client.AccountServiceClient.getAccount': [12]}}, + '/current': {'get': {'responses': {}}, + 'put': {'responses': {}}, + 'x-atom-usages': {'com.piggymetrics.statistics.controller.StatisticsController.getCurrentAccountStatistics': [20, + 22]}}, + '/latest': {'get': {'responses': {}}, + 'x-atom-usages': {'com.piggymetrics.statistics.client.ExchangeRatesClient.getRates': [13]}}, '/statistics/{accountName}': {'parameters': [{'in': 'path', 'name': 'accountName', 'required': True}], - 'put': {'responses': {}}}, - '/uaa/users': {'post': {'responses': {}}}, + 'put': {'responses': {}}, + 'x-atom-usages': {'com.piggymetrics.account.client.StatisticsServiceClient.updateStatistics': [13]}}, + '/uaa/users': {'post': {'responses': {}}, + 'x-atom-usages': {'com.piggymetrics.account.client.AuthServiceClient.createUser': [12]}}, '/{accountName}': {'get': {'responses': {}}, 'parameters': [{'in': 'path', 'name': 'accountName', 'required': True}], - 'put': {'responses': {}}}, + 'put': {'responses': {}}, + 'x-atom-usages': {'com.piggymetrics.statistics.controller.StatisticsController.saveAccountStatistics': [32]}}, '/{name}': {'get': {'responses': {}}, 'parameters': [{'in': 'path', 'name': 'name', - 'required': True}]}}} + 'required': True}], + 'x-atom-usages': {'com.piggymetrics.account.controller.AccountController.getAccountByName': [20]}}}}