From 96b06eb6413d9c2a3e54dc299b711bce308a03a4 Mon Sep 17 00:00:00 2001 From: Alexey Pelykh Date: Thu, 28 Feb 2019 23:02:18 +0200 Subject: [PATCH 1/4] [IMP] oca-publish-modules: tool for publishing modules --- setup.py | 2 + tools/config.py | 7 ++ tools/publish_modules.py | 234 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 243 insertions(+) create mode 100755 tools/publish_modules.py diff --git a/setup.py b/setup.py index b5771ca5..4354a3d4 100644 --- a/setup.py +++ b/setup.py @@ -40,6 +40,7 @@ 'requests', 'toml>=0.10.0', # for oca-towncrier 'towncrier>=19.2', # for oca-towncrier + 'selenium', 'twine', 'wheel', ], @@ -66,6 +67,7 @@ 'oca-gen-addons-table = tools.gen_addons_table:gen_addons_table', 'oca-migrate-branch = tools.migrate_branch:main', 'oca-migrate-branch-empty = tools.migrate_branch_empty:main', + 'oca-publish-modules = tools.publish_modules:main', 'oca-pypi-upload = tools.pypi_upload:cli', 'oca-gen-addon-readme = tools.gen_addon_readme:gen_addon_readme', 'oca-gen-addon-icon = tools.gen_addon_icon:gen_addon_icon', diff --git a/tools/config.py b/tools/config.py index 0ace1194..35635807 100644 --- a/tools/config.py +++ b/tools/config.py @@ -16,6 +16,13 @@ def init_config(): config.add_section("odoo") config.set("odoo", "username", "") config.set("odoo", "password", "") + config.add_section("apps.odoo.com") + config.set("apps.odoo.com", "username", "") + config.set("apps.odoo.com", "password", "") + config.set( + "apps.odoo.com", "chromedriver_path", + "/usr/lib/chromium-browser/chromedriver", + ) write_config(config) diff --git a/tools/publish_modules.py b/tools/publish_modules.py new file mode 100755 index 00000000..d5add6bd --- /dev/null +++ b/tools/publish_modules.py @@ -0,0 +1,234 @@ +# Copyright 2019 Brainbean Apps (https://brainbeanapps.com) +# Copyright 2019 Tecnativa - Pedro M. Baeza +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). +""" +This script helps you to add all the OCA repositories in batch in apps.odoo.com +platform. For now it's not adapted to other organization as it takes the +information from static mapping here. It shouldn't be too much difficult to +adapt this to other organizations. + +It also serves as status scanner for knowing which repos are not yet being +automatically scanned (showing the conflict but ommited in case of simply +empty repo), and allowing to force the scan in that moment. Note that current +platform auto-scan all the repositories daily, so this operation is not really +needed except being in a hurry. + +WARNING: This work is based on current platform implementation. This might +stop working if Odoo changes it through time. + +Installation +============ + +It requires to have Python library `selenium`, that you can install regularly +through pip doing `sudo pip3 install selenium` or using specific OS packages. + +It also requires chromedriver binary. If you are in Ubuntu or derivative, you +can do: + +`sudo apt-get install chromium-chromedriver` + +You would need to enter afterwards the path for this binary field. Installing +the package above, the path is /usr/lib/chromium-browser/chromedriver. + +Configuration +============= + +You can have a file called oca.cfg on the same folder of the script for +storing credentials parameters. You can generate an skeleton config running +this script for a first time. + +The credentials are stored in the section [apps.odoo.com], with the names +"username", "password" and "chromedriver_path", which are self-explanatory. + +If not set, a prompt will ask you to enter user and password. You would also +need to specify through an environment variable called "CHROMEDRIVER" the +chromedriver path if not specified in the config file. + +Usage +===== + +oca-publish-modules [OPTIONS] + +Options: + --branch TEXT Limit to specific Odoo series. eg 11.0. + --repository TEXT Limit to a repository. eg contract. + --registration / --no-registration + Perform the registration of repositories. + --status / --no-status Retrieve the status of bad repositories. + --force-scan / --no-force-scan If auto-scan not activated, activate it and + perform a scan in that moment. + --scan-skip-empty / --scan-no-skip-empty + Skip scan of empty repositories (no matter + force scan value). + --help Show the help. +""" + +from __future__ import print_function +import os +import logging +from getpass import getpass +import click +from selenium.common.exceptions import NoSuchElementException +from selenium import webdriver +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.chrome.options import Options + +from .oca_projects import get_repositories_and_branches, url +from .config import read_config + +_logger = logging.getLogger(__name__) + + +@click.command() +@click.option('--branch', 'target_branch', + help="Limit to specific Odoo series. eg 11.0.") +@click.option('--repository', 'target_repository', + help="Limit to a repository. eg contract.") +@click.option('--registration/--no-registration', 'do_registration', + default=True, + help="Perform the registration of repositories.") +@click.option('--status/--no-status', 'do_status', + default=True, + help="Retrieve the status of bad repositories.") +@click.option('--force-scan/--no-force-scan', 'force_scan', + default=False, + help="If auto-scan not activated, activate it and perform a " + "scan in that moment.") +@click.option('--scan-skip-empty/--scan-no-skip-empty', 'scan_skip_empty', + default=True, + help="Skip scan of empty repositories (no matter force scan " + "value).") +def main(target_branch, target_repository, do_registration, do_status, + force_scan, scan_skip_empty): + config = read_config() + chrome = config.get('apps.odoo.com', 'chromedriver_path') + if not chrome: + chrome = os.getenv('CHROMEDRIVER') + if not chrome: + print( + 'Please specify CHROMEDRIVER environment variable or config' + ' option pointing to a valid Google Chrome Driver executable file' + ' and ensure its location is available via PATH environment' + ' variable.\n\nSee http://chromedriver.chromium.org/downloads' + ) + return + user = config.get('apps.odoo.com', 'username') + if not user: + user = input('Odoo.com publisher account:') + password = config.get('apps.odoo.com', 'password') + if not password: + password = getpass(prompt='Odoo.com account password:') + # Selenium options + options = Options() + options.headless = True + driver = webdriver.Chrome( + executable_path=chrome, + options=options, + ) + login(driver, user, password) + # First pass: register all repositories (if already registered, there + # will be an immediate warning in the current browser page and won't + # continue, so we can simply overwrite the value in the field). + if do_registration: + for repository, branch in get_repositories_and_branches(): + if target_branch and branch != target_branch: + continue + if target_repository and target_repository != repository: + continue + repository_url = url(repository) + '#' + branch + print('INFO: Adding %s#%s from %s... (if not yet present)' % ( + repository, + branch, + repository_url, + )) + register_repository(driver, repository_url) + # Second pass: check published state and try to publish if not yet done + if do_status: + driver.get('https://apps.odoo.com/apps/dashboard/repos') + for repository, branch in get_repositories_and_branches(): + if target_branch and branch != target_branch: + continue + if target_repository and target_repository != repository: + continue + repository_url = url(repository) + '#' + branch + scan_repository( + driver, repository_url, force_scan, scan_skip_empty, + ) + + +def login(driver, user, password): + wait = WebDriverWait(driver, 10) + driver.get( + 'https://www.odoo.com/web/login?redirect=%2Foauth2%2Fauth%2F%3Fscope' + '%3Duserinfo%26redirect_uri%3Dhttps%253A%252F%252Fapps.odoo.com%252F' + 'auth_oauth%252Fsignin%26state%3D%257B%2522p%2522%253A%2B1%252C%2B' + '%2522r%2522%253A%2B%2522%25252F%25252Fapps.odoo.com%25252Fapps%25' + '22%252C%2B%2522d%2522%253A%2B%2522apps%2522%257D%26response_type%3D' + 'token%26client_id%3Da0a30d16-6095-11e2-9c70-002590a17fd8&scope=user' + 'info&mode=login&redirect_hostname=https%3A%2F%2Fapps.odoo.com&login=' + ) + login_field = driver.find_element_by_id('login') + login_field.clear() + login_field.send_keys(user) + password_field = driver.find_element_by_id('password') + password_field.clear() + password_field.send_keys(password) + login_button = driver.find_element_by_xpath( + './/form[@action="/web/login"]//button[@type="submit"]' + ) + login_button.click() + wait.until( + lambda driver: driver.current_url == 'https://apps.odoo.com/apps' + ) + + +def register_repository(driver, repository): + driver.get('https://apps.odoo.com/apps/upload') + url_field = driver.find_element_by_name('url') + url_field.clear() + url_field.send_keys(repository) + submit_button = driver.find_element_by_id('apps_submit_repo_button') + submit_button.click() + + +def scan_repository(driver, repository_url, force_scan, scan_skip_empty): + wait = WebDriverWait(driver, 300) + try: + item_container = driver.find_element_by_xpath( + './/span[@id="repo_url" and text()="%s"]' + '/ancestor::li[1]' % repository_url + ) + except NoSuchElementException: + print("WARNING: %s not registered in this account." % repository_url) + return + try: + error_item = item_container.find_element_by_xpath( + './/div[@id="help_error"]', + ) + except NoSuchElementException: + error_item = False + is_empty = False + if error_item: + error_text = error_item.find_element_by_xpath('.//p') + is_empty = error_text.text.startswith('No module found in repository') + if not is_empty: + print("ERROR: %s:\n%s" % (repository_url, error_text.text)) + if force_scan and not (is_empty and scan_skip_empty): + print("INFO: Doing the scan on %s" % repository_url) + auto_scan_checkbox = item_container.find_element_by_xpath( + './/input[@name="auto_scan"]' + ) + if not auto_scan_checkbox.is_selected(): + auto_scan_checkbox.click() + scan_link = item_container.find_element_by_class_name( + 'js_repo_scan' + ) + scan_link.click() + wait.until( + lambda driver: driver.current_url + == 'https://apps.odoo.com/apps/dashboard/repos' + ) + + +if __name__ == '__main__': + main() From 597b3efbd6cbb853c60a0499e310039b62eddcb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 13 Jun 2020 12:44:07 +0200 Subject: [PATCH 2/4] add ssh protocol support in url mappings --- tools/oca_projects.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/oca_projects.py b/tools/oca_projects.py index e74d3443..864fe44c 100644 --- a/tools/oca_projects.py +++ b/tools/oca_projects.py @@ -242,6 +242,7 @@ def get_repositories_and_branches(repos=(), branches=MAIN_BRANCHES): _OCA_REPOSITORY_NAMES = set(OCA_REPOSITORY_NAMES) _URL_MAPPINGS = {'git': 'git@github.com:%s/%s.git', + 'ssh': 'ssh://git@github.com/%s/%s.git', 'https': 'https://github.com/%s/%s.git', } From e31a40585935a8ed208e58c6fa965e570c7bf582 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 13 Jun 2020 13:03:40 +0200 Subject: [PATCH 3/4] Tweak oca-publish-modules Make it work with current apps.odoo.com. - ssh/https url support - avoid multi-page search results on apps.odoo.com - some scraper update to match current apps.odoo.com Assume chromedriver is in path (could not make it work otherwise) --- tools/publish_modules.py | 68 ++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 37 deletions(-) diff --git a/tools/publish_modules.py b/tools/publish_modules.py index d5add6bd..f03cb225 100755 --- a/tools/publish_modules.py +++ b/tools/publish_modules.py @@ -27,8 +27,7 @@ `sudo apt-get install chromium-chromedriver` -You would need to enter afterwards the path for this binary field. Installing -the package above, the path is /usr/lib/chromium-browser/chromedriver. +The 'chromedriver' executable must be in your PATH. Configuration ============= @@ -38,11 +37,9 @@ this script for a first time. The credentials are stored in the section [apps.odoo.com], with the names -"username", "password" and "chromedriver_path", which are self-explanatory. +"username" and "password", which are self-explanatory. -If not set, a prompt will ask you to enter user and password. You would also -need to specify through an environment variable called "CHROMEDRIVER" the -chromedriver path if not specified in the config file. +If not set, a prompt will ask you to enter user and password. Usage ===== @@ -64,7 +61,6 @@ """ from __future__ import print_function -import os import logging from getpass import getpass import click @@ -84,6 +80,7 @@ help="Limit to specific Odoo series. eg 11.0.") @click.option('--repository', 'target_repository', help="Limit to a repository. eg contract.") +@click.option('--org', help="GitHub organization (default=OCA)", default="OCA") @click.option('--registration/--no-registration', 'do_registration', default=True, help="Perform the registration of repositories.") @@ -92,26 +89,15 @@ help="Retrieve the status of bad repositories.") @click.option('--force-scan/--no-force-scan', 'force_scan', default=False, - help="If auto-scan not activated, activate it and perform a " - "scan in that moment.") + help="If auto-scan not activated, activate it and request " + "apps.odoo.com to perform a scan in that moment.") @click.option('--scan-skip-empty/--scan-no-skip-empty', 'scan_skip_empty', default=True, help="Skip scan of empty repositories (no matter force scan " "value).") -def main(target_branch, target_repository, do_registration, do_status, +def main(target_branch, target_repository, org, do_registration, do_status, force_scan, scan_skip_empty): config = read_config() - chrome = config.get('apps.odoo.com', 'chromedriver_path') - if not chrome: - chrome = os.getenv('CHROMEDRIVER') - if not chrome: - print( - 'Please specify CHROMEDRIVER environment variable or config' - ' option pointing to a valid Google Chrome Driver executable file' - ' and ensure its location is available via PATH environment' - ' variable.\n\nSee http://chromedriver.chromium.org/downloads' - ) - return user = config.get('apps.odoo.com', 'username') if not user: user = input('Odoo.com publisher account:') @@ -122,7 +108,6 @@ def main(target_branch, target_repository, do_registration, do_status, options = Options() options.headless = True driver = webdriver.Chrome( - executable_path=chrome, options=options, ) login(driver, user, password) @@ -144,15 +129,18 @@ def main(target_branch, target_repository, do_registration, do_status, register_repository(driver, repository_url) # Second pass: check published state and try to publish if not yet done if do_status: - driver.get('https://apps.odoo.com/apps/dashboard/repos') for repository, branch in get_repositories_and_branches(): if target_branch and branch != target_branch: continue if target_repository and target_repository != repository: continue - repository_url = url(repository) + '#' + branch + # assume this query returns everything we need in one page + driver.get( + f'https://apps.odoo.com/apps/dashboard/repos?' + f'search_in=url&search={org}/{repository}' + ) scan_repository( - driver, repository_url, force_scan, scan_skip_empty, + driver, org, repository, branch, force_scan, scan_skip_empty, ) @@ -191,28 +179,34 @@ def register_repository(driver, repository): submit_button.click() -def scan_repository(driver, repository_url, force_scan, scan_skip_empty): +def scan_repository(driver, org, repository, branch, force_scan, scan_skip_empty): wait = WebDriverWait(driver, 300) - try: - item_container = driver.find_element_by_xpath( - './/span[@id="repo_url" and text()="%s"]' - '/ancestor::li[1]' % repository_url - ) - except NoSuchElementException: - print("WARNING: %s not registered in this account." % repository_url) + for protocol in ('https', 'ssh'): + repository_url = url(repository, protocol='ssh', org_name=org) + '#' + branch + try: + item_container = driver.find_element_by_xpath( + './/span[@id="repo_url" and text()="%s"]' + '/ancestor::li[1]' % repository_url + ) + except NoSuchElementException: + pass + else: + break # found + else: + # not found + print(f"WARNING: {org}/{repository}#{branch} not registered in this account.") return try: error_item = item_container.find_element_by_xpath( - './/div[@id="help_error"]', + './/div[@id="help_error"]/div/p', ) except NoSuchElementException: error_item = False is_empty = False if error_item: - error_text = error_item.find_element_by_xpath('.//p') - is_empty = error_text.text.startswith('No module found in repository') + is_empty = error_item.text.startswith('No module found in repository') if not is_empty: - print("ERROR: %s:\n%s" % (repository_url, error_text.text)) + print("ERROR: %s:\n%s" % (repository_url, error_item.text)) if force_scan and not (is_empty and scan_skip_empty): print("INFO: Doing the scan on %s" % repository_url) auto_scan_checkbox = item_container.find_element_by_xpath( From acf06ff3de18aa2d90cb442a54d61fba9d320a71 Mon Sep 17 00:00:00 2001 From: Alexey Pelykh Date: Sun, 14 Jun 2020 08:20:08 +0200 Subject: [PATCH 4/4] Update tools/publish_modules.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Stéphane Bidoul --- tools/publish_modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/publish_modules.py b/tools/publish_modules.py index f03cb225..2ea39fc3 100755 --- a/tools/publish_modules.py +++ b/tools/publish_modules.py @@ -182,7 +182,7 @@ def register_repository(driver, repository): def scan_repository(driver, org, repository, branch, force_scan, scan_skip_empty): wait = WebDriverWait(driver, 300) for protocol in ('https', 'ssh'): - repository_url = url(repository, protocol='ssh', org_name=org) + '#' + branch + repository_url = url(repository, protocol=protocol, org_name=org) + '#' + branch try: item_container = driver.find_element_by_xpath( './/span[@id="repo_url" and text()="%s"]'