diff --git a/setup.py b/setup.py index f6478a80..92517ae2 100644 --- a/setup.py +++ b/setup.py @@ -35,6 +35,7 @@ 'polib', 'pygments', 'requests', + 'selenium', 'twine', 'wheel', ], @@ -62,6 +63,7 @@ 'oca-migrate-branch = tools.migrate_branch:main', 'oca-migrate-branch-empty = tools.migrate_branch_empty:main', 'oca-main-branch-bot = tools.main_branch_bot:main', + 'oca-publish-modules = tools.publish_modules:main', 'oca-pypi-upload = tools.pypi_upload:cli', 'oca-dist-to-simple-index = tools.dist_to_simple_index:main', 'oca-gen-addon-readme = tools.gen_addon_readme:gen_addon_readme', diff --git a/tools/config.py b/tools/config.py index 9dc9300f..247ec0c1 100644 --- a/tools/config.py +++ b/tools/config.py @@ -16,6 +16,13 @@ def init_config(): config.add_section("odoo") config.set("odoo", "username", "") config.set("odoo", "password", "") + config.add_section("apps.odoo.com") + config.set("apps.odoo.com", "username", "") + config.set("apps.odoo.com", "password", "") + config.set( + "apps.odoo.com", "chromedriver_path", + "/usr/lib/chromium-browser/chromedriver", + ) write_config(config) diff --git a/tools/oca_publish_modules.py b/tools/oca_publish_modules.py deleted file mode 100755 index 9107aff7..00000000 --- a/tools/oca_publish_modules.py +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env python -# Copyright 2019 Brainbean Apps (https://brainbeanapps.com) -# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). - -import os -import logging -from getpass import getpass -from selenium import webdriver -from selenium.webdriver.support.ui import WebDriverWait -from selenium.webdriver.chrome.options import Options - -from .oca_projects import get_repositories_and_branches, url - -_logger = logging.getLogger(__name__) - - -def main(argv): - chrome = os.getenv('CHROMEDRIVER') - if not chrome: - print( - 'Please specify CHROMEDRIVER environment variable pointing to' - ' valid Google Chrome Driver executable file and ensure its' - ' location is available via PATH environment variable.\n\n' - 'See http://chromedriver.chromium.org/downloads' - ) - return - - login = input('Odoo.com publisher account:') - password = getpass(prompt='Odoo.com account password:') - - for repository, branch in get_repositories_and_branches(): - try: - repository_url = url(repository) + '#' + branch - print('Publishing %s#%s from %s...' % ( - repository, - branch, - repository_url, - )) - publish_repository(chrome, login, password, repository_url) - except Exception as ex: - _logger.debug('Error: %s', ex) - print('Repository publishing failed!') - - -def publish_repository(chrome, login, password, repository_url): - options = Options() - options.headless = True - - driver = webdriver.Chrome( - executable_path=chrome, - options=options, - ) - - login(driver, login, password) - - try: - scan_repository(driver, repository_url) - except Exception as ex: - _logger.debug('Scanning failed: %s', ex) - register_repository(driver, repository_url) - scan_repository(driver, repository_url) - - -def login(driver, login, password): - wait = WebDriverWait(driver, 10) - - driver.get( - 'https://www.odoo.com/web/login?redirect=%2Foauth2%2Fauth%2F%3Fscope' - '%3Duserinfo%26redirect_uri%3Dhttps%253A%252F%252Fapps.odoo.com%252F' - 'auth_oauth%252Fsignin%26state%3D%257B%2522p%2522%253A%2B1%252C%2B' - '%2522r%2522%253A%2B%2522%25252F%25252Fapps.odoo.com%25252Fapps%25' - '22%252C%2B%2522d%2522%253A%2B%2522apps%2522%257D%26response_type%3D' - 'token%26client_id%3Da0a30d16-6095-11e2-9c70-002590a17fd8&scope=user' - 'info&mode=login&redirect_hostname=https%3A%2F%2Fapps.odoo.com&login=' - ) - login_field = driver.find_element_by_id('login') - login_field.clear() - login_field.send_keys(login) - password_field = driver.find_element_by_id('password') - password_field.clear() - password_field.send_keys(password) - login_button = driver.find_element_by_xpath( - './/form[@action="/web/login"]//button[@type="submit"]' - ) - login_button.click() - wait.until( - lambda driver: driver.current_url == 'https://apps.odoo.com/apps' - ) - - -def register_repository(driver, repository): - wait = WebDriverWait(driver, 10) - - driver.get('https://apps.odoo.com/apps/upload') - url_field = driver.find_element_by_id('url') - url_field.clear() - url_field.send_keys(repository) - submit_button = driver.find_element_by_id('apps_submit_repo_button') - submit_button.click() - wait.until( - lambda driver: driver.current_url - == 'https://apps.odoo.com/apps/dashboard/repos' - ) - - -def scan_repository(driver, repository): - item_container = driver.find_element_by_xpath( - './/span[@id="repo_url" and text()="%s"]' - '/ancestor::li[1]' % ( - repository - ) - ) - auto_scan_checkbox = item_container.find_element_by_xpath( - './/input[@name="auto_scan"]' - ) - if not auto_scan_checkbox.is_selected(): - auto_scan_checkbox.click() - scan_link = item_container.find_element_by_class_name('js_repo_scan') - scan_link.click_and_wait(5000) - - -if __name__ == '__main__': - main() diff --git a/tools/publish_modules.py b/tools/publish_modules.py new file mode 100755 index 00000000..4ab07b6e --- /dev/null +++ b/tools/publish_modules.py @@ -0,0 +1,233 @@ +# Copyright 2019 Brainbean Apps (https://brainbeanapps.com) +# Copyright 2019 Tecnativa - Pedro M. Baeza +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). +""" +This script helps you to add all the OCA repositories in batch in apps.odoo.com +platform. For now it's not adapted to other organization as it takes the +information from static mapping here. It shouldn't be too much difficult to +adapt this to other organizations. + +It also serves as status scanner for knowing which repos are not yet being +automatically scanned (showing the conflict but ommited in case of simply +empty repo), and allowing to force the scan in that moment. Note that current +platform auto-scan all the repositories daily, so this operation is not really +needed except being in a hurry. + +WARNING: This work is based on current platform implementation. This might +stop working if Odoo changes it through time. + +Installation +============ + +It requires to have Python library `selenium`, that you can install regularly +through pip doing `sudo pip3 install selenium` or using specific OS packages. + +It also requires chromedriver binary. If you are in Ubuntu or derivative, you +can do: + +`sudo apt-get install chromium-chromedriver` + +You would need to enter afterwards the path for this binary field. Installing +the package above, the path is /usr/lib/chromium-browser/chromedriver. + +Configuration +============= + +You can have a file called oca.cfg on the same folder of the script for +storing credentials parameters. You can generate an skeleton config running +this script for a first time. + +The credentials are stored in the section [apps.odoo.com], with the names +"username", "password" and "chromedriver_path", which are self-explanatory. + +If not set, a prompt will ask you to enter user and password. You would also +need to specify through an environment variable called "CHROMEDRIVER" the +chromedriver path if not specified in the config file. + +Usage +===== + +oca-publish-modules [OPTIONS] + +Options: + --branch TEXT Limit to specific Odoo series. eg 11.0. + --repository TEXT Limit to a repository. eg contract. + --registration / --no-registration + Perform the registration of repositories. + --status / --no-status Retrieve the status of bad repositories. + --force-scan / --no-force-scan If auto-scan not activated, activate it and + perform a scan in that moment. + --scan-skip-empty / --scan-no-skip-empty + Skip scan of empty repositories (no matter + force scan value). + --help Show the help. +""" + +import os +import logging +from getpass import getpass +import click +from selenium.common.exceptions import NoSuchElementException +from selenium import webdriver +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.chrome.options import Options + +from .oca_projects import get_repositories_and_branches, url +from .config import read_config, write_config + +_logger = logging.getLogger(__name__) + + +@click.command() +@click.option('--branch', 'target_branch', + help="Limit to specific Odoo series. eg 11.0.") +@click.option('--repository', 'target_repository', + help="Limit to a repository. eg contract.") +@click.option('--registration/--no-registration', 'do_registration', + default=True, + help="Perform the registration of repositories.") +@click.option('--status/--no-status', 'do_status', + default=True, + help="Retrieve the status of bad repositories.") +@click.option('--force-scan/--no-force-scan', 'force_scan', + default=False, + help="If auto-scan not activated, activate it and perform a " + "scan in that moment.") +@click.option('--scan-skip-empty/--scan-no-skip-empty', 'scan_skip_empty', + default=True, + help="Skip scan of empty repositories (no matter force scan " + "value).") +def main(target_branch, target_repository, do_registration, do_status, + force_scan, scan_skip_empty): + config = read_config() + chrome = config.get('apps.odoo.com', 'chromedriver_path') + if not chrome: + chrome = os.getenv('CHROMEDRIVER') + if not chrome: + print( + 'Please specify CHROMEDRIVER environment variable or config' + ' option pointing to a valid Google Chrome Driver executable file' + ' and ensure its location is available via PATH environment' + ' variable.\n\nSee http://chromedriver.chromium.org/downloads' + ) + return + user = config.get('apps.odoo.com', 'username') + if not user: + user = input('Odoo.com publisher account:') + password = config.get('apps.odoo.com', 'password') + if not password: + password = getpass(prompt='Odoo.com account password:') + # Selenium options + options = Options() + options.headless = True + driver = webdriver.Chrome( + executable_path=chrome, + options=options, + ) + login(driver, user, password) + # First pass: register all repositories (if already registered, there + # will be an immediate warning in the current browser page and won't + # continue, so we can simply overwrite the value in the field). + if do_registration: + for repository, branch in get_repositories_and_branches(): + if target_branch and branch != target_branch: + continue + if target_repository and target_repository != repository: + continue + repository_url = url(repository) + '#' + branch + print('INFO: Adding %s#%s from %s... (if not yet present)' % ( + repository, + branch, + repository_url, + )) + register_repository(driver, repository_url) + # Second pass: check published state and try to publish if not yet done + if do_status: + driver.get('https://apps.odoo.com/apps/dashboard/repos') + for repository, branch in get_repositories_and_branches(): + if target_branch and branch != target_branch: + continue + if target_repository and target_repository != repository: + continue + repository_url = url(repository) + '#' + branch + scan_repository( + driver, repository_url, force_scan, scan_skip_empty, + ) + + +def login(driver, user, password): + wait = WebDriverWait(driver, 10) + driver.get( + 'https://www.odoo.com/web/login?redirect=%2Foauth2%2Fauth%2F%3Fscope' + '%3Duserinfo%26redirect_uri%3Dhttps%253A%252F%252Fapps.odoo.com%252F' + 'auth_oauth%252Fsignin%26state%3D%257B%2522p%2522%253A%2B1%252C%2B' + '%2522r%2522%253A%2B%2522%25252F%25252Fapps.odoo.com%25252Fapps%25' + '22%252C%2B%2522d%2522%253A%2B%2522apps%2522%257D%26response_type%3D' + 'token%26client_id%3Da0a30d16-6095-11e2-9c70-002590a17fd8&scope=user' + 'info&mode=login&redirect_hostname=https%3A%2F%2Fapps.odoo.com&login=' + ) + login_field = driver.find_element_by_id('login') + login_field.clear() + login_field.send_keys(user) + password_field = driver.find_element_by_id('password') + password_field.clear() + password_field.send_keys(password) + login_button = driver.find_element_by_xpath( + './/form[@action="/web/login"]//button[@type="submit"]' + ) + login_button.click() + wait.until( + lambda driver: driver.current_url == 'https://apps.odoo.com/apps' + ) + + +def register_repository(driver, repository): + driver.get('https://apps.odoo.com/apps/upload') + url_field = driver.find_element_by_name('url') + url_field.clear() + url_field.send_keys(repository) + submit_button = driver.find_element_by_id('apps_submit_repo_button') + submit_button.click() + + +def scan_repository(driver, repository_url, force_scan, scan_skip_empty): + wait = WebDriverWait(driver, 300) + try: + item_container = driver.find_element_by_xpath( + './/span[@id="repo_url" and text()="%s"]' + '/ancestor::li[1]' % repository_url + ) + except NoSuchElementException: + print("WARNING: %s not registered in this account." % repository_url) + return + try: + error_item = item_container.find_element_by_xpath( + './/div[@id="help_error"]', + ) + except NoSuchElementException: + error_item = False + is_empty = False + if error_item: + error_text = error_item.find_element_by_xpath('.//p') + is_empty = error_text.text.startswith('No module found in repository') + if not is_empty: + print("ERROR: %s:\n%s" % (repository_url, error_text.text)) + if force_scan and not (is_empty and scan_skip_empty): + print("INFO: Doing the scan on %s" % repository_url) + auto_scan_checkbox = item_container.find_element_by_xpath( + './/input[@name="auto_scan"]' + ) + if not auto_scan_checkbox.is_selected(): + auto_scan_checkbox.click() + scan_link = item_container.find_element_by_class_name( + 'js_repo_scan' + ) + scan_link.click() + wait.until( + lambda driver: driver.current_url + == 'https://apps.odoo.com/apps/dashboard/repos' + ) + + +if __name__ == '__main__': + main()